drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  66 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
  67 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
  68 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
  69 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
  70 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
  71 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
  72
  73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  75 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  76 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  77 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  78 extern void sumo_rlc_fini(struct radeon_device *rdev);
  79 extern int sumo_rlc_init(struct radeon_device *rdev);
  80 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  81 extern void si_rlc_reset(struct radeon_device *rdev);
  82 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
  83 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
  84 extern int cik_sdma_resume(struct radeon_device *rdev);
  85 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
  86 extern void cik_sdma_fini(struct radeon_device *rdev);
  87 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
  88 static void cik_rlc_stop(struct radeon_device *rdev);
  89 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  90 static void cik_program_aspm(struct radeon_device *rdev);
  91 static void cik_init_pg(struct radeon_device *rdev);
  92 static void cik_init_cg(struct radeon_device *rdev);
  93 static void cik_fini_pg(struct radeon_device *rdev);
  94 static void cik_fini_cg(struct radeon_device *rdev);
  95 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
  96                                           bool enable);
  97
  98 /* get temperature in millidegrees */
  99 int ci_get_temp(struct radeon_device *rdev)
 100 {
 101         u32 temp;
 102         int actual_temp = 0;
 103
 104         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 105                 CTF_TEMP_SHIFT;
 106
 107         if (temp & 0x200)
 108                 actual_temp = 255;
 109         else
 110                 actual_temp = temp & 0x1ff;
 111
 112         actual_temp = actual_temp * 1000;
 113
 114         return actual_temp;
 115 }
 116
 117 /* get temperature in millidegrees */
 118 int kv_get_temp(struct radeon_device *rdev)
 119 {
 120         u32 temp;
 121         int actual_temp = 0;
 122
 123         temp = RREG32_SMC(0xC0300E0C);
 124
 125         if (temp)
 126                 actual_temp = (temp / 8) - 49;
 127         else
 128                 actual_temp = 0;
 129
 130         actual_temp = actual_temp * 1000;
 131
 132         return actual_temp;
 133 }
 134
 135 /*
 136  * Indirect registers accessor
 137  */
 138 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 139 {
 140         unsigned long flags;
 141         u32 r;
 142
 143         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 144         WREG32(PCIE_INDEX, reg);
 145         (void)RREG32(PCIE_INDEX);
 146         r = RREG32(PCIE_DATA);
 147         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 148         return r;
 149 }
 150
 151 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 152 {
 153         unsigned long flags;
 154
 155         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 156         WREG32(PCIE_INDEX, reg);
 157         (void)RREG32(PCIE_INDEX);
 158         WREG32(PCIE_DATA, v);
 159         (void)RREG32(PCIE_DATA);
 160         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 161 }
 162
 163 static const u32 spectre_rlc_save_restore_register_list[] =
 164 {
 165         (0x0e00 << 16) | (0xc12c >> 2),
 166         0x00000000,
 167         (0x0e00 << 16) | (0xc140 >> 2),
 168         0x00000000,
 169         (0x0e00 << 16) | (0xc150 >> 2),
 170         0x00000000,
 171         (0x0e00 << 16) | (0xc15c >> 2),
 172         0x00000000,
 173         (0x0e00 << 16) | (0xc168 >> 2),
 174         0x00000000,
 175         (0x0e00 << 16) | (0xc170 >> 2),
 176         0x00000000,
 177         (0x0e00 << 16) | (0xc178 >> 2),
 178         0x00000000,
 179         (0x0e00 << 16) | (0xc204 >> 2),
 180         0x00000000,
 181         (0x0e00 << 16) | (0xc2b4 >> 2),
 182         0x00000000,
 183         (0x0e00 << 16) | (0xc2b8 >> 2),
 184         0x00000000,
 185         (0x0e00 << 16) | (0xc2bc >> 2),
 186         0x00000000,
 187         (0x0e00 << 16) | (0xc2c0 >> 2),
 188         0x00000000,
 189         (0x0e00 << 16) | (0x8228 >> 2),
 190         0x00000000,
 191         (0x0e00 << 16) | (0x829c >> 2),
 192         0x00000000,
 193         (0x0e00 << 16) | (0x869c >> 2),
 194         0x00000000,
 195         (0x0600 << 16) | (0x98f4 >> 2),
 196         0x00000000,
 197         (0x0e00 << 16) | (0x98f8 >> 2),
 198         0x00000000,
 199         (0x0e00 << 16) | (0x9900 >> 2),
 200         0x00000000,
 201         (0x0e00 << 16) | (0xc260 >> 2),
 202         0x00000000,
 203         (0x0e00 << 16) | (0x90e8 >> 2),
 204         0x00000000,
 205         (0x0e00 << 16) | (0x3c000 >> 2),
 206         0x00000000,
 207         (0x0e00 << 16) | (0x3c00c >> 2),
 208         0x00000000,
 209         (0x0e00 << 16) | (0x8c1c >> 2),
 210         0x00000000,
 211         (0x0e00 << 16) | (0x9700 >> 2),
 212         0x00000000,
 213         (0x0e00 << 16) | (0xcd20 >> 2),
 214         0x00000000,
 215         (0x4e00 << 16) | (0xcd20 >> 2),
 216         0x00000000,
 217         (0x5e00 << 16) | (0xcd20 >> 2),
 218         0x00000000,
 219         (0x6e00 << 16) | (0xcd20 >> 2),
 220         0x00000000,
 221         (0x7e00 << 16) | (0xcd20 >> 2),
 222         0x00000000,
 223         (0x8e00 << 16) | (0xcd20 >> 2),
 224         0x00000000,
 225         (0x9e00 << 16) | (0xcd20 >> 2),
 226         0x00000000,
 227         (0xae00 << 16) | (0xcd20 >> 2),
 228         0x00000000,
 229         (0xbe00 << 16) | (0xcd20 >> 2),
 230         0x00000000,
 231         (0x0e00 << 16) | (0x89bc >> 2),
 232         0x00000000,
 233         (0x0e00 << 16) | (0x8900 >> 2),
 234         0x00000000,
 235         0x3,
 236         (0x0e00 << 16) | (0xc130 >> 2),
 237         0x00000000,
 238         (0x0e00 << 16) | (0xc134 >> 2),
 239         0x00000000,
 240         (0x0e00 << 16) | (0xc1fc >> 2),
 241         0x00000000,
 242         (0x0e00 << 16) | (0xc208 >> 2),
 243         0x00000000,
 244         (0x0e00 << 16) | (0xc264 >> 2),
 245         0x00000000,
 246         (0x0e00 << 16) | (0xc268 >> 2),
 247         0x00000000,
 248         (0x0e00 << 16) | (0xc26c >> 2),
 249         0x00000000,
 250         (0x0e00 << 16) | (0xc270 >> 2),
 251         0x00000000,
 252         (0x0e00 << 16) | (0xc274 >> 2),
 253         0x00000000,
 254         (0x0e00 << 16) | (0xc278 >> 2),
 255         0x00000000,
 256         (0x0e00 << 16) | (0xc27c >> 2),
 257         0x00000000,
 258         (0x0e00 << 16) | (0xc280 >> 2),
 259         0x00000000,
 260         (0x0e00 << 16) | (0xc284 >> 2),
 261         0x00000000,
 262         (0x0e00 << 16) | (0xc288 >> 2),
 263         0x00000000,
 264         (0x0e00 << 16) | (0xc28c >> 2),
 265         0x00000000,
 266         (0x0e00 << 16) | (0xc290 >> 2),
 267         0x00000000,
 268         (0x0e00 << 16) | (0xc294 >> 2),
 269         0x00000000,
 270         (0x0e00 << 16) | (0xc298 >> 2),
 271         0x00000000,
 272         (0x0e00 << 16) | (0xc29c >> 2),
 273         0x00000000,
 274         (0x0e00 << 16) | (0xc2a0 >> 2),
 275         0x00000000,
 276         (0x0e00 << 16) | (0xc2a4 >> 2),
 277         0x00000000,
 278         (0x0e00 << 16) | (0xc2a8 >> 2),
 279         0x00000000,
 280         (0x0e00 << 16) | (0xc2ac  >> 2),
 281         0x00000000,
 282         (0x0e00 << 16) | (0xc2b0 >> 2),
 283         0x00000000,
 284         (0x0e00 << 16) | (0x301d0 >> 2),
 285         0x00000000,
 286         (0x0e00 << 16) | (0x30238 >> 2),
 287         0x00000000,
 288         (0x0e00 << 16) | (0x30250 >> 2),
 289         0x00000000,
 290         (0x0e00 << 16) | (0x30254 >> 2),
 291         0x00000000,
 292         (0x0e00 << 16) | (0x30258 >> 2),
 293         0x00000000,
 294         (0x0e00 << 16) | (0x3025c >> 2),
 295         0x00000000,
 296         (0x4e00 << 16) | (0xc900 >> 2),
 297         0x00000000,
 298         (0x5e00 << 16) | (0xc900 >> 2),
 299         0x00000000,
 300         (0x6e00 << 16) | (0xc900 >> 2),
 301         0x00000000,
 302         (0x7e00 << 16) | (0xc900 >> 2),
 303         0x00000000,
 304         (0x8e00 << 16) | (0xc900 >> 2),
 305         0x00000000,
 306         (0x9e00 << 16) | (0xc900 >> 2),
 307         0x00000000,
 308         (0xae00 << 16) | (0xc900 >> 2),
 309         0x00000000,
 310         (0xbe00 << 16) | (0xc900 >> 2),
 311         0x00000000,
 312         (0x4e00 << 16) | (0xc904 >> 2),
 313         0x00000000,
 314         (0x5e00 << 16) | (0xc904 >> 2),
 315         0x00000000,
 316         (0x6e00 << 16) | (0xc904 >> 2),
 317         0x00000000,
 318         (0x7e00 << 16) | (0xc904 >> 2),
 319         0x00000000,
 320         (0x8e00 << 16) | (0xc904 >> 2),
 321         0x00000000,
 322         (0x9e00 << 16) | (0xc904 >> 2),
 323         0x00000000,
 324         (0xae00 << 16) | (0xc904 >> 2),
 325         0x00000000,
 326         (0xbe00 << 16) | (0xc904 >> 2),
 327         0x00000000,
 328         (0x4e00 << 16) | (0xc908 >> 2),
 329         0x00000000,
 330         (0x5e00 << 16) | (0xc908 >> 2),
 331         0x00000000,
 332         (0x6e00 << 16) | (0xc908 >> 2),
 333         0x00000000,
 334         (0x7e00 << 16) | (0xc908 >> 2),
 335         0x00000000,
 336         (0x8e00 << 16) | (0xc908 >> 2),
 337         0x00000000,
 338         (0x9e00 << 16) | (0xc908 >> 2),
 339         0x00000000,
 340         (0xae00 << 16) | (0xc908 >> 2),
 341         0x00000000,
 342         (0xbe00 << 16) | (0xc908 >> 2),
 343         0x00000000,
 344         (0x4e00 << 16) | (0xc90c >> 2),
 345         0x00000000,
 346         (0x5e00 << 16) | (0xc90c >> 2),
 347         0x00000000,
 348         (0x6e00 << 16) | (0xc90c >> 2),
 349         0x00000000,
 350         (0x7e00 << 16) | (0xc90c >> 2),
 351         0x00000000,
 352         (0x8e00 << 16) | (0xc90c >> 2),
 353         0x00000000,
 354         (0x9e00 << 16) | (0xc90c >> 2),
 355         0x00000000,
 356         (0xae00 << 16) | (0xc90c >> 2),
 357         0x00000000,
 358         (0xbe00 << 16) | (0xc90c >> 2),
 359         0x00000000,
 360         (0x4e00 << 16) | (0xc910 >> 2),
 361         0x00000000,
 362         (0x5e00 << 16) | (0xc910 >> 2),
 363         0x00000000,
 364         (0x6e00 << 16) | (0xc910 >> 2),
 365         0x00000000,
 366         (0x7e00 << 16) | (0xc910 >> 2),
 367         0x00000000,
 368         (0x8e00 << 16) | (0xc910 >> 2),
 369         0x00000000,
 370         (0x9e00 << 16) | (0xc910 >> 2),
 371         0x00000000,
 372         (0xae00 << 16) | (0xc910 >> 2),
 373         0x00000000,
 374         (0xbe00 << 16) | (0xc910 >> 2),
 375         0x00000000,
 376         (0x0e00 << 16) | (0xc99c >> 2),
 377         0x00000000,
 378         (0x0e00 << 16) | (0x9834 >> 2),
 379         0x00000000,
 380         (0x0000 << 16) | (0x30f00 >> 2),
 381         0x00000000,
 382         (0x0001 << 16) | (0x30f00 >> 2),
 383         0x00000000,
 384         (0x0000 << 16) | (0x30f04 >> 2),
 385         0x00000000,
 386         (0x0001 << 16) | (0x30f04 >> 2),
 387         0x00000000,
 388         (0x0000 << 16) | (0x30f08 >> 2),
 389         0x00000000,
 390         (0x0001 << 16) | (0x30f08 >> 2),
 391         0x00000000,
 392         (0x0000 << 16) | (0x30f0c >> 2),
 393         0x00000000,
 394         (0x0001 << 16) | (0x30f0c >> 2),
 395         0x00000000,
 396         (0x0600 << 16) | (0x9b7c >> 2),
 397         0x00000000,
 398         (0x0e00 << 16) | (0x8a14 >> 2),
 399         0x00000000,
 400         (0x0e00 << 16) | (0x8a18 >> 2),
 401         0x00000000,
 402         (0x0600 << 16) | (0x30a00 >> 2),
 403         0x00000000,
 404         (0x0e00 << 16) | (0x8bf0 >> 2),
 405         0x00000000,
 406         (0x0e00 << 16) | (0x8bcc >> 2),
 407         0x00000000,
 408         (0x0e00 << 16) | (0x8b24 >> 2),
 409         0x00000000,
 410         (0x0e00 << 16) | (0x30a04 >> 2),
 411         0x00000000,
 412         (0x0600 << 16) | (0x30a10 >> 2),
 413         0x00000000,
 414         (0x0600 << 16) | (0x30a14 >> 2),
 415         0x00000000,
 416         (0x0600 << 16) | (0x30a18 >> 2),
 417         0x00000000,
 418         (0x0600 << 16) | (0x30a2c >> 2),
 419         0x00000000,
 420         (0x0e00 << 16) | (0xc700 >> 2),
 421         0x00000000,
 422         (0x0e00 << 16) | (0xc704 >> 2),
 423         0x00000000,
 424         (0x0e00 << 16) | (0xc708 >> 2),
 425         0x00000000,
 426         (0x0e00 << 16) | (0xc768 >> 2),
 427         0x00000000,
 428         (0x0400 << 16) | (0xc770 >> 2),
 429         0x00000000,
 430         (0x0400 << 16) | (0xc774 >> 2),
 431         0x00000000,
 432         (0x0400 << 16) | (0xc778 >> 2),
 433         0x00000000,
 434         (0x0400 << 16) | (0xc77c >> 2),
 435         0x00000000,
 436         (0x0400 << 16) | (0xc780 >> 2),
 437         0x00000000,
 438         (0x0400 << 16) | (0xc784 >> 2),
 439         0x00000000,
 440         (0x0400 << 16) | (0xc788 >> 2),
 441         0x00000000,
 442         (0x0400 << 16) | (0xc78c >> 2),
 443         0x00000000,
 444         (0x0400 << 16) | (0xc798 >> 2),
 445         0x00000000,
 446         (0x0400 << 16) | (0xc79c >> 2),
 447         0x00000000,
 448         (0x0400 << 16) | (0xc7a0 >> 2),
 449         0x00000000,
 450         (0x0400 << 16) | (0xc7a4 >> 2),
 451         0x00000000,
 452         (0x0400 << 16) | (0xc7a8 >> 2),
 453         0x00000000,
 454         (0x0400 << 16) | (0xc7ac >> 2),
 455         0x00000000,
 456         (0x0400 << 16) | (0xc7b0 >> 2),
 457         0x00000000,
 458         (0x0400 << 16) | (0xc7b4 >> 2),
 459         0x00000000,
 460         (0x0e00 << 16) | (0x9100 >> 2),
 461         0x00000000,
 462         (0x0e00 << 16) | (0x3c010 >> 2),
 463         0x00000000,
 464         (0x0e00 << 16) | (0x92a8 >> 2),
 465         0x00000000,
 466         (0x0e00 << 16) | (0x92ac >> 2),
 467         0x00000000,
 468         (0x0e00 << 16) | (0x92b4 >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0x92b8 >> 2),
 471         0x00000000,
 472         (0x0e00 << 16) | (0x92bc >> 2),
 473         0x00000000,
 474         (0x0e00 << 16) | (0x92c0 >> 2),
 475         0x00000000,
 476         (0x0e00 << 16) | (0x92c4 >> 2),
 477         0x00000000,
 478         (0x0e00 << 16) | (0x92c8 >> 2),
 479         0x00000000,
 480         (0x0e00 << 16) | (0x92cc >> 2),
 481         0x00000000,
 482         (0x0e00 << 16) | (0x92d0 >> 2),
 483         0x00000000,
 484         (0x0e00 << 16) | (0x8c00 >> 2),
 485         0x00000000,
 486         (0x0e00 << 16) | (0x8c04 >> 2),
 487         0x00000000,
 488         (0x0e00 << 16) | (0x8c20 >> 2),
 489         0x00000000,
 490         (0x0e00 << 16) | (0x8c38 >> 2),
 491         0x00000000,
 492         (0x0e00 << 16) | (0x8c3c >> 2),
 493         0x00000000,
 494         (0x0e00 << 16) | (0xae00 >> 2),
 495         0x00000000,
 496         (0x0e00 << 16) | (0x9604 >> 2),
 497         0x00000000,
 498         (0x0e00 << 16) | (0xac08 >> 2),
 499         0x00000000,
 500         (0x0e00 << 16) | (0xac0c >> 2),
 501         0x00000000,
 502         (0x0e00 << 16) | (0xac10 >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0xac14 >> 2),
 505         0x00000000,
 506         (0x0e00 << 16) | (0xac58 >> 2),
 507         0x00000000,
 508         (0x0e00 << 16) | (0xac68 >> 2),
 509         0x00000000,
 510         (0x0e00 << 16) | (0xac6c >> 2),
 511         0x00000000,
 512         (0x0e00 << 16) | (0xac70 >> 2),
 513         0x00000000,
 514         (0x0e00 << 16) | (0xac74 >> 2),
 515         0x00000000,
 516         (0x0e00 << 16) | (0xac78 >> 2),
 517         0x00000000,
 518         (0x0e00 << 16) | (0xac7c >> 2),
 519         0x00000000,
 520         (0x0e00 << 16) | (0xac80 >> 2),
 521         0x00000000,
 522         (0x0e00 << 16) | (0xac84 >> 2),
 523         0x00000000,
 524         (0x0e00 << 16) | (0xac88 >> 2),
 525         0x00000000,
 526         (0x0e00 << 16) | (0xac8c >> 2),
 527         0x00000000,
 528         (0x0e00 << 16) | (0x970c >> 2),
 529         0x00000000,
 530         (0x0e00 << 16) | (0x9714 >> 2),
 531         0x00000000,
 532         (0x0e00 << 16) | (0x9718 >> 2),
 533         0x00000000,
 534         (0x0e00 << 16) | (0x971c >> 2),
 535         0x00000000,
 536         (0x0e00 << 16) | (0x31068 >> 2),
 537         0x00000000,
 538         (0x4e00 << 16) | (0x31068 >> 2),
 539         0x00000000,
 540         (0x5e00 << 16) | (0x31068 >> 2),
 541         0x00000000,
 542         (0x6e00 << 16) | (0x31068 >> 2),
 543         0x00000000,
 544         (0x7e00 << 16) | (0x31068 >> 2),
 545         0x00000000,
 546         (0x8e00 << 16) | (0x31068 >> 2),
 547         0x00000000,
 548         (0x9e00 << 16) | (0x31068 >> 2),
 549         0x00000000,
 550         (0xae00 << 16) | (0x31068 >> 2),
 551         0x00000000,
 552         (0xbe00 << 16) | (0x31068 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0xcd10 >> 2),
 555         0x00000000,
 556         (0x0e00 << 16) | (0xcd14 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0x88b0 >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0x88b4 >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0x88b8 >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0x88bc >> 2),
 565         0x00000000,
 566         (0x0400 << 16) | (0x89c0 >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0x88c4 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0x88c8 >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x88d0 >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x88d4 >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x88d8 >> 2),
 577         0x00000000,
 578         (0x0e00 << 16) | (0x8980 >> 2),
 579         0x00000000,
 580         (0x0e00 << 16) | (0x30938 >> 2),
 581         0x00000000,
 582         (0x0e00 << 16) | (0x3093c >> 2),
 583         0x00000000,
 584         (0x0e00 << 16) | (0x30940 >> 2),
 585         0x00000000,
 586         (0x0e00 << 16) | (0x89a0 >> 2),
 587         0x00000000,
 588         (0x0e00 << 16) | (0x30900 >> 2),
 589         0x00000000,
 590         (0x0e00 << 16) | (0x30904 >> 2),
 591         0x00000000,
 592         (0x0e00 << 16) | (0x89b4 >> 2),
 593         0x00000000,
 594         (0x0e00 << 16) | (0x3c210 >> 2),
 595         0x00000000,
 596         (0x0e00 << 16) | (0x3c214 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0x3c218 >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0x8904 >> 2),
 601         0x00000000,
 602         0x5,
 603         (0x0e00 << 16) | (0x8c28 >> 2),
 604         (0x0e00 << 16) | (0x8c2c >> 2),
 605         (0x0e00 << 16) | (0x8c30 >> 2),
 606         (0x0e00 << 16) | (0x8c34 >> 2),
 607         (0x0e00 << 16) | (0x9600 >> 2),
 608 };
 609
 610 static const u32 kalindi_rlc_save_restore_register_list[] =
 611 {
 612         (0x0e00 << 16) | (0xc12c >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0xc140 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0xc150 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0xc15c >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0xc168 >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0xc170 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0xc204 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0xc2b4 >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0xc2b8 >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0xc2bc >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0xc2c0 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0x8228 >> 2),
 635         0x00000000,
 636         (0x0e00 << 16) | (0x829c >> 2),
 637         0x00000000,
 638         (0x0e00 << 16) | (0x869c >> 2),
 639         0x00000000,
 640         (0x0600 << 16) | (0x98f4 >> 2),
 641         0x00000000,
 642         (0x0e00 << 16) | (0x98f8 >> 2),
 643         0x00000000,
 644         (0x0e00 << 16) | (0x9900 >> 2),
 645         0x00000000,
 646         (0x0e00 << 16) | (0xc260 >> 2),
 647         0x00000000,
 648         (0x0e00 << 16) | (0x90e8 >> 2),
 649         0x00000000,
 650         (0x0e00 << 16) | (0x3c000 >> 2),
 651         0x00000000,
 652         (0x0e00 << 16) | (0x3c00c >> 2),
 653         0x00000000,
 654         (0x0e00 << 16) | (0x8c1c >> 2),
 655         0x00000000,
 656         (0x0e00 << 16) | (0x9700 >> 2),
 657         0x00000000,
 658         (0x0e00 << 16) | (0xcd20 >> 2),
 659         0x00000000,
 660         (0x4e00 << 16) | (0xcd20 >> 2),
 661         0x00000000,
 662         (0x5e00 << 16) | (0xcd20 >> 2),
 663         0x00000000,
 664         (0x6e00 << 16) | (0xcd20 >> 2),
 665         0x00000000,
 666         (0x7e00 << 16) | (0xcd20 >> 2),
 667         0x00000000,
 668         (0x0e00 << 16) | (0x89bc >> 2),
 669         0x00000000,
 670         (0x0e00 << 16) | (0x8900 >> 2),
 671         0x00000000,
 672         0x3,
 673         (0x0e00 << 16) | (0xc130 >> 2),
 674         0x00000000,
 675         (0x0e00 << 16) | (0xc134 >> 2),
 676         0x00000000,
 677         (0x0e00 << 16) | (0xc1fc >> 2),
 678         0x00000000,
 679         (0x0e00 << 16) | (0xc208 >> 2),
 680         0x00000000,
 681         (0x0e00 << 16) | (0xc264 >> 2),
 682         0x00000000,
 683         (0x0e00 << 16) | (0xc268 >> 2),
 684         0x00000000,
 685         (0x0e00 << 16) | (0xc26c >> 2),
 686         0x00000000,
 687         (0x0e00 << 16) | (0xc270 >> 2),
 688         0x00000000,
 689         (0x0e00 << 16) | (0xc274 >> 2),
 690         0x00000000,
 691         (0x0e00 << 16) | (0xc28c >> 2),
 692         0x00000000,
 693         (0x0e00 << 16) | (0xc290 >> 2),
 694         0x00000000,
 695         (0x0e00 << 16) | (0xc294 >> 2),
 696         0x00000000,
 697         (0x0e00 << 16) | (0xc298 >> 2),
 698         0x00000000,
 699         (0x0e00 << 16) | (0xc2a0 >> 2),
 700         0x00000000,
 701         (0x0e00 << 16) | (0xc2a4 >> 2),
 702         0x00000000,
 703         (0x0e00 << 16) | (0xc2a8 >> 2),
 704         0x00000000,
 705         (0x0e00 << 16) | (0xc2ac >> 2),
 706         0x00000000,
 707         (0x0e00 << 16) | (0x301d0 >> 2),
 708         0x00000000,
 709         (0x0e00 << 16) | (0x30238 >> 2),
 710         0x00000000,
 711         (0x0e00 << 16) | (0x30250 >> 2),
 712         0x00000000,
 713         (0x0e00 << 16) | (0x30254 >> 2),
 714         0x00000000,
 715         (0x0e00 << 16) | (0x30258 >> 2),
 716         0x00000000,
 717         (0x0e00 << 16) | (0x3025c >> 2),
 718         0x00000000,
 719         (0x4e00 << 16) | (0xc900 >> 2),
 720         0x00000000,
 721         (0x5e00 << 16) | (0xc900 >> 2),
 722         0x00000000,
 723         (0x6e00 << 16) | (0xc900 >> 2),
 724         0x00000000,
 725         (0x7e00 << 16) | (0xc900 >> 2),
 726         0x00000000,
 727         (0x4e00 << 16) | (0xc904 >> 2),
 728         0x00000000,
 729         (0x5e00 << 16) | (0xc904 >> 2),
 730         0x00000000,
 731         (0x6e00 << 16) | (0xc904 >> 2),
 732         0x00000000,
 733         (0x7e00 << 16) | (0xc904 >> 2),
 734         0x00000000,
 735         (0x4e00 << 16) | (0xc908 >> 2),
 736         0x00000000,
 737         (0x5e00 << 16) | (0xc908 >> 2),
 738         0x00000000,
 739         (0x6e00 << 16) | (0xc908 >> 2),
 740         0x00000000,
 741         (0x7e00 << 16) | (0xc908 >> 2),
 742         0x00000000,
 743         (0x4e00 << 16) | (0xc90c >> 2),
 744         0x00000000,
 745         (0x5e00 << 16) | (0xc90c >> 2),
 746         0x00000000,
 747         (0x6e00 << 16) | (0xc90c >> 2),
 748         0x00000000,
 749         (0x7e00 << 16) | (0xc90c >> 2),
 750         0x00000000,
 751         (0x4e00 << 16) | (0xc910 >> 2),
 752         0x00000000,
 753         (0x5e00 << 16) | (0xc910 >> 2),
 754         0x00000000,
 755         (0x6e00 << 16) | (0xc910 >> 2),
 756         0x00000000,
 757         (0x7e00 << 16) | (0xc910 >> 2),
 758         0x00000000,
 759         (0x0e00 << 16) | (0xc99c >> 2),
 760         0x00000000,
 761         (0x0e00 << 16) | (0x9834 >> 2),
 762         0x00000000,
 763         (0x0000 << 16) | (0x30f00 >> 2),
 764         0x00000000,
 765         (0x0000 << 16) | (0x30f04 >> 2),
 766         0x00000000,
 767         (0x0000 << 16) | (0x30f08 >> 2),
 768         0x00000000,
 769         (0x0000 << 16) | (0x30f0c >> 2),
 770         0x00000000,
 771         (0x0600 << 16) | (0x9b7c >> 2),
 772         0x00000000,
 773         (0x0e00 << 16) | (0x8a14 >> 2),
 774         0x00000000,
 775         (0x0e00 << 16) | (0x8a18 >> 2),
 776         0x00000000,
 777         (0x0600 << 16) | (0x30a00 >> 2),
 778         0x00000000,
 779         (0x0e00 << 16) | (0x8bf0 >> 2),
 780         0x00000000,
 781         (0x0e00 << 16) | (0x8bcc >> 2),
 782         0x00000000,
 783         (0x0e00 << 16) | (0x8b24 >> 2),
 784         0x00000000,
 785         (0x0e00 << 16) | (0x30a04 >> 2),
 786         0x00000000,
 787         (0x0600 << 16) | (0x30a10 >> 2),
 788         0x00000000,
 789         (0x0600 << 16) | (0x30a14 >> 2),
 790         0x00000000,
 791         (0x0600 << 16) | (0x30a18 >> 2),
 792         0x00000000,
 793         (0x0600 << 16) | (0x30a2c >> 2),
 794         0x00000000,
 795         (0x0e00 << 16) | (0xc700 >> 2),
 796         0x00000000,
 797         (0x0e00 << 16) | (0xc704 >> 2),
 798         0x00000000,
 799         (0x0e00 << 16) | (0xc708 >> 2),
 800         0x00000000,
 801         (0x0e00 << 16) | (0xc768 >> 2),
 802         0x00000000,
 803         (0x0400 << 16) | (0xc770 >> 2),
 804         0x00000000,
 805         (0x0400 << 16) | (0xc774 >> 2),
 806         0x00000000,
 807         (0x0400 << 16) | (0xc798 >> 2),
 808         0x00000000,
 809         (0x0400 << 16) | (0xc79c >> 2),
 810         0x00000000,
 811         (0x0e00 << 16) | (0x9100 >> 2),
 812         0x00000000,
 813         (0x0e00 << 16) | (0x3c010 >> 2),
 814         0x00000000,
 815         (0x0e00 << 16) | (0x8c00 >> 2),
 816         0x00000000,
 817         (0x0e00 << 16) | (0x8c04 >> 2),
 818         0x00000000,
 819         (0x0e00 << 16) | (0x8c20 >> 2),
 820         0x00000000,
 821         (0x0e00 << 16) | (0x8c38 >> 2),
 822         0x00000000,
 823         (0x0e00 << 16) | (0x8c3c >> 2),
 824         0x00000000,
 825         (0x0e00 << 16) | (0xae00 >> 2),
 826         0x00000000,
 827         (0x0e00 << 16) | (0x9604 >> 2),
 828         0x00000000,
 829         (0x0e00 << 16) | (0xac08 >> 2),
 830         0x00000000,
 831         (0x0e00 << 16) | (0xac0c >> 2),
 832         0x00000000,
 833         (0x0e00 << 16) | (0xac10 >> 2),
 834         0x00000000,
 835         (0x0e00 << 16) | (0xac14 >> 2),
 836         0x00000000,
 837         (0x0e00 << 16) | (0xac58 >> 2),
 838         0x00000000,
 839         (0x0e00 << 16) | (0xac68 >> 2),
 840         0x00000000,
 841         (0x0e00 << 16) | (0xac6c >> 2),
 842         0x00000000,
 843         (0x0e00 << 16) | (0xac70 >> 2),
 844         0x00000000,
 845         (0x0e00 << 16) | (0xac74 >> 2),
 846         0x00000000,
 847         (0x0e00 << 16) | (0xac78 >> 2),
 848         0x00000000,
 849         (0x0e00 << 16) | (0xac7c >> 2),
 850         0x00000000,
 851         (0x0e00 << 16) | (0xac80 >> 2),
 852         0x00000000,
 853         (0x0e00 << 16) | (0xac84 >> 2),
 854         0x00000000,
 855         (0x0e00 << 16) | (0xac88 >> 2),
 856         0x00000000,
 857         (0x0e00 << 16) | (0xac8c >> 2),
 858         0x00000000,
 859         (0x0e00 << 16) | (0x970c >> 2),
 860         0x00000000,
 861         (0x0e00 << 16) | (0x9714 >> 2),
 862         0x00000000,
 863         (0x0e00 << 16) | (0x9718 >> 2),
 864         0x00000000,
 865         (0x0e00 << 16) | (0x971c >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0x31068 >> 2),
 868         0x00000000,
 869         (0x4e00 << 16) | (0x31068 >> 2),
 870         0x00000000,
 871         (0x5e00 << 16) | (0x31068 >> 2),
 872         0x00000000,
 873         (0x6e00 << 16) | (0x31068 >> 2),
 874         0x00000000,
 875         (0x7e00 << 16) | (0x31068 >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0xcd10 >> 2),
 878         0x00000000,
 879         (0x0e00 << 16) | (0xcd14 >> 2),
 880         0x00000000,
 881         (0x0e00 << 16) | (0x88b0 >> 2),
 882         0x00000000,
 883         (0x0e00 << 16) | (0x88b4 >> 2),
 884         0x00000000,
 885         (0x0e00 << 16) | (0x88b8 >> 2),
 886         0x00000000,
 887         (0x0e00 << 16) | (0x88bc >> 2),
 888         0x00000000,
 889         (0x0400 << 16) | (0x89c0 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0x88c4 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0x88c8 >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0x88d0 >> 2),
 896         0x00000000,
 897         (0x0e00 << 16) | (0x88d4 >> 2),
 898         0x00000000,
 899         (0x0e00 << 16) | (0x88d8 >> 2),
 900         0x00000000,
 901         (0x0e00 << 16) | (0x8980 >> 2),
 902         0x00000000,
 903         (0x0e00 << 16) | (0x30938 >> 2),
 904         0x00000000,
 905         (0x0e00 << 16) | (0x3093c >> 2),
 906         0x00000000,
 907         (0x0e00 << 16) | (0x30940 >> 2),
 908         0x00000000,
 909         (0x0e00 << 16) | (0x89a0 >> 2),
 910         0x00000000,
 911         (0x0e00 << 16) | (0x30900 >> 2),
 912         0x00000000,
 913         (0x0e00 << 16) | (0x30904 >> 2),
 914         0x00000000,
 915         (0x0e00 << 16) | (0x89b4 >> 2),
 916         0x00000000,
 917         (0x0e00 << 16) | (0x3e1fc >> 2),
 918         0x00000000,
 919         (0x0e00 << 16) | (0x3c210 >> 2),
 920         0x00000000,
 921         (0x0e00 << 16) | (0x3c214 >> 2),
 922         0x00000000,
 923         (0x0e00 << 16) | (0x3c218 >> 2),
 924         0x00000000,
 925         (0x0e00 << 16) | (0x8904 >> 2),
 926         0x00000000,
 927         0x5,
 928         (0x0e00 << 16) | (0x8c28 >> 2),
 929         (0x0e00 << 16) | (0x8c2c >> 2),
 930         (0x0e00 << 16) | (0x8c30 >> 2),
 931         (0x0e00 << 16) | (0x8c34 >> 2),
 932         (0x0e00 << 16) | (0x9600 >> 2),
 933 };
 934
 935 static const u32 bonaire_golden_spm_registers[] =
 936 {
 937         0x30800, 0xe0ffffff, 0xe0000000
 938 };
 939
 940 static const u32 bonaire_golden_common_registers[] =
 941 {
 942         0xc770, 0xffffffff, 0x00000800,
 943         0xc774, 0xffffffff, 0x00000800,
 944         0xc798, 0xffffffff, 0x00007fbf,
 945         0xc79c, 0xffffffff, 0x00007faf
 946 };
 947
 948 static const u32 bonaire_golden_registers[] =
 949 {
 950         0x3354, 0x00000333, 0x00000333,
 951         0x3350, 0x000c0fc0, 0x00040200,
 952         0x9a10, 0x00010000, 0x00058208,
 953         0x3c000, 0xffff1fff, 0x00140000,
 954         0x3c200, 0xfdfc0fff, 0x00000100,
 955         0x3c234, 0x40000000, 0x40000200,
 956         0x9830, 0xffffffff, 0x00000000,
 957         0x9834, 0xf00fffff, 0x00000400,
 958         0x9838, 0x0002021c, 0x00020200,
 959         0xc78, 0x00000080, 0x00000000,
 960         0x5bb0, 0x000000f0, 0x00000070,
 961         0x5bc0, 0xf0311fff, 0x80300000,
 962         0x98f8, 0x73773777, 0x12010001,
 963         0x350c, 0x00810000, 0x408af000,
 964         0x7030, 0x31000111, 0x00000011,
 965         0x2f48, 0x73773777, 0x12010001,
 966         0x220c, 0x00007fb6, 0x0021a1b1,
 967         0x2210, 0x00007fb6, 0x002021b1,
 968         0x2180, 0x00007fb6, 0x00002191,
 969         0x2218, 0x00007fb6, 0x002121b1,
 970         0x221c, 0x00007fb6, 0x002021b1,
 971         0x21dc, 0x00007fb6, 0x00002191,
 972         0x21e0, 0x00007fb6, 0x00002191,
 973         0x3628, 0x0000003f, 0x0000000a,
 974         0x362c, 0x0000003f, 0x0000000a,
 975         0x2ae4, 0x00073ffe, 0x000022a2,
 976         0x240c, 0x000007ff, 0x00000000,
 977         0x8a14, 0xf000003f, 0x00000007,
 978         0x8bf0, 0x00002001, 0x00000001,
 979         0x8b24, 0xffffffff, 0x00ffffff,
 980         0x30a04, 0x0000ff0f, 0x00000000,
 981         0x28a4c, 0x07ffffff, 0x06000000,
 982         0x4d8, 0x00000fff, 0x00000100,
 983         0x3e78, 0x00000001, 0x00000002,
 984         0x9100, 0x03000000, 0x0362c688,
 985         0x8c00, 0x000000ff, 0x00000001,
 986         0xe40, 0x00001fff, 0x00001fff,
 987         0x9060, 0x0000007f, 0x00000020,
 988         0x9508, 0x00010000, 0x00010000,
 989         0xac14, 0x000003ff, 0x000000f3,
 990         0xac0c, 0xffffffff, 0x00001032
 991 };
 992
 993 static const u32 bonaire_mgcg_cgcg_init[] =
 994 {
 995         0xc420, 0xffffffff, 0xfffffffc,
 996         0x30800, 0xffffffff, 0xe0000000,
 997         0x3c2a0, 0xffffffff, 0x00000100,
 998         0x3c208, 0xffffffff, 0x00000100,
 999         0x3c2c0, 0xffffffff, 0xc0000100,
1000         0x3c2c8, 0xffffffff, 0xc0000100,
1001         0x3c2c4, 0xffffffff, 0xc0000100,
1002         0x55e4, 0xffffffff, 0x00600100,
1003         0x3c280, 0xffffffff, 0x00000100,
1004         0x3c214, 0xffffffff, 0x06000100,
1005         0x3c220, 0xffffffff, 0x00000100,
1006         0x3c218, 0xffffffff, 0x06000100,
1007         0x3c204, 0xffffffff, 0x00000100,
1008         0x3c2e0, 0xffffffff, 0x00000100,
1009         0x3c224, 0xffffffff, 0x00000100,
1010         0x3c200, 0xffffffff, 0x00000100,
1011         0x3c230, 0xffffffff, 0x00000100,
1012         0x3c234, 0xffffffff, 0x00000100,
1013         0x3c250, 0xffffffff, 0x00000100,
1014         0x3c254, 0xffffffff, 0x00000100,
1015         0x3c258, 0xffffffff, 0x00000100,
1016         0x3c25c, 0xffffffff, 0x00000100,
1017         0x3c260, 0xffffffff, 0x00000100,
1018         0x3c27c, 0xffffffff, 0x00000100,
1019         0x3c278, 0xffffffff, 0x00000100,
1020         0x3c210, 0xffffffff, 0x06000100,
1021         0x3c290, 0xffffffff, 0x00000100,
1022         0x3c274, 0xffffffff, 0x00000100,
1023         0x3c2b4, 0xffffffff, 0x00000100,
1024         0x3c2b0, 0xffffffff, 0x00000100,
1025         0x3c270, 0xffffffff, 0x00000100,
1026         0x30800, 0xffffffff, 0xe0000000,
1027         0x3c020, 0xffffffff, 0x00010000,
1028         0x3c024, 0xffffffff, 0x00030002,
1029         0x3c028, 0xffffffff, 0x00040007,
1030         0x3c02c, 0xffffffff, 0x00060005,
1031         0x3c030, 0xffffffff, 0x00090008,
1032         0x3c034, 0xffffffff, 0x00010000,
1033         0x3c038, 0xffffffff, 0x00030002,
1034         0x3c03c, 0xffffffff, 0x00040007,
1035         0x3c040, 0xffffffff, 0x00060005,
1036         0x3c044, 0xffffffff, 0x00090008,
1037         0x3c048, 0xffffffff, 0x00010000,
1038         0x3c04c, 0xffffffff, 0x00030002,
1039         0x3c050, 0xffffffff, 0x00040007,
1040         0x3c054, 0xffffffff, 0x00060005,
1041         0x3c058, 0xffffffff, 0x00090008,
1042         0x3c05c, 0xffffffff, 0x00010000,
1043         0x3c060, 0xffffffff, 0x00030002,
1044         0x3c064, 0xffffffff, 0x00040007,
1045         0x3c068, 0xffffffff, 0x00060005,
1046         0x3c06c, 0xffffffff, 0x00090008,
1047         0x3c070, 0xffffffff, 0x00010000,
1048         0x3c074, 0xffffffff, 0x00030002,
1049         0x3c078, 0xffffffff, 0x00040007,
1050         0x3c07c, 0xffffffff, 0x00060005,
1051         0x3c080, 0xffffffff, 0x00090008,
1052         0x3c084, 0xffffffff, 0x00010000,
1053         0x3c088, 0xffffffff, 0x00030002,
1054         0x3c08c, 0xffffffff, 0x00040007,
1055         0x3c090, 0xffffffff, 0x00060005,
1056         0x3c094, 0xffffffff, 0x00090008,
1057         0x3c098, 0xffffffff, 0x00010000,
1058         0x3c09c, 0xffffffff, 0x00030002,
1059         0x3c0a0, 0xffffffff, 0x00040007,
1060         0x3c0a4, 0xffffffff, 0x00060005,
1061         0x3c0a8, 0xffffffff, 0x00090008,
1062         0x3c000, 0xffffffff, 0x96e00200,
1063         0x8708, 0xffffffff, 0x00900100,
1064         0xc424, 0xffffffff, 0x0020003f,
1065         0x38, 0xffffffff, 0x0140001c,
1066         0x3c, 0x000f0000, 0x000f0000,
1067         0x220, 0xffffffff, 0xC060000C,
1068         0x224, 0xc0000fff, 0x00000100,
1069         0xf90, 0xffffffff, 0x00000100,
1070         0xf98, 0x00000101, 0x00000000,
1071         0x20a8, 0xffffffff, 0x00000104,
1072         0x55e4, 0xff000fff, 0x00000100,
1073         0x30cc, 0xc0000fff, 0x00000104,
1074         0xc1e4, 0x00000001, 0x00000001,
1075         0xd00c, 0xff000ff0, 0x00000100,
1076         0xd80c, 0xff000ff0, 0x00000100
1077 };
1078
1079 static const u32 spectre_golden_spm_registers[] =
1080 {
1081         0x30800, 0xe0ffffff, 0xe0000000
1082 };
1083
1084 static const u32 spectre_golden_common_registers[] =
1085 {
1086         0xc770, 0xffffffff, 0x00000800,
1087         0xc774, 0xffffffff, 0x00000800,
1088         0xc798, 0xffffffff, 0x00007fbf,
1089         0xc79c, 0xffffffff, 0x00007faf
1090 };
1091
1092 static const u32 spectre_golden_registers[] =
1093 {
1094         0x3c000, 0xffff1fff, 0x96940200,
1095         0x3c00c, 0xffff0001, 0xff000000,
1096         0x3c200, 0xfffc0fff, 0x00000100,
1097         0x6ed8, 0x00010101, 0x00010000,
1098         0x9834, 0xf00fffff, 0x00000400,
1099         0x9838, 0xfffffffc, 0x00020200,
1100         0x5bb0, 0x000000f0, 0x00000070,
1101         0x5bc0, 0xf0311fff, 0x80300000,
1102         0x98f8, 0x73773777, 0x12010001,
1103         0x9b7c, 0x00ff0000, 0x00fc0000,
1104         0x2f48, 0x73773777, 0x12010001,
1105         0x8a14, 0xf000003f, 0x00000007,
1106         0x8b24, 0xffffffff, 0x00ffffff,
1107         0x28350, 0x3f3f3fff, 0x00000082,
1108         0x28354, 0x0000003f, 0x00000000,
1109         0x3e78, 0x00000001, 0x00000002,
1110         0x913c, 0xffff03df, 0x00000004,
1111         0xc768, 0x00000008, 0x00000008,
1112         0x8c00, 0x000008ff, 0x00000800,
1113         0x9508, 0x00010000, 0x00010000,
1114         0xac0c, 0xffffffff, 0x54763210,
1115         0x214f8, 0x01ff01ff, 0x00000002,
1116         0x21498, 0x007ff800, 0x00200000,
1117         0x2015c, 0xffffffff, 0x00000f40,
1118         0x30934, 0xffffffff, 0x00000001
1119 };
1120
1121 static const u32 spectre_mgcg_cgcg_init[] =
1122 {
1123         0xc420, 0xffffffff, 0xfffffffc,
1124         0x30800, 0xffffffff, 0xe0000000,
1125         0x3c2a0, 0xffffffff, 0x00000100,
1126         0x3c208, 0xffffffff, 0x00000100,
1127         0x3c2c0, 0xffffffff, 0x00000100,
1128         0x3c2c8, 0xffffffff, 0x00000100,
1129         0x3c2c4, 0xffffffff, 0x00000100,
1130         0x55e4, 0xffffffff, 0x00600100,
1131         0x3c280, 0xffffffff, 0x00000100,
1132         0x3c214, 0xffffffff, 0x06000100,
1133         0x3c220, 0xffffffff, 0x00000100,
1134         0x3c218, 0xffffffff, 0x06000100,
1135         0x3c204, 0xffffffff, 0x00000100,
1136         0x3c2e0, 0xffffffff, 0x00000100,
1137         0x3c224, 0xffffffff, 0x00000100,
1138         0x3c200, 0xffffffff, 0x00000100,
1139         0x3c230, 0xffffffff, 0x00000100,
1140         0x3c234, 0xffffffff, 0x00000100,
1141         0x3c250, 0xffffffff, 0x00000100,
1142         0x3c254, 0xffffffff, 0x00000100,
1143         0x3c258, 0xffffffff, 0x00000100,
1144         0x3c25c, 0xffffffff, 0x00000100,
1145         0x3c260, 0xffffffff, 0x00000100,
1146         0x3c27c, 0xffffffff, 0x00000100,
1147         0x3c278, 0xffffffff, 0x00000100,
1148         0x3c210, 0xffffffff, 0x06000100,
1149         0x3c290, 0xffffffff, 0x00000100,
1150         0x3c274, 0xffffffff, 0x00000100,
1151         0x3c2b4, 0xffffffff, 0x00000100,
1152         0x3c2b0, 0xffffffff, 0x00000100,
1153         0x3c270, 0xffffffff, 0x00000100,
1154         0x30800, 0xffffffff, 0xe0000000,
1155         0x3c020, 0xffffffff, 0x00010000,
1156         0x3c024, 0xffffffff, 0x00030002,
1157         0x3c028, 0xffffffff, 0x00040007,
1158         0x3c02c, 0xffffffff, 0x00060005,
1159         0x3c030, 0xffffffff, 0x00090008,
1160         0x3c034, 0xffffffff, 0x00010000,
1161         0x3c038, 0xffffffff, 0x00030002,
1162         0x3c03c, 0xffffffff, 0x00040007,
1163         0x3c040, 0xffffffff, 0x00060005,
1164         0x3c044, 0xffffffff, 0x00090008,
1165         0x3c048, 0xffffffff, 0x00010000,
1166         0x3c04c, 0xffffffff, 0x00030002,
1167         0x3c050, 0xffffffff, 0x00040007,
1168         0x3c054, 0xffffffff, 0x00060005,
1169         0x3c058, 0xffffffff, 0x00090008,
1170         0x3c05c, 0xffffffff, 0x00010000,
1171         0x3c060, 0xffffffff, 0x00030002,
1172         0x3c064, 0xffffffff, 0x00040007,
1173         0x3c068, 0xffffffff, 0x00060005,
1174         0x3c06c, 0xffffffff, 0x00090008,
1175         0x3c070, 0xffffffff, 0x00010000,
1176         0x3c074, 0xffffffff, 0x00030002,
1177         0x3c078, 0xffffffff, 0x00040007,
1178         0x3c07c, 0xffffffff, 0x00060005,
1179         0x3c080, 0xffffffff, 0x00090008,
1180         0x3c084, 0xffffffff, 0x00010000,
1181         0x3c088, 0xffffffff, 0x00030002,
1182         0x3c08c, 0xffffffff, 0x00040007,
1183         0x3c090, 0xffffffff, 0x00060005,
1184         0x3c094, 0xffffffff, 0x00090008,
1185         0x3c098, 0xffffffff, 0x00010000,
1186         0x3c09c, 0xffffffff, 0x00030002,
1187         0x3c0a0, 0xffffffff, 0x00040007,
1188         0x3c0a4, 0xffffffff, 0x00060005,
1189         0x3c0a8, 0xffffffff, 0x00090008,
1190         0x3c0ac, 0xffffffff, 0x00010000,
1191         0x3c0b0, 0xffffffff, 0x00030002,
1192         0x3c0b4, 0xffffffff, 0x00040007,
1193         0x3c0b8, 0xffffffff, 0x00060005,
1194         0x3c0bc, 0xffffffff, 0x00090008,
1195         0x3c000, 0xffffffff, 0x96e00200,
1196         0x8708, 0xffffffff, 0x00900100,
1197         0xc424, 0xffffffff, 0x0020003f,
1198         0x38, 0xffffffff, 0x0140001c,
1199         0x3c, 0x000f0000, 0x000f0000,
1200         0x220, 0xffffffff, 0xC060000C,
1201         0x224, 0xc0000fff, 0x00000100,
1202         0xf90, 0xffffffff, 0x00000100,
1203         0xf98, 0x00000101, 0x00000000,
1204         0x20a8, 0xffffffff, 0x00000104,
1205         0x55e4, 0xff000fff, 0x00000100,
1206         0x30cc, 0xc0000fff, 0x00000104,
1207         0xc1e4, 0x00000001, 0x00000001,
1208         0xd00c, 0xff000ff0, 0x00000100,
1209         0xd80c, 0xff000ff0, 0x00000100
1210 };
1211
1212 static const u32 kalindi_golden_spm_registers[] =
1213 {
1214         0x30800, 0xe0ffffff, 0xe0000000
1215 };
1216
1217 static const u32 kalindi_golden_common_registers[] =
1218 {
1219         0xc770, 0xffffffff, 0x00000800,
1220         0xc774, 0xffffffff, 0x00000800,
1221         0xc798, 0xffffffff, 0x00007fbf,
1222         0xc79c, 0xffffffff, 0x00007faf
1223 };
1224
1225 static const u32 kalindi_golden_registers[] =
1226 {
1227         0x3c000, 0xffffdfff, 0x6e944040,
1228         0x55e4, 0xff607fff, 0xfc000100,
1229         0x3c220, 0xff000fff, 0x00000100,
1230         0x3c224, 0xff000fff, 0x00000100,
1231         0x3c200, 0xfffc0fff, 0x00000100,
1232         0x6ed8, 0x00010101, 0x00010000,
1233         0x9830, 0xffffffff, 0x00000000,
1234         0x9834, 0xf00fffff, 0x00000400,
1235         0x5bb0, 0x000000f0, 0x00000070,
1236         0x5bc0, 0xf0311fff, 0x80300000,
1237         0x98f8, 0x73773777, 0x12010001,
1238         0x98fc, 0xffffffff, 0x00000010,
1239         0x9b7c, 0x00ff0000, 0x00fc0000,
1240         0x8030, 0x00001f0f, 0x0000100a,
1241         0x2f48, 0x73773777, 0x12010001,
1242         0x2408, 0x000fffff, 0x000c007f,
1243         0x8a14, 0xf000003f, 0x00000007,
1244         0x8b24, 0x3fff3fff, 0x00ffcfff,
1245         0x30a04, 0x0000ff0f, 0x00000000,
1246         0x28a4c, 0x07ffffff, 0x06000000,
1247         0x4d8, 0x00000fff, 0x00000100,
1248         0x3e78, 0x00000001, 0x00000002,
1249         0xc768, 0x00000008, 0x00000008,
1250         0x8c00, 0x000000ff, 0x00000003,
1251         0x214f8, 0x01ff01ff, 0x00000002,
1252         0x21498, 0x007ff800, 0x00200000,
1253         0x2015c, 0xffffffff, 0x00000f40,
1254         0x88c4, 0x001f3ae3, 0x00000082,
1255         0x88d4, 0x0000001f, 0x00000010,
1256         0x30934, 0xffffffff, 0x00000000
1257 };
1258
1259 static const u32 kalindi_mgcg_cgcg_init[] =
1260 {
1261         0xc420, 0xffffffff, 0xfffffffc,
1262         0x30800, 0xffffffff, 0xe0000000,
1263         0x3c2a0, 0xffffffff, 0x00000100,
1264         0x3c208, 0xffffffff, 0x00000100,
1265         0x3c2c0, 0xffffffff, 0x00000100,
1266         0x3c2c8, 0xffffffff, 0x00000100,
1267         0x3c2c4, 0xffffffff, 0x00000100,
1268         0x55e4, 0xffffffff, 0x00600100,
1269         0x3c280, 0xffffffff, 0x00000100,
1270         0x3c214, 0xffffffff, 0x06000100,
1271         0x3c220, 0xffffffff, 0x00000100,
1272         0x3c218, 0xffffffff, 0x06000100,
1273         0x3c204, 0xffffffff, 0x00000100,
1274         0x3c2e0, 0xffffffff, 0x00000100,
1275         0x3c224, 0xffffffff, 0x00000100,
1276         0x3c200, 0xffffffff, 0x00000100,
1277         0x3c230, 0xffffffff, 0x00000100,
1278         0x3c234, 0xffffffff, 0x00000100,
1279         0x3c250, 0xffffffff, 0x00000100,
1280         0x3c254, 0xffffffff, 0x00000100,
1281         0x3c258, 0xffffffff, 0x00000100,
1282         0x3c25c, 0xffffffff, 0x00000100,
1283         0x3c260, 0xffffffff, 0x00000100,
1284         0x3c27c, 0xffffffff, 0x00000100,
1285         0x3c278, 0xffffffff, 0x00000100,
1286         0x3c210, 0xffffffff, 0x06000100,
1287         0x3c290, 0xffffffff, 0x00000100,
1288         0x3c274, 0xffffffff, 0x00000100,
1289         0x3c2b4, 0xffffffff, 0x00000100,
1290         0x3c2b0, 0xffffffff, 0x00000100,
1291         0x3c270, 0xffffffff, 0x00000100,
1292         0x30800, 0xffffffff, 0xe0000000,
1293         0x3c020, 0xffffffff, 0x00010000,
1294         0x3c024, 0xffffffff, 0x00030002,
1295         0x3c028, 0xffffffff, 0x00040007,
1296         0x3c02c, 0xffffffff, 0x00060005,
1297         0x3c030, 0xffffffff, 0x00090008,
1298         0x3c034, 0xffffffff, 0x00010000,
1299         0x3c038, 0xffffffff, 0x00030002,
1300         0x3c03c, 0xffffffff, 0x00040007,
1301         0x3c040, 0xffffffff, 0x00060005,
1302         0x3c044, 0xffffffff, 0x00090008,
1303         0x3c000, 0xffffffff, 0x96e00200,
1304         0x8708, 0xffffffff, 0x00900100,
1305         0xc424, 0xffffffff, 0x0020003f,
1306         0x38, 0xffffffff, 0x0140001c,
1307         0x3c, 0x000f0000, 0x000f0000,
1308         0x220, 0xffffffff, 0xC060000C,
1309         0x224, 0xc0000fff, 0x00000100,
1310         0x20a8, 0xffffffff, 0x00000104,
1311         0x55e4, 0xff000fff, 0x00000100,
1312         0x30cc, 0xc0000fff, 0x00000104,
1313         0xc1e4, 0x00000001, 0x00000001,
1314         0xd00c, 0xff000ff0, 0x00000100,
1315         0xd80c, 0xff000ff0, 0x00000100
1316 };
1317
1318 static const u32 hawaii_golden_spm_registers[] =
1319 {
1320         0x30800, 0xe0ffffff, 0xe0000000
1321 };
1322
1323 static const u32 hawaii_golden_common_registers[] =
1324 {
1325         0x30800, 0xffffffff, 0xe0000000,
1326         0x28350, 0xffffffff, 0x3a00161a,
1327         0x28354, 0xffffffff, 0x0000002e,
1328         0x9a10, 0xffffffff, 0x00018208,
1329         0x98f8, 0xffffffff, 0x12011003
1330 };
1331
1332 static const u32 hawaii_golden_registers[] =
1333 {
1334         0x3354, 0x00000333, 0x00000333,
1335         0x9a10, 0x00010000, 0x00058208,
1336         0x9830, 0xffffffff, 0x00000000,
1337         0x9834, 0xf00fffff, 0x00000400,
1338         0x9838, 0x0002021c, 0x00020200,
1339         0xc78, 0x00000080, 0x00000000,
1340         0x5bb0, 0x000000f0, 0x00000070,
1341         0x5bc0, 0xf0311fff, 0x80300000,
1342         0x350c, 0x00810000, 0x408af000,
1343         0x7030, 0x31000111, 0x00000011,
1344         0x2f48, 0x73773777, 0x12010001,
1345         0x2120, 0x0000007f, 0x0000001b,
1346         0x21dc, 0x00007fb6, 0x00002191,
1347         0x3628, 0x0000003f, 0x0000000a,
1348         0x362c, 0x0000003f, 0x0000000a,
1349         0x2ae4, 0x00073ffe, 0x000022a2,
1350         0x240c, 0x000007ff, 0x00000000,
1351         0x8bf0, 0x00002001, 0x00000001,
1352         0x8b24, 0xffffffff, 0x00ffffff,
1353         0x30a04, 0x0000ff0f, 0x00000000,
1354         0x28a4c, 0x07ffffff, 0x06000000,
1355         0x3e78, 0x00000001, 0x00000002,
1356         0xc768, 0x00000008, 0x00000008,
1357         0xc770, 0x00000f00, 0x00000800,
1358         0xc774, 0x00000f00, 0x00000800,
1359         0xc798, 0x00ffffff, 0x00ff7fbf,
1360         0xc79c, 0x00ffffff, 0x00ff7faf,
1361         0x8c00, 0x000000ff, 0x00000800,
1362         0xe40, 0x00001fff, 0x00001fff,
1363         0x9060, 0x0000007f, 0x00000020,
1364         0x9508, 0x00010000, 0x00010000,
1365         0xae00, 0x00100000, 0x000ff07c,
1366         0xac14, 0x000003ff, 0x0000000f,
1367         0xac10, 0xffffffff, 0x7564fdec,
1368         0xac0c, 0xffffffff, 0x3120b9a8,
1369         0xac08, 0x20000000, 0x0f9c0000
1370 };
1371
1372 static const u32 hawaii_mgcg_cgcg_init[] =
1373 {
1374         0xc420, 0xffffffff, 0xfffffffd,
1375         0x30800, 0xffffffff, 0xe0000000,
1376         0x3c2a0, 0xffffffff, 0x00000100,
1377         0x3c208, 0xffffffff, 0x00000100,
1378         0x3c2c0, 0xffffffff, 0x00000100,
1379         0x3c2c8, 0xffffffff, 0x00000100,
1380         0x3c2c4, 0xffffffff, 0x00000100,
1381         0x55e4, 0xffffffff, 0x00200100,
1382         0x3c280, 0xffffffff, 0x00000100,
1383         0x3c214, 0xffffffff, 0x06000100,
1384         0x3c220, 0xffffffff, 0x00000100,
1385         0x3c218, 0xffffffff, 0x06000100,
1386         0x3c204, 0xffffffff, 0x00000100,
1387         0x3c2e0, 0xffffffff, 0x00000100,
1388         0x3c224, 0xffffffff, 0x00000100,
1389         0x3c200, 0xffffffff, 0x00000100,
1390         0x3c230, 0xffffffff, 0x00000100,
1391         0x3c234, 0xffffffff, 0x00000100,
1392         0x3c250, 0xffffffff, 0x00000100,
1393         0x3c254, 0xffffffff, 0x00000100,
1394         0x3c258, 0xffffffff, 0x00000100,
1395         0x3c25c, 0xffffffff, 0x00000100,
1396         0x3c260, 0xffffffff, 0x00000100,
1397         0x3c27c, 0xffffffff, 0x00000100,
1398         0x3c278, 0xffffffff, 0x00000100,
1399         0x3c210, 0xffffffff, 0x06000100,
1400         0x3c290, 0xffffffff, 0x00000100,
1401         0x3c274, 0xffffffff, 0x00000100,
1402         0x3c2b4, 0xffffffff, 0x00000100,
1403         0x3c2b0, 0xffffffff, 0x00000100,
1404         0x3c270, 0xffffffff, 0x00000100,
1405         0x30800, 0xffffffff, 0xe0000000,
1406         0x3c020, 0xffffffff, 0x00010000,
1407         0x3c024, 0xffffffff, 0x00030002,
1408         0x3c028, 0xffffffff, 0x00040007,
1409         0x3c02c, 0xffffffff, 0x00060005,
1410         0x3c030, 0xffffffff, 0x00090008,
1411         0x3c034, 0xffffffff, 0x00010000,
1412         0x3c038, 0xffffffff, 0x00030002,
1413         0x3c03c, 0xffffffff, 0x00040007,
1414         0x3c040, 0xffffffff, 0x00060005,
1415         0x3c044, 0xffffffff, 0x00090008,
1416         0x3c048, 0xffffffff, 0x00010000,
1417         0x3c04c, 0xffffffff, 0x00030002,
1418         0x3c050, 0xffffffff, 0x00040007,
1419         0x3c054, 0xffffffff, 0x00060005,
1420         0x3c058, 0xffffffff, 0x00090008,
1421         0x3c05c, 0xffffffff, 0x00010000,
1422         0x3c060, 0xffffffff, 0x00030002,
1423         0x3c064, 0xffffffff, 0x00040007,
1424         0x3c068, 0xffffffff, 0x00060005,
1425         0x3c06c, 0xffffffff, 0x00090008,
1426         0x3c070, 0xffffffff, 0x00010000,
1427         0x3c074, 0xffffffff, 0x00030002,
1428         0x3c078, 0xffffffff, 0x00040007,
1429         0x3c07c, 0xffffffff, 0x00060005,
1430         0x3c080, 0xffffffff, 0x00090008,
1431         0x3c084, 0xffffffff, 0x00010000,
1432         0x3c088, 0xffffffff, 0x00030002,
1433         0x3c08c, 0xffffffff, 0x00040007,
1434         0x3c090, 0xffffffff, 0x00060005,
1435         0x3c094, 0xffffffff, 0x00090008,
1436         0x3c098, 0xffffffff, 0x00010000,
1437         0x3c09c, 0xffffffff, 0x00030002,
1438         0x3c0a0, 0xffffffff, 0x00040007,
1439         0x3c0a4, 0xffffffff, 0x00060005,
1440         0x3c0a8, 0xffffffff, 0x00090008,
1441         0x3c0ac, 0xffffffff, 0x00010000,
1442         0x3c0b0, 0xffffffff, 0x00030002,
1443         0x3c0b4, 0xffffffff, 0x00040007,
1444         0x3c0b8, 0xffffffff, 0x00060005,
1445         0x3c0bc, 0xffffffff, 0x00090008,
1446         0x3c0c0, 0xffffffff, 0x00010000,
1447         0x3c0c4, 0xffffffff, 0x00030002,
1448         0x3c0c8, 0xffffffff, 0x00040007,
1449         0x3c0cc, 0xffffffff, 0x00060005,
1450         0x3c0d0, 0xffffffff, 0x00090008,
1451         0x3c0d4, 0xffffffff, 0x00010000,
1452         0x3c0d8, 0xffffffff, 0x00030002,
1453         0x3c0dc, 0xffffffff, 0x00040007,
1454         0x3c0e0, 0xffffffff, 0x00060005,
1455         0x3c0e4, 0xffffffff, 0x00090008,
1456         0x3c0e8, 0xffffffff, 0x00010000,
1457         0x3c0ec, 0xffffffff, 0x00030002,
1458         0x3c0f0, 0xffffffff, 0x00040007,
1459         0x3c0f4, 0xffffffff, 0x00060005,
1460         0x3c0f8, 0xffffffff, 0x00090008,
1461         0xc318, 0xffffffff, 0x00020200,
1462         0x3350, 0xffffffff, 0x00000200,
1463         0x15c0, 0xffffffff, 0x00000400,
1464         0x55e8, 0xffffffff, 0x00000000,
1465         0x2f50, 0xffffffff, 0x00000902,
1466         0x3c000, 0xffffffff, 0x96940200,
1467         0x8708, 0xffffffff, 0x00900100,
1468         0xc424, 0xffffffff, 0x0020003f,
1469         0x38, 0xffffffff, 0x0140001c,
1470         0x3c, 0x000f0000, 0x000f0000,
1471         0x220, 0xffffffff, 0xc060000c,
1472         0x224, 0xc0000fff, 0x00000100,
1473         0xf90, 0xffffffff, 0x00000100,
1474         0xf98, 0x00000101, 0x00000000,
1475         0x20a8, 0xffffffff, 0x00000104,
1476         0x55e4, 0xff000fff, 0x00000100,
1477         0x30cc, 0xc0000fff, 0x00000104,
1478         0xc1e4, 0x00000001, 0x00000001,
1479         0xd00c, 0xff000ff0, 0x00000100,
1480         0xd80c, 0xff000ff0, 0x00000100
1481 };
1482
1483 static const u32 godavari_golden_registers[] =
1484 {
1485         0x55e4, 0xff607fff, 0xfc000100,
1486         0x6ed8, 0x00010101, 0x00010000,
1487         0x9830, 0xffffffff, 0x00000000,
1488         0x98302, 0xf00fffff, 0x00000400,
1489         0x6130, 0xffffffff, 0x00010000,
1490         0x5bb0, 0x000000f0, 0x00000070,
1491         0x5bc0, 0xf0311fff, 0x80300000,
1492         0x98f8, 0x73773777, 0x12010001,
1493         0x98fc, 0xffffffff, 0x00000010,
1494         0x8030, 0x00001f0f, 0x0000100a,
1495         0x2f48, 0x73773777, 0x12010001,
1496         0x2408, 0x000fffff, 0x000c007f,
1497         0x8a14, 0xf000003f, 0x00000007,
1498         0x8b24, 0xffffffff, 0x00ff0fff,
1499         0x30a04, 0x0000ff0f, 0x00000000,
1500         0x28a4c, 0x07ffffff, 0x06000000,
1501         0x4d8, 0x00000fff, 0x00000100,
1502         0xd014, 0x00010000, 0x00810001,
1503         0xd814, 0x00010000, 0x00810001,
1504         0x3e78, 0x00000001, 0x00000002,
1505         0xc768, 0x00000008, 0x00000008,
1506         0xc770, 0x00000f00, 0x00000800,
1507         0xc774, 0x00000f00, 0x00000800,
1508         0xc798, 0x00ffffff, 0x00ff7fbf,
1509         0xc79c, 0x00ffffff, 0x00ff7faf,
1510         0x8c00, 0x000000ff, 0x00000001,
1511         0x214f8, 0x01ff01ff, 0x00000002,
1512         0x21498, 0x007ff800, 0x00200000,
1513         0x2015c, 0xffffffff, 0x00000f40,
1514         0x88c4, 0x001f3ae3, 0x00000082,
1515         0x88d4, 0x0000001f, 0x00000010,
1516         0x30934, 0xffffffff, 0x00000000
1517 };
1518
1519
1520 static void cik_init_golden_registers(struct radeon_device *rdev)
1521 {
1522         switch (rdev->family) {
1523         case CHIP_BONAIRE:
1524                 radeon_program_register_sequence(rdev,
1525                                                  bonaire_mgcg_cgcg_init,
1526                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1527                 radeon_program_register_sequence(rdev,
1528                                                  bonaire_golden_registers,
1529                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1530                 radeon_program_register_sequence(rdev,
1531                                                  bonaire_golden_common_registers,
1532                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1533                 radeon_program_register_sequence(rdev,
1534                                                  bonaire_golden_spm_registers,
1535                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1536                 break;
1537         case CHIP_KABINI:
1538                 radeon_program_register_sequence(rdev,
1539                                                  kalindi_mgcg_cgcg_init,
1540                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1541                 radeon_program_register_sequence(rdev,
1542                                                  kalindi_golden_registers,
1543                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1544                 radeon_program_register_sequence(rdev,
1545                                                  kalindi_golden_common_registers,
1546                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1547                 radeon_program_register_sequence(rdev,
1548                                                  kalindi_golden_spm_registers,
1549                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1550                 break;
1551         case CHIP_MULLINS:
1552                 radeon_program_register_sequence(rdev,
1553                                                  kalindi_mgcg_cgcg_init,
1554                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1555                 radeon_program_register_sequence(rdev,
1556                                                  godavari_golden_registers,
1557                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
1558                 radeon_program_register_sequence(rdev,
1559                                                  kalindi_golden_common_registers,
1560                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1561                 radeon_program_register_sequence(rdev,
1562                                                  kalindi_golden_spm_registers,
1563                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1564                 break;
1565         case CHIP_KAVERI:
1566                 radeon_program_register_sequence(rdev,
1567                                                  spectre_mgcg_cgcg_init,
1568                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1569                 radeon_program_register_sequence(rdev,
1570                                                  spectre_golden_registers,
1571                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1572                 radeon_program_register_sequence(rdev,
1573                                                  spectre_golden_common_registers,
1574                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1575                 radeon_program_register_sequence(rdev,
1576                                                  spectre_golden_spm_registers,
1577                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1578                 break;
1579         case CHIP_HAWAII:
1580                 radeon_program_register_sequence(rdev,
1581                                                  hawaii_mgcg_cgcg_init,
1582                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1583                 radeon_program_register_sequence(rdev,
1584                                                  hawaii_golden_registers,
1585                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1586                 radeon_program_register_sequence(rdev,
1587                                                  hawaii_golden_common_registers,
1588                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1589                 radeon_program_register_sequence(rdev,
1590                                                  hawaii_golden_spm_registers,
1591                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1592                 break;
1593         default:
1594                 break;
1595         }
1596 }
1597
1598 /**
1599  * cik_get_xclk - get the xclk
1600  *
1601  * @rdev: radeon_device pointer
1602  *
1603  * Returns the reference clock used by the gfx engine
1604  * (CIK).
1605  */
1606 u32 cik_get_xclk(struct radeon_device *rdev)
1607 {
1608         u32 reference_clock = rdev->clock.spll.reference_freq;
1609
1610         if (rdev->flags & RADEON_IS_IGP) {
1611                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1612                         return reference_clock / 2;
1613         } else {
1614                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1615                         return reference_clock / 4;
1616         }
1617         return reference_clock;
1618 }
1619
1620 /**
1621  * cik_mm_rdoorbell - read a doorbell dword
1622  *
1623  * @rdev: radeon_device pointer
1624  * @index: doorbell index
1625  *
1626  * Returns the value in the doorbell aperture at the
1627  * requested doorbell index (CIK).
1628  */
1629 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1630 {
1631         if (index < rdev->doorbell.num_doorbells) {
1632                 return readl(rdev->doorbell.ptr + index);
1633         } else {
1634                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1635                 return 0;
1636         }
1637 }
1638
1639 /**
1640  * cik_mm_wdoorbell - write a doorbell dword
1641  *
1642  * @rdev: radeon_device pointer
1643  * @index: doorbell index
1644  * @v: value to write
1645  *
1646  * Writes @v to the doorbell aperture at the
1647  * requested doorbell index (CIK).
1648  */
1649 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1650 {
1651         if (index < rdev->doorbell.num_doorbells) {
1652                 writel(v, rdev->doorbell.ptr + index);
1653         } else {
1654                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1655         }
1656 }
1657
1658 #define BONAIRE_IO_MC_REGS_SIZE 36
1659
1660 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1661 {
1662         {0x00000070, 0x04400000},
1663         {0x00000071, 0x80c01803},
1664         {0x00000072, 0x00004004},
1665         {0x00000073, 0x00000100},
1666         {0x00000074, 0x00ff0000},
1667         {0x00000075, 0x34000000},
1668         {0x00000076, 0x08000014},
1669         {0x00000077, 0x00cc08ec},
1670         {0x00000078, 0x00000400},
1671         {0x00000079, 0x00000000},
1672         {0x0000007a, 0x04090000},
1673         {0x0000007c, 0x00000000},
1674         {0x0000007e, 0x4408a8e8},
1675         {0x0000007f, 0x00000304},
1676         {0x00000080, 0x00000000},
1677         {0x00000082, 0x00000001},
1678         {0x00000083, 0x00000002},
1679         {0x00000084, 0xf3e4f400},
1680         {0x00000085, 0x052024e3},
1681         {0x00000087, 0x00000000},
1682         {0x00000088, 0x01000000},
1683         {0x0000008a, 0x1c0a0000},
1684         {0x0000008b, 0xff010000},
1685         {0x0000008d, 0xffffefff},
1686         {0x0000008e, 0xfff3efff},
1687         {0x0000008f, 0xfff3efbf},
1688         {0x00000092, 0xf7ffffff},
1689         {0x00000093, 0xffffff7f},
1690         {0x00000095, 0x00101101},
1691         {0x00000096, 0x00000fff},
1692         {0x00000097, 0x00116fff},
1693         {0x00000098, 0x60010000},
1694         {0x00000099, 0x10010000},
1695         {0x0000009a, 0x00006000},
1696         {0x0000009b, 0x00001000},
1697         {0x0000009f, 0x00b48000}
1698 };
1699
1700 #define HAWAII_IO_MC_REGS_SIZE 22
1701
1702 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1703 {
1704         {0x0000007d, 0x40000000},
1705         {0x0000007e, 0x40180304},
1706         {0x0000007f, 0x0000ff00},
1707         {0x00000081, 0x00000000},
1708         {0x00000083, 0x00000800},
1709         {0x00000086, 0x00000000},
1710         {0x00000087, 0x00000100},
1711         {0x00000088, 0x00020100},
1712         {0x00000089, 0x00000000},
1713         {0x0000008b, 0x00040000},
1714         {0x0000008c, 0x00000100},
1715         {0x0000008e, 0xff010000},
1716         {0x00000090, 0xffffefff},
1717         {0x00000091, 0xfff3efff},
1718         {0x00000092, 0xfff3efbf},
1719         {0x00000093, 0xf7ffffff},
1720         {0x00000094, 0xffffff7f},
1721         {0x00000095, 0x00000fff},
1722         {0x00000096, 0x00116fff},
1723         {0x00000097, 0x60010000},
1724         {0x00000098, 0x10010000},
1725         {0x0000009f, 0x00c79000}
1726 };
1727
1728
1729 /**
1730  * cik_srbm_select - select specific register instances
1731  *
1732  * @rdev: radeon_device pointer
1733  * @me: selected ME (micro engine)
1734  * @pipe: pipe
1735  * @queue: queue
1736  * @vmid: VMID
1737  *
1738  * Switches the currently active registers instances.  Some
1739  * registers are instanced per VMID, others are instanced per
1740  * me/pipe/queue combination.
1741  */
1742 static void cik_srbm_select(struct radeon_device *rdev,
1743                             u32 me, u32 pipe, u32 queue, u32 vmid)
1744 {
1745         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1746                              MEID(me & 0x3) |
1747                              VMID(vmid & 0xf) |
1748                              QUEUEID(queue & 0x7));
1749         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1750 }
1751
1752 /* ucode loading */
1753 /**
1754  * ci_mc_load_microcode - load MC ucode into the hw
1755  *
1756  * @rdev: radeon_device pointer
1757  *
1758  * Load the GDDR MC ucode into the hw (CIK).
1759  * Returns 0 on success, error on failure.
1760  */
1761 int ci_mc_load_microcode(struct radeon_device *rdev)
1762 {
1763         const __be32 *fw_data;
1764         u32 running, blackout = 0;
1765         u32 *io_mc_regs;
1766         int i, regs_size, ucode_size;
1767
1768         if (!rdev->mc_fw)
1769                 return -EINVAL;
1770
1771         ucode_size = rdev->mc_fw->size / 4;
1772
1773         switch (rdev->family) {
1774         case CHIP_BONAIRE:
1775                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1776                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1777                 break;
1778         case CHIP_HAWAII:
1779                 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1780                 regs_size = HAWAII_IO_MC_REGS_SIZE;
1781                 break;
1782         default:
1783                 return -EINVAL;
1784         }
1785
1786         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1787
1788         if (running == 0) {
1789                 if (running) {
1790                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1791                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1792                 }
1793
1794                 /* reset the engine and set to writable */
1795                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1796                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1797
1798                 /* load mc io regs */
1799                 for (i = 0; i < regs_size; i++) {
1800                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1801                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1802                 }
1803                 /* load the MC ucode */
1804                 fw_data = (const __be32 *)rdev->mc_fw->data;
1805                 for (i = 0; i < ucode_size; i++)
1806                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1807
1808                 /* put the engine back into the active state */
1809                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1810                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1811                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1812
1813                 /* wait for training to complete */
1814                 for (i = 0; i < rdev->usec_timeout; i++) {
1815                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1816                                 break;
1817                         udelay(1);
1818                 }
1819                 for (i = 0; i < rdev->usec_timeout; i++) {
1820                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1821                                 break;
1822                         udelay(1);
1823                 }
1824
1825                 if (running)
1826                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1827         }
1828
1829         return 0;
1830 }
1831
1832 /**
1833  * cik_init_microcode - load ucode images from disk
1834  *
1835  * @rdev: radeon_device pointer
1836  *
1837  * Use the firmware interface to load the ucode images into
1838  * the driver (not loaded into hw).
1839  * Returns 0 on success, error on failure.
1840  */
1841 static int cik_init_microcode(struct radeon_device *rdev)
1842 {
1843         const char *chip_name;
1844         size_t pfp_req_size, me_req_size, ce_req_size,
1845                 mec_req_size, rlc_req_size, mc_req_size = 0,
1846                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1847         char fw_name[30];
1848         int err;
1849
1850         DRM_DEBUG("\n");
1851
1852         switch (rdev->family) {
1853         case CHIP_BONAIRE:
1854                 chip_name = "BONAIRE";
1855                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1856                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1857                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1858                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1859                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1860                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1861                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1862                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1863                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1864                 break;
1865         case CHIP_HAWAII:
1866                 chip_name = "HAWAII";
1867                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1868                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1869                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1870                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1871                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1872                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1873                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1874                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1875                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1876                 break;
1877         case CHIP_KAVERI:
1878                 chip_name = "KAVERI";
1879                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1880                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1881                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1882                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1883                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1884                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1885                 break;
1886         case CHIP_KABINI:
1887                 chip_name = "KABINI";
1888                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1889                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1890                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1891                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1892                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1893                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1894                 break;
1895         case CHIP_MULLINS:
1896                 chip_name = "MULLINS";
1897                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1898                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1899                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1900                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1901                 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1902                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1903                 break;
1904         default: BUG();
1905         }
1906
1907         DRM_INFO("Loading %s Microcode\n", chip_name);
1908
1909         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1910         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1911         if (err)
1912                 goto out;
1913         if (rdev->pfp_fw->size != pfp_req_size) {
1914                 printk(KERN_ERR
1915                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1916                        rdev->pfp_fw->size, fw_name);
1917                 err = -EINVAL;
1918                 goto out;
1919         }
1920
1921         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1922         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1923         if (err)
1924                 goto out;
1925         if (rdev->me_fw->size != me_req_size) {
1926                 printk(KERN_ERR
1927                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1928                        rdev->me_fw->size, fw_name);
1929                 err = -EINVAL;
1930         }
1931
1932         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1933         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1934         if (err)
1935                 goto out;
1936         if (rdev->ce_fw->size != ce_req_size) {
1937                 printk(KERN_ERR
1938                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1939                        rdev->ce_fw->size, fw_name);
1940                 err = -EINVAL;
1941         }
1942
1943         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1944         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1945         if (err)
1946                 goto out;
1947         if (rdev->mec_fw->size != mec_req_size) {
1948                 printk(KERN_ERR
1949                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1950                        rdev->mec_fw->size, fw_name);
1951                 err = -EINVAL;
1952         }
1953
1954         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1955         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1956         if (err)
1957                 goto out;
1958         if (rdev->rlc_fw->size != rlc_req_size) {
1959                 printk(KERN_ERR
1960                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1961                        rdev->rlc_fw->size, fw_name);
1962                 err = -EINVAL;
1963         }
1964
1965         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1966         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1967         if (err)
1968                 goto out;
1969         if (rdev->sdma_fw->size != sdma_req_size) {
1970                 printk(KERN_ERR
1971                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1972                        rdev->sdma_fw->size, fw_name);
1973                 err = -EINVAL;
1974         }
1975
1976         /* No SMC, MC ucode on APUs */
1977         if (!(rdev->flags & RADEON_IS_IGP)) {
1978                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1979                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1980                 if (err) {
1981                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1982                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1983                         if (err)
1984                                 goto out;
1985                 }
1986                 if ((rdev->mc_fw->size != mc_req_size) &&
1987                     (rdev->mc_fw->size != mc2_req_size)){
1988                         printk(KERN_ERR
1989                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1990                                rdev->mc_fw->size, fw_name);
1991                         err = -EINVAL;
1992                 }
1993                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1994
1995                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1996                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1997                 if (err) {
1998                         printk(KERN_ERR
1999                                "smc: error loading firmware \"%s\"\n",
2000                                fw_name);
2001                         release_firmware(rdev->smc_fw);
2002                         rdev->smc_fw = NULL;
2003                         err = 0;
2004                 } else if (rdev->smc_fw->size != smc_req_size) {
2005                         printk(KERN_ERR
2006                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2007                                rdev->smc_fw->size, fw_name);
2008                         err = -EINVAL;
2009                 }
2010         }
2011
2012 out:
2013         if (err) {
2014                 if (err != -EINVAL)
2015                         printk(KERN_ERR
2016                                "cik_cp: Failed to load firmware \"%s\"\n",
2017                                fw_name);
2018                 release_firmware(rdev->pfp_fw);
2019                 rdev->pfp_fw = NULL;
2020                 release_firmware(rdev->me_fw);
2021                 rdev->me_fw = NULL;
2022                 release_firmware(rdev->ce_fw);
2023                 rdev->ce_fw = NULL;
2024                 release_firmware(rdev->rlc_fw);
2025                 rdev->rlc_fw = NULL;
2026                 release_firmware(rdev->mc_fw);
2027                 rdev->mc_fw = NULL;
2028                 release_firmware(rdev->smc_fw);
2029                 rdev->smc_fw = NULL;
2030         }
2031         return err;
2032 }
2033
2034 /*
2035  * Core functions
2036  */
2037 /**
2038  * cik_tiling_mode_table_init - init the hw tiling table
2039  *
2040  * @rdev: radeon_device pointer
2041  *
2042  * Starting with SI, the tiling setup is done globally in a
2043  * set of 32 tiling modes.  Rather than selecting each set of
2044  * parameters per surface as on older asics, we just select
2045  * which index in the tiling table we want to use, and the
2046  * surface uses those parameters (CIK).
2047  */
2048 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2049 {
2050         const u32 num_tile_mode_states = 32;
2051         const u32 num_secondary_tile_mode_states = 16;
2052         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2053         u32 num_pipe_configs;
2054         u32 num_rbs = rdev->config.cik.max_backends_per_se *
2055                 rdev->config.cik.max_shader_engines;
2056
2057         switch (rdev->config.cik.mem_row_size_in_kb) {
2058         case 1:
2059                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2060                 break;
2061         case 2:
2062         default:
2063                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2064                 break;
2065         case 4:
2066                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2067                 break;
2068         }
2069
2070         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2071         if (num_pipe_configs > 8)
2072                 num_pipe_configs = 16;
2073
2074         if (num_pipe_configs == 16) {
2075                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2076                         switch (reg_offset) {
2077                         case 0:
2078                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2079                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2080                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2081                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2082                                 break;
2083                         case 1:
2084                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2085                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2086                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2088                                 break;
2089                         case 2:
2090                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2092                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2094                                 break;
2095                         case 3:
2096                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2097                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2098                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2099                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2100                                 break;
2101                         case 4:
2102                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2103                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2104                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105                                                  TILE_SPLIT(split_equal_to_row_size));
2106                                 break;
2107                         case 5:
2108                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2111                                 break;
2112                         case 6:
2113                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2114                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2115                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2117                                 break;
2118                         case 7:
2119                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2120                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2121                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2122                                                  TILE_SPLIT(split_equal_to_row_size));
2123                                 break;
2124                         case 8:
2125                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2126                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2127                                 break;
2128                         case 9:
2129                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2130                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2131                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2132                                 break;
2133                         case 10:
2134                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2135                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2136                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2137                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2138                                 break;
2139                         case 11:
2140                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2141                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2142                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2143                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2144                                 break;
2145                         case 12:
2146                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2147                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2148                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2149                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2150                                 break;
2151                         case 13:
2152                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2153                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2154                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2155                                 break;
2156                         case 14:
2157                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2158                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2159                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2160                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2161                                 break;
2162                         case 16:
2163                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2164                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2165                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2166                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2167                                 break;
2168                         case 17:
2169                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2170                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2171                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2172                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2173                                 break;
2174                         case 27:
2175                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2176                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2177                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2178                                 break;
2179                         case 28:
2180                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2181                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2182                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2183                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2184                                 break;
2185                         case 29:
2186                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2187                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2188                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2189                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2190                                 break;
2191                         case 30:
2192                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2193                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2194                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2195                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2196                                 break;
2197                         default:
2198                                 gb_tile_moden = 0;
2199                                 break;
2200                         }
2201                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2202                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2203                 }
2204                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2205                         switch (reg_offset) {
2206                         case 0:
2207                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2208                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2209                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2210                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2211                                 break;
2212                         case 1:
2213                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2214                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2215                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2216                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2217                                 break;
2218                         case 2:
2219                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2220                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2221                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2222                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2223                                 break;
2224                         case 3:
2225                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2227                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2228                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2229                                 break;
2230                         case 4:
2231                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2232                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2233                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2234                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2235                                 break;
2236                         case 5:
2237                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2239                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2240                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2241                                 break;
2242                         case 6:
2243                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2245                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2246                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2247                                 break;
2248                         case 8:
2249                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2250                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2251                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2252                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2253                                 break;
2254                         case 9:
2255                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2256                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2257                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2258                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2259                                 break;
2260                         case 10:
2261                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2262                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2263                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2264                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2265                                 break;
2266                         case 11:
2267                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2268                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2269                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2270                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2271                                 break;
2272                         case 12:
2273                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2274                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2275                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2276                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2277                                 break;
2278                         case 13:
2279                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2280                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2281                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2282                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2283                                 break;
2284                         case 14:
2285                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2287                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2288                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2289                                 break;
2290                         default:
2291                                 gb_tile_moden = 0;
2292                                 break;
2293                         }
2294                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2295                 }
2296         } else if (num_pipe_configs == 8) {
2297                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2298                         switch (reg_offset) {
2299                         case 0:
2300                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2301                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2302                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2303                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2304                                 break;
2305                         case 1:
2306                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2307                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2308                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2310                                 break;
2311                         case 2:
2312                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2313                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2314                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2315                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2316                                 break;
2317                         case 3:
2318                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2319                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2320                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2321                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2322                                 break;
2323                         case 4:
2324                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2326                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2327                                                  TILE_SPLIT(split_equal_to_row_size));
2328                                 break;
2329                         case 5:
2330                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2331                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2333                                 break;
2334                         case 6:
2335                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2336                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2337                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2338                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2339                                 break;
2340                         case 7:
2341                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2342                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2343                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2344                                                  TILE_SPLIT(split_equal_to_row_size));
2345                                 break;
2346                         case 8:
2347                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2348                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2349                                 break;
2350                         case 9:
2351                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2352                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2353                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2354                                 break;
2355                         case 10:
2356                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2357                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2358                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2359                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2360                                 break;
2361                         case 11:
2362                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2363                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2364                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2365                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2366                                 break;
2367                         case 12:
2368                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2369                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2370                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2371                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2372                                 break;
2373                         case 13:
2374                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2375                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2376                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2377                                 break;
2378                         case 14:
2379                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2380                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2381                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2382                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2383                                 break;
2384                         case 16:
2385                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2386                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2387                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2388                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2389                                 break;
2390                         case 17:
2391                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2392                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2393                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2394                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2395                                 break;
2396                         case 27:
2397                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2398                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2399                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2400                                 break;
2401                         case 28:
2402                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2403                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2404                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2405                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2406                                 break;
2407                         case 29:
2408                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2409                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2410                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2411                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2412                                 break;
2413                         case 30:
2414                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2415                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2416                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2417                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2418                                 break;
2419                         default:
2420                                 gb_tile_moden = 0;
2421                                 break;
2422                         }
2423                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2424                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2425                 }
2426                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2427                         switch (reg_offset) {
2428                         case 0:
2429                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2431                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2432                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2433                                 break;
2434                         case 1:
2435                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2436                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2437                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2438                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2439                                 break;
2440                         case 2:
2441                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2442                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2443                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2444                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2445                                 break;
2446                         case 3:
2447                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2448                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2449                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2450                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2451                                 break;
2452                         case 4:
2453                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2456                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2457                                 break;
2458                         case 5:
2459                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2460                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2461                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2462                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2463                                 break;
2464                         case 6:
2465                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2467                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2468                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2469                                 break;
2470                         case 8:
2471                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2472                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2473                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2474                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2475                                 break;
2476                         case 9:
2477                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2479                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2480                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2481                                 break;
2482                         case 10:
2483                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2484                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2485                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2486                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2487                                 break;
2488                         case 11:
2489                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2492                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2493                                 break;
2494                         case 12:
2495                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2497                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2498                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2499                                 break;
2500                         case 13:
2501                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2503                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2504                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2505                                 break;
2506                         case 14:
2507                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2508                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2509                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2510                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2511                                 break;
2512                         default:
2513                                 gb_tile_moden = 0;
2514                                 break;
2515                         }
2516                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2517                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2518                 }
2519         } else if (num_pipe_configs == 4) {
2520                 if (num_rbs == 4) {
2521                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2522                                 switch (reg_offset) {
2523                                 case 0:
2524                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2526                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2527                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2528                                         break;
2529                                 case 1:
2530                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2531                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2532                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2533                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2534                                         break;
2535                                 case 2:
2536                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2540                                         break;
2541                                 case 3:
2542                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2544                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2545                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2546                                         break;
2547                                 case 4:
2548                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2550                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551                                                          TILE_SPLIT(split_equal_to_row_size));
2552                                         break;
2553                                 case 5:
2554                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2555                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2556                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2557                                         break;
2558                                 case 6:
2559                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2561                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2563                                         break;
2564                                 case 7:
2565                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2566                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2567                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2568                                                          TILE_SPLIT(split_equal_to_row_size));
2569                                         break;
2570                                 case 8:
2571                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2572                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2573                                         break;
2574                                 case 9:
2575                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2577                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2578                                         break;
2579                                 case 10:
2580                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2581                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2582                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2583                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2584                                         break;
2585                                 case 11:
2586                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2587                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2589                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2590                                         break;
2591                                 case 12:
2592                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2593                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2594                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2595                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2596                                         break;
2597                                 case 13:
2598                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2599                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2600                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2601                                         break;
2602                                 case 14:
2603                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2605                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2606                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2607                                         break;
2608                                 case 16:
2609                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2610                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2611                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2612                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2613                                         break;
2614                                 case 17:
2615                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2616                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2617                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2618                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2619                                         break;
2620                                 case 27:
2621                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2622                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2623                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2624                                         break;
2625                                 case 28:
2626                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2627                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2628                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2629                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2630                                         break;
2631                                 case 29:
2632                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2633                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2634                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2635                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2636                                         break;
2637                                 case 30:
2638                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2639                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2640                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2641                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2642                                         break;
2643                                 default:
2644                                         gb_tile_moden = 0;
2645                                         break;
2646                                 }
2647                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2648                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2649                         }
2650                 } else if (num_rbs < 4) {
2651                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2652                                 switch (reg_offset) {
2653                                 case 0:
2654                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2655                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2656                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2657                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2658                                         break;
2659                                 case 1:
2660                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2662                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2664                                         break;
2665                                 case 2:
2666                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2668                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2670                                         break;
2671                                 case 3:
2672                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2674                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2676                                         break;
2677                                 case 4:
2678                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2679                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2680                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681                                                          TILE_SPLIT(split_equal_to_row_size));
2682                                         break;
2683                                 case 5:
2684                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2686                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2687                                         break;
2688                                 case 6:
2689                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2690                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2691                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2693                                         break;
2694                                 case 7:
2695                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2696                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2697                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2698                                                          TILE_SPLIT(split_equal_to_row_size));
2699                                         break;
2700                                 case 8:
2701                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2702                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2703                                         break;
2704                                 case 9:
2705                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2706                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2707                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2708                                         break;
2709                                 case 10:
2710                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2711                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2712                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2713                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2714                                         break;
2715                                 case 11:
2716                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2717                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2718                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2719                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2720                                         break;
2721                                 case 12:
2722                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2723                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2724                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2725                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2726                                         break;
2727                                 case 13:
2728                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2729                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2730                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2731                                         break;
2732                                 case 14:
2733                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2734                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2735                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2736                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2737                                         break;
2738                                 case 16:
2739                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2740                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2741                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2742                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2743                                         break;
2744                                 case 17:
2745                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2746                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2747                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2749                                         break;
2750                                 case 27:
2751                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2752                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2753                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2754                                         break;
2755                                 case 28:
2756                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2757                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2758                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2760                                         break;
2761                                 case 29:
2762                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2763                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2764                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2765                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2766                                         break;
2767                                 case 30:
2768                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2769                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2770                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2772                                         break;
2773                                 default:
2774                                         gb_tile_moden = 0;
2775                                         break;
2776                                 }
2777                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2778                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2779                         }
2780                 }
2781                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2782                         switch (reg_offset) {
2783                         case 0:
2784                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2785                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2786                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2787                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2788                                 break;
2789                         case 1:
2790                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2791                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2792                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2793                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2794                                 break;
2795                         case 2:
2796                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2797                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2798                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2799                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2800                                 break;
2801                         case 3:
2802                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2803                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2804                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2805                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2806                                 break;
2807                         case 4:
2808                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2809                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2810                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2811                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2812                                 break;
2813                         case 5:
2814                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2815                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2816                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2817                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2818                                 break;
2819                         case 6:
2820                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2822                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2823                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2824                                 break;
2825                         case 8:
2826                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2827                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2828                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2829                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2830                                 break;
2831                         case 9:
2832                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2833                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2834                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2835                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2836                                 break;
2837                         case 10:
2838                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2839                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2840                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2841                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2842                                 break;
2843                         case 11:
2844                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2846                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2848                                 break;
2849                         case 12:
2850                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2851                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2852                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2853                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2854                                 break;
2855                         case 13:
2856                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2859                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2860                                 break;
2861                         case 14:
2862                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2863                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2864                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2865                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2866                                 break;
2867                         default:
2868                                 gb_tile_moden = 0;
2869                                 break;
2870                         }
2871                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2872                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2873                 }
2874         } else if (num_pipe_configs == 2) {
2875                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2876                         switch (reg_offset) {
2877                         case 0:
2878                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2879                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2880                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2881                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2882                                 break;
2883                         case 1:
2884                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2886                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2887                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2888                                 break;
2889                         case 2:
2890                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2891                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2892                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2893                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2894                                 break;
2895                         case 3:
2896                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2897                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2898                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2899                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2900                                 break;
2901                         case 4:
2902                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2903                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2904                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2905                                                  TILE_SPLIT(split_equal_to_row_size));
2906                                 break;
2907                         case 5:
2908                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2909                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2910                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2911                                 break;
2912                         case 6:
2913                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2914                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2915                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2916                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2917                                 break;
2918                         case 7:
2919                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2920                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2921                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2922                                                  TILE_SPLIT(split_equal_to_row_size));
2923                                 break;
2924                         case 8:
2925                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2926                                                 PIPE_CONFIG(ADDR_SURF_P2);
2927                                 break;
2928                         case 9:
2929                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2930                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2931                                                  PIPE_CONFIG(ADDR_SURF_P2));
2932                                 break;
2933                         case 10:
2934                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2935                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2936                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2937                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2938                                 break;
2939                         case 11:
2940                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2941                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2942                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2943                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2944                                 break;
2945                         case 12:
2946                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2947                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2948                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2949                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2950                                 break;
2951                         case 13:
2952                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2953                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2954                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2955                                 break;
2956                         case 14:
2957                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2958                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2959                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2960                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2961                                 break;
2962                         case 16:
2963                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2964                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2965                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2966                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2967                                 break;
2968                         case 17:
2969                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2970                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2971                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2972                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2973                                 break;
2974                         case 27:
2975                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2976                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2977                                                  PIPE_CONFIG(ADDR_SURF_P2));
2978                                 break;
2979                         case 28:
2980                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2981                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2982                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2983                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2984                                 break;
2985                         case 29:
2986                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2987                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2988                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2989                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2990                                 break;
2991                         case 30:
2992                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2993                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2994                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2995                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2996                                 break;
2997                         default:
2998                                 gb_tile_moden = 0;
2999                                 break;
3000                         }
3001                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3002                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3003                 }
3004                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3005                         switch (reg_offset) {
3006                         case 0:
3007                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3008                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3009                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3010                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3011                                 break;
3012                         case 1:
3013                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3014                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3015                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3016                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3017                                 break;
3018                         case 2:
3019                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3020                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3021                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3022                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3023                                 break;
3024                         case 3:
3025                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3026                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3027                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3028                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3029                                 break;
3030                         case 4:
3031                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3032                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3033                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3034                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3035                                 break;
3036                         case 5:
3037                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3038                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3039                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3040                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3041                                 break;
3042                         case 6:
3043                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3044                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3045                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3046                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3047                                 break;
3048                         case 8:
3049                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3050                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3051                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3052                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3053                                 break;
3054                         case 9:
3055                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3056                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3057                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3058                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3059                                 break;
3060                         case 10:
3061                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3062                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3063                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3064                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3065                                 break;
3066                         case 11:
3067                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3068                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3069                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3070                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3071                                 break;
3072                         case 12:
3073                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3074                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3075                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3076                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3077                                 break;
3078                         case 13:
3079                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3080                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3081                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3082                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3083                                 break;
3084                         case 14:
3085                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3086                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3087                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3088                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3089                                 break;
3090                         default:
3091                                 gb_tile_moden = 0;
3092                                 break;
3093                         }
3094                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3095                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3096                 }
3097         } else
3098                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3099 }
3100
3101 /**
3102  * cik_select_se_sh - select which SE, SH to address
3103  *
3104  * @rdev: radeon_device pointer
3105  * @se_num: shader engine to address
3106  * @sh_num: sh block to address
3107  *
3108  * Select which SE, SH combinations to address. Certain
3109  * registers are instanced per SE or SH.  0xffffffff means
3110  * broadcast to all SEs or SHs (CIK).
3111  */
3112 static void cik_select_se_sh(struct radeon_device *rdev,
3113                              u32 se_num, u32 sh_num)
3114 {
3115         u32 data = INSTANCE_BROADCAST_WRITES;
3116
3117         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3118                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3119         else if (se_num == 0xffffffff)
3120                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3121         else if (sh_num == 0xffffffff)
3122                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3123         else
3124                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3125         WREG32(GRBM_GFX_INDEX, data);
3126 }
3127
3128 /**
3129  * cik_create_bitmask - create a bitmask
3130  *
3131  * @bit_width: length of the mask
3132  *
3133  * create a variable length bit mask (CIK).
3134  * Returns the bitmask.
3135  */
3136 static u32 cik_create_bitmask(u32 bit_width)
3137 {
3138         u32 i, mask = 0;
3139
3140         for (i = 0; i < bit_width; i++) {
3141                 mask <<= 1;
3142                 mask |= 1;
3143         }
3144         return mask;
3145 }
3146
3147 /**
3148  * cik_get_rb_disabled - computes the mask of disabled RBs
3149  *
3150  * @rdev: radeon_device pointer
3151  * @max_rb_num: max RBs (render backends) for the asic
3152  * @se_num: number of SEs (shader engines) for the asic
3153  * @sh_per_se: number of SH blocks per SE for the asic
3154  *
3155  * Calculates the bitmask of disabled RBs (CIK).
3156  * Returns the disabled RB bitmask.
3157  */
3158 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3159                               u32 max_rb_num_per_se,
3160                               u32 sh_per_se)
3161 {
3162         u32 data, mask;
3163
3164         data = RREG32(CC_RB_BACKEND_DISABLE);
3165         if (data & 1)
3166                 data &= BACKEND_DISABLE_MASK;
3167         else
3168                 data = 0;
3169         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3170
3171         data >>= BACKEND_DISABLE_SHIFT;
3172
3173         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3174
3175         return data & mask;
3176 }
3177
3178 /**
3179  * cik_setup_rb - setup the RBs on the asic
3180  *
3181  * @rdev: radeon_device pointer
3182  * @se_num: number of SEs (shader engines) for the asic
3183  * @sh_per_se: number of SH blocks per SE for the asic
3184  * @max_rb_num: max RBs (render backends) for the asic
3185  *
3186  * Configures per-SE/SH RB registers (CIK).
3187  */
3188 static void cik_setup_rb(struct radeon_device *rdev,
3189                          u32 se_num, u32 sh_per_se,
3190                          u32 max_rb_num_per_se)
3191 {
3192         int i, j;
3193         u32 data, mask;
3194         u32 disabled_rbs = 0;
3195         u32 enabled_rbs = 0;
3196
3197         for (i = 0; i < se_num; i++) {
3198                 for (j = 0; j < sh_per_se; j++) {
3199                         cik_select_se_sh(rdev, i, j);
3200                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3201                         if (rdev->family == CHIP_HAWAII)
3202                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3203                         else
3204                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3205                 }
3206         }
3207         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3208
3209         mask = 1;
3210         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3211                 if (!(disabled_rbs & mask))
3212                         enabled_rbs |= mask;
3213                 mask <<= 1;
3214         }
3215
3216         rdev->config.cik.backend_enable_mask = enabled_rbs;
3217
3218         for (i = 0; i < se_num; i++) {
3219                 cik_select_se_sh(rdev, i, 0xffffffff);
3220                 data = 0;
3221                 for (j = 0; j < sh_per_se; j++) {
3222                         switch (enabled_rbs & 3) {
3223                         case 0:
3224                                 if (j == 0)
3225                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3226                                 else
3227                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3228                                 break;
3229                         case 1:
3230                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3231                                 break;
3232                         case 2:
3233                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3234                                 break;
3235                         case 3:
3236                         default:
3237                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3238                                 break;
3239                         }
3240                         enabled_rbs >>= 2;
3241                 }
3242                 WREG32(PA_SC_RASTER_CONFIG, data);
3243         }
3244         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3245 }
3246
3247 /**
3248  * cik_gpu_init - setup the 3D engine
3249  *
3250  * @rdev: radeon_device pointer
3251  *
3252  * Configures the 3D engine and tiling configuration
3253  * registers so that the 3D engine is usable.
3254  */
3255 static void cik_gpu_init(struct radeon_device *rdev)
3256 {
3257         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3258         u32 mc_shared_chmap, mc_arb_ramcfg;
3259         u32 hdp_host_path_cntl;
3260         u32 tmp;
3261         int i, j, k;
3262
3263         switch (rdev->family) {
3264         case CHIP_BONAIRE:
3265                 rdev->config.cik.max_shader_engines = 2;
3266                 rdev->config.cik.max_tile_pipes = 4;
3267                 rdev->config.cik.max_cu_per_sh = 7;
3268                 rdev->config.cik.max_sh_per_se = 1;
3269                 rdev->config.cik.max_backends_per_se = 2;
3270                 rdev->config.cik.max_texture_channel_caches = 4;
3271                 rdev->config.cik.max_gprs = 256;
3272                 rdev->config.cik.max_gs_threads = 32;
3273                 rdev->config.cik.max_hw_contexts = 8;
3274
3275                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3276                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3277                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3278                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3279                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3280                 break;
3281         case CHIP_HAWAII:
3282                 rdev->config.cik.max_shader_engines = 4;
3283                 rdev->config.cik.max_tile_pipes = 16;
3284                 rdev->config.cik.max_cu_per_sh = 11;
3285                 rdev->config.cik.max_sh_per_se = 1;
3286                 rdev->config.cik.max_backends_per_se = 4;
3287                 rdev->config.cik.max_texture_channel_caches = 16;
3288                 rdev->config.cik.max_gprs = 256;
3289                 rdev->config.cik.max_gs_threads = 32;
3290                 rdev->config.cik.max_hw_contexts = 8;
3291
3292                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3293                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3294                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3295                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3296                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3297                 break;
3298         case CHIP_KAVERI:
3299                 rdev->config.cik.max_shader_engines = 1;
3300                 rdev->config.cik.max_tile_pipes = 4;
3301                 if ((rdev->pdev->device == 0x1304) ||
3302                     (rdev->pdev->device == 0x1305) ||
3303                     (rdev->pdev->device == 0x130C) ||
3304                     (rdev->pdev->device == 0x130F) ||
3305                     (rdev->pdev->device == 0x1310) ||
3306                     (rdev->pdev->device == 0x1311) ||
3307                     (rdev->pdev->device == 0x131C)) {
3308                         rdev->config.cik.max_cu_per_sh = 8;
3309                         rdev->config.cik.max_backends_per_se = 2;
3310                 } else if ((rdev->pdev->device == 0x1309) ||
3311                            (rdev->pdev->device == 0x130A) ||
3312                            (rdev->pdev->device == 0x130D) ||
3313                            (rdev->pdev->device == 0x1313) ||
3314                            (rdev->pdev->device == 0x131D)) {
3315                         rdev->config.cik.max_cu_per_sh = 6;
3316                         rdev->config.cik.max_backends_per_se = 2;
3317                 } else if ((rdev->pdev->device == 0x1306) ||
3318                            (rdev->pdev->device == 0x1307) ||
3319                            (rdev->pdev->device == 0x130B) ||
3320                            (rdev->pdev->device == 0x130E) ||
3321                            (rdev->pdev->device == 0x1315) ||
3322                            (rdev->pdev->device == 0x131B)) {
3323                         rdev->config.cik.max_cu_per_sh = 4;
3324                         rdev->config.cik.max_backends_per_se = 1;
3325                 } else {
3326                         rdev->config.cik.max_cu_per_sh = 3;
3327                         rdev->config.cik.max_backends_per_se = 1;
3328                 }
3329                 rdev->config.cik.max_sh_per_se = 1;
3330                 rdev->config.cik.max_texture_channel_caches = 4;
3331                 rdev->config.cik.max_gprs = 256;
3332                 rdev->config.cik.max_gs_threads = 16;
3333                 rdev->config.cik.max_hw_contexts = 8;
3334
3335                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3336                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3337                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3338                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3339                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3340                 break;
3341         case CHIP_KABINI:
3342         case CHIP_MULLINS:
3343         default:
3344                 rdev->config.cik.max_shader_engines = 1;
3345                 rdev->config.cik.max_tile_pipes = 2;
3346                 rdev->config.cik.max_cu_per_sh = 2;
3347                 rdev->config.cik.max_sh_per_se = 1;
3348                 rdev->config.cik.max_backends_per_se = 1;
3349                 rdev->config.cik.max_texture_channel_caches = 2;
3350                 rdev->config.cik.max_gprs = 256;
3351                 rdev->config.cik.max_gs_threads = 16;
3352                 rdev->config.cik.max_hw_contexts = 8;
3353
3354                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3355                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3356                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3357                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3358                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3359                 break;
3360         }
3361
3362         /* Initialize HDP */
3363         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3364                 WREG32((0x2c14 + j), 0x00000000);
3365                 WREG32((0x2c18 + j), 0x00000000);
3366                 WREG32((0x2c1c + j), 0x00000000);
3367                 WREG32((0x2c20 + j), 0x00000000);
3368                 WREG32((0x2c24 + j), 0x00000000);
3369         }
3370
3371         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3372
3373         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3374
3375         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3376         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3377
3378         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3379         rdev->config.cik.mem_max_burst_length_bytes = 256;
3380         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3381         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3382         if (rdev->config.cik.mem_row_size_in_kb > 4)
3383                 rdev->config.cik.mem_row_size_in_kb = 4;
3384         /* XXX use MC settings? */
3385         rdev->config.cik.shader_engine_tile_size = 32;
3386         rdev->config.cik.num_gpus = 1;
3387         rdev->config.cik.multi_gpu_tile_size = 64;
3388
3389         /* fix up row size */
3390         gb_addr_config &= ~ROW_SIZE_MASK;
3391         switch (rdev->config.cik.mem_row_size_in_kb) {
3392         case 1:
3393         default:
3394                 gb_addr_config |= ROW_SIZE(0);
3395                 break;
3396         case 2:
3397                 gb_addr_config |= ROW_SIZE(1);
3398                 break;
3399         case 4:
3400                 gb_addr_config |= ROW_SIZE(2);
3401                 break;
3402         }
3403
3404         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3405          * not have bank info, so create a custom tiling dword.
3406          * bits 3:0   num_pipes
3407          * bits 7:4   num_banks
3408          * bits 11:8  group_size
3409          * bits 15:12 row_size
3410          */
3411         rdev->config.cik.tile_config = 0;
3412         switch (rdev->config.cik.num_tile_pipes) {
3413         case 1:
3414                 rdev->config.cik.tile_config |= (0 << 0);
3415                 break;
3416         case 2:
3417                 rdev->config.cik.tile_config |= (1 << 0);
3418                 break;
3419         case 4:
3420                 rdev->config.cik.tile_config |= (2 << 0);
3421                 break;
3422         case 8:
3423         default:
3424                 /* XXX what about 12? */
3425                 rdev->config.cik.tile_config |= (3 << 0);
3426                 break;
3427         }
3428         rdev->config.cik.tile_config |=
3429                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3430         rdev->config.cik.tile_config |=
3431                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3432         rdev->config.cik.tile_config |=
3433                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3434
3435         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3436         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3437         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3438         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3439         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3440         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3441         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3442         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3443
3444         cik_tiling_mode_table_init(rdev);
3445
3446         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3447                      rdev->config.cik.max_sh_per_se,
3448                      rdev->config.cik.max_backends_per_se);
3449
3450         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3451                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3452                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3453                                 rdev->config.cik.active_cus +=
3454                                         hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3455                         }
3456                 }
3457         }
3458
3459         /* set HW defaults for 3D engine */
3460         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3461
3462         WREG32(SX_DEBUG_1, 0x20);
3463
3464         WREG32(TA_CNTL_AUX, 0x00010000);
3465
3466         tmp = RREG32(SPI_CONFIG_CNTL);
3467         tmp |= 0x03000000;
3468         WREG32(SPI_CONFIG_CNTL, tmp);
3469
3470         WREG32(SQ_CONFIG, 1);
3471
3472         WREG32(DB_DEBUG, 0);
3473
3474         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3475         tmp |= 0x00000400;
3476         WREG32(DB_DEBUG2, tmp);
3477
3478         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3479         tmp |= 0x00020200;
3480         WREG32(DB_DEBUG3, tmp);
3481
3482         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3483         tmp |= 0x00018208;
3484         WREG32(CB_HW_CONTROL, tmp);
3485
3486         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3487
3488         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3489                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3490                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3491                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3492
3493         WREG32(VGT_NUM_INSTANCES, 1);
3494
3495         WREG32(CP_PERFMON_CNTL, 0);
3496
3497         WREG32(SQ_CONFIG, 0);
3498
3499         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3500                                           FORCE_EOV_MAX_REZ_CNT(255)));
3501
3502         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3503                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3504
3505         WREG32(VGT_GS_VERTEX_REUSE, 16);
3506         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3507
3508         tmp = RREG32(HDP_MISC_CNTL);
3509         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3510         WREG32(HDP_MISC_CNTL, tmp);
3511
3512         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3513         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3514
3515         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3516         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3517
3518         udelay(50);
3519 }
3520
3521 /*
3522  * GPU scratch registers helpers function.
3523  */
3524 /**
3525  * cik_scratch_init - setup driver info for CP scratch regs
3526  *
3527  * @rdev: radeon_device pointer
3528  *
3529  * Set up the number and offset of the CP scratch registers.
3530  * NOTE: use of CP scratch registers is a legacy inferface and
3531  * is not used by default on newer asics (r6xx+).  On newer asics,
3532  * memory buffers are used for fences rather than scratch regs.
3533  */
3534 static void cik_scratch_init(struct radeon_device *rdev)
3535 {
3536         int i;
3537
3538         rdev->scratch.num_reg = 7;
3539         rdev->scratch.reg_base = SCRATCH_REG0;
3540         for (i = 0; i < rdev->scratch.num_reg; i++) {
3541                 rdev->scratch.free[i] = true;
3542                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3543         }
3544 }
3545
3546 /**
3547  * cik_ring_test - basic gfx ring test
3548  *
3549  * @rdev: radeon_device pointer
3550  * @ring: radeon_ring structure holding ring information
3551  *
3552  * Allocate a scratch register and write to it using the gfx ring (CIK).
3553  * Provides a basic gfx ring test to verify that the ring is working.
3554  * Used by cik_cp_gfx_resume();
3555  * Returns 0 on success, error on failure.
3556  */
3557 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3558 {
3559         uint32_t scratch;
3560         uint32_t tmp = 0;
3561         unsigned i;
3562         int r;
3563
3564         r = radeon_scratch_get(rdev, &scratch);
3565         if (r) {
3566                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3567                 return r;
3568         }
3569         WREG32(scratch, 0xCAFEDEAD);
3570         r = radeon_ring_lock(rdev, ring, 3);
3571         if (r) {
3572                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3573                 radeon_scratch_free(rdev, scratch);
3574                 return r;
3575         }
3576         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3577         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3578         radeon_ring_write(ring, 0xDEADBEEF);
3579         radeon_ring_unlock_commit(rdev, ring);
3580
3581         for (i = 0; i < rdev->usec_timeout; i++) {
3582                 tmp = RREG32(scratch);
3583                 if (tmp == 0xDEADBEEF)
3584                         break;
3585                 DRM_UDELAY(1);
3586         }
3587         if (i < rdev->usec_timeout) {
3588                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3589         } else {
3590                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3591                           ring->idx, scratch, tmp);
3592                 r = -EINVAL;
3593         }
3594         radeon_scratch_free(rdev, scratch);
3595         return r;
3596 }
3597
3598 /**
3599  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3600  *
3601  * @rdev: radeon_device pointer
3602  * @ridx: radeon ring index
3603  *
3604  * Emits an hdp flush on the cp.
3605  */
3606 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3607                                        int ridx)
3608 {
3609         struct radeon_ring *ring = &rdev->ring[ridx];
3610         u32 ref_and_mask;
3611
3612         switch (ring->idx) {
3613         case CAYMAN_RING_TYPE_CP1_INDEX:
3614         case CAYMAN_RING_TYPE_CP2_INDEX:
3615         default:
3616                 switch (ring->me) {
3617                 case 0:
3618                         ref_and_mask = CP2 << ring->pipe;
3619                         break;
3620                 case 1:
3621                         ref_and_mask = CP6 << ring->pipe;
3622                         break;
3623                 default:
3624                         return;
3625                 }
3626                 break;
3627         case RADEON_RING_TYPE_GFX_INDEX:
3628                 ref_and_mask = CP0;
3629                 break;
3630         }
3631
3632         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3633         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3634                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3635                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3636         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3637         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3638         radeon_ring_write(ring, ref_and_mask);
3639         radeon_ring_write(ring, ref_and_mask);
3640         radeon_ring_write(ring, 0x20); /* poll interval */
3641 }
3642
3643 /**
3644  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3645  *
3646  * @rdev: radeon_device pointer
3647  * @fence: radeon fence object
3648  *
3649  * Emits a fence sequnce number on the gfx ring and flushes
3650  * GPU caches.
3651  */
3652 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3653                              struct radeon_fence *fence)
3654 {
3655         struct radeon_ring *ring = &rdev->ring[fence->ring];
3656         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3657
3658         /* EVENT_WRITE_EOP - flush caches, send int */
3659         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3660         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3661                                  EOP_TC_ACTION_EN |
3662                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3663                                  EVENT_INDEX(5)));
3664         radeon_ring_write(ring, addr & 0xfffffffc);
3665         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3666         radeon_ring_write(ring, fence->seq);
3667         radeon_ring_write(ring, 0);
3668         /* HDP flush */
3669         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3670 }
3671
3672 /**
3673  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3674  *
3675  * @rdev: radeon_device pointer
3676  * @fence: radeon fence object
3677  *
3678  * Emits a fence sequnce number on the compute ring and flushes
3679  * GPU caches.
3680  */
3681 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3682                                  struct radeon_fence *fence)
3683 {
3684         struct radeon_ring *ring = &rdev->ring[fence->ring];
3685         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3686
3687         /* RELEASE_MEM - flush caches, send int */
3688         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3689         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3690                                  EOP_TC_ACTION_EN |
3691                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3692                                  EVENT_INDEX(5)));
3693         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3694         radeon_ring_write(ring, addr & 0xfffffffc);
3695         radeon_ring_write(ring, upper_32_bits(addr));
3696         radeon_ring_write(ring, fence->seq);
3697         radeon_ring_write(ring, 0);
3698         /* HDP flush */
3699         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3700 }
3701
3702 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3703                              struct radeon_ring *ring,
3704                              struct radeon_semaphore *semaphore,
3705                              bool emit_wait)
3706 {
3707         uint64_t addr = semaphore->gpu_addr;
3708         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3709
3710         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3711         radeon_ring_write(ring, lower_32_bits(addr));
3712         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3713
3714         return true;
3715 }
3716
3717 /**
3718  * cik_copy_cpdma - copy pages using the CP DMA engine
3719  *
3720  * @rdev: radeon_device pointer
3721  * @src_offset: src GPU address
3722  * @dst_offset: dst GPU address
3723  * @num_gpu_pages: number of GPU pages to xfer
3724  * @fence: radeon fence object
3725  *
3726  * Copy GPU paging using the CP DMA engine (CIK+).
3727  * Used by the radeon ttm implementation to move pages if
3728  * registered as the asic copy callback.
3729  */
3730 int cik_copy_cpdma(struct radeon_device *rdev,
3731                    uint64_t src_offset, uint64_t dst_offset,
3732                    unsigned num_gpu_pages,
3733                    struct radeon_fence **fence)
3734 {
3735         struct radeon_semaphore *sem = NULL;
3736         int ring_index = rdev->asic->copy.blit_ring_index;
3737         struct radeon_ring *ring = &rdev->ring[ring_index];
3738         u32 size_in_bytes, cur_size_in_bytes, control;
3739         int i, num_loops;
3740         int r = 0;
3741
3742         r = radeon_semaphore_create(rdev, &sem);
3743         if (r) {
3744                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3745                 return r;
3746         }
3747
3748         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3749         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3750         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3751         if (r) {
3752                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3753                 radeon_semaphore_free(rdev, &sem, NULL);
3754                 return r;
3755         }
3756
3757         radeon_semaphore_sync_to(sem, *fence);
3758         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3759
3760         for (i = 0; i < num_loops; i++) {
3761                 cur_size_in_bytes = size_in_bytes;
3762                 if (cur_size_in_bytes > 0x1fffff)
3763                         cur_size_in_bytes = 0x1fffff;
3764                 size_in_bytes -= cur_size_in_bytes;
3765                 control = 0;
3766                 if (size_in_bytes == 0)
3767                         control |= PACKET3_DMA_DATA_CP_SYNC;
3768                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3769                 radeon_ring_write(ring, control);
3770                 radeon_ring_write(ring, lower_32_bits(src_offset));
3771                 radeon_ring_write(ring, upper_32_bits(src_offset));
3772                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3773                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3774                 radeon_ring_write(ring, cur_size_in_bytes);
3775                 src_offset += cur_size_in_bytes;
3776                 dst_offset += cur_size_in_bytes;
3777         }
3778
3779         r = radeon_fence_emit(rdev, fence, ring->idx);
3780         if (r) {
3781                 radeon_ring_unlock_undo(rdev, ring);
3782                 radeon_semaphore_free(rdev, &sem, NULL);
3783                 return r;
3784         }
3785
3786         radeon_ring_unlock_commit(rdev, ring);
3787         radeon_semaphore_free(rdev, &sem, *fence);
3788
3789         return r;
3790 }
3791
3792 /*
3793  * IB stuff
3794  */
3795 /**
3796  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3797  *
3798  * @rdev: radeon_device pointer
3799  * @ib: radeon indirect buffer object
3800  *
3801  * Emits an DE (drawing engine) or CE (constant engine) IB
3802  * on the gfx ring.  IBs are usually generated by userspace
3803  * acceleration drivers and submitted to the kernel for
3804  * sheduling on the ring.  This function schedules the IB
3805  * on the gfx ring for execution by the GPU.
3806  */
3807 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3808 {
3809         struct radeon_ring *ring = &rdev->ring[ib->ring];
3810         u32 header, control = INDIRECT_BUFFER_VALID;
3811
3812         if (ib->is_const_ib) {
3813                 /* set switch buffer packet before const IB */
3814                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3815                 radeon_ring_write(ring, 0);
3816
3817                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3818         } else {
3819                 u32 next_rptr;
3820                 if (ring->rptr_save_reg) {
3821                         next_rptr = ring->wptr + 3 + 4;
3822                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3823                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3824                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3825                         radeon_ring_write(ring, next_rptr);
3826                 } else if (rdev->wb.enabled) {
3827                         next_rptr = ring->wptr + 5 + 4;
3828                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3829                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3830                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3831                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3832                         radeon_ring_write(ring, next_rptr);
3833                 }
3834
3835                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3836         }
3837
3838         control |= ib->length_dw |
3839                 (ib->vm ? (ib->vm->id << 24) : 0);
3840
3841         radeon_ring_write(ring, header);
3842         radeon_ring_write(ring,
3843 #ifdef __BIG_ENDIAN
3844                           (2 << 0) |
3845 #endif
3846                           (ib->gpu_addr & 0xFFFFFFFC));
3847         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3848         radeon_ring_write(ring, control);
3849 }
3850
3851 /**
3852  * cik_ib_test - basic gfx ring IB test
3853  *
3854  * @rdev: radeon_device pointer
3855  * @ring: radeon_ring structure holding ring information
3856  *
3857  * Allocate an IB and execute it on the gfx ring (CIK).
3858  * Provides a basic gfx ring test to verify that IBs are working.
3859  * Returns 0 on success, error on failure.
3860  */
3861 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3862 {
3863         struct radeon_ib ib;
3864         uint32_t scratch;
3865         uint32_t tmp = 0;
3866         unsigned i;
3867         int r;
3868
3869         r = radeon_scratch_get(rdev, &scratch);
3870         if (r) {
3871                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3872                 return r;
3873         }
3874         WREG32(scratch, 0xCAFEDEAD);
3875         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3876         if (r) {
3877                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3878                 radeon_scratch_free(rdev, scratch);
3879                 return r;
3880         }
3881         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3882         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3883         ib.ptr[2] = 0xDEADBEEF;
3884         ib.length_dw = 3;
3885         r = radeon_ib_schedule(rdev, &ib, NULL);
3886         if (r) {
3887                 radeon_scratch_free(rdev, scratch);
3888                 radeon_ib_free(rdev, &ib);
3889                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3890                 return r;
3891         }
3892         r = radeon_fence_wait(ib.fence, false);
3893         if (r) {
3894                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3895                 radeon_scratch_free(rdev, scratch);
3896                 radeon_ib_free(rdev, &ib);
3897                 return r;
3898         }
3899         for (i = 0; i < rdev->usec_timeout; i++) {
3900                 tmp = RREG32(scratch);
3901                 if (tmp == 0xDEADBEEF)
3902                         break;
3903                 DRM_UDELAY(1);
3904         }
3905         if (i < rdev->usec_timeout) {
3906                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3907         } else {
3908                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3909                           scratch, tmp);
3910                 r = -EINVAL;
3911         }
3912         radeon_scratch_free(rdev, scratch);
3913         radeon_ib_free(rdev, &ib);
3914         return r;
3915 }
3916
3917 /*
3918  * CP.
3919  * On CIK, gfx and compute now have independant command processors.
3920  *
3921  * GFX
3922  * Gfx consists of a single ring and can process both gfx jobs and
3923  * compute jobs.  The gfx CP consists of three microengines (ME):
3924  * PFP - Pre-Fetch Parser
3925  * ME - Micro Engine
3926  * CE - Constant Engine
3927  * The PFP and ME make up what is considered the Drawing Engine (DE).
3928  * The CE is an asynchronous engine used for updating buffer desciptors
3929  * used by the DE so that they can be loaded into cache in parallel
3930  * while the DE is processing state update packets.
3931  *
3932  * Compute
3933  * The compute CP consists of two microengines (ME):
3934  * MEC1 - Compute MicroEngine 1
3935  * MEC2 - Compute MicroEngine 2
3936  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3937  * The queues are exposed to userspace and are programmed directly
3938  * by the compute runtime.
3939  */
3940 /**
3941  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3942  *
3943  * @rdev: radeon_device pointer
3944  * @enable: enable or disable the MEs
3945  *
3946  * Halts or unhalts the gfx MEs.
3947  */
3948 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3949 {
3950         if (enable)
3951                 WREG32(CP_ME_CNTL, 0);
3952         else {
3953                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3954                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3955                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3956                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3957         }
3958         udelay(50);
3959 }
3960
3961 /**
3962  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3963  *
3964  * @rdev: radeon_device pointer
3965  *
3966  * Loads the gfx PFP, ME, and CE ucode.
3967  * Returns 0 for success, -EINVAL if the ucode is not available.
3968  */
3969 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3970 {
3971         const __be32 *fw_data;
3972         int i;
3973
3974         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3975                 return -EINVAL;
3976
3977         cik_cp_gfx_enable(rdev, false);
3978
3979         /* PFP */
3980         fw_data = (const __be32 *)rdev->pfp_fw->data;
3981         WREG32(CP_PFP_UCODE_ADDR, 0);
3982         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3983                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3984         WREG32(CP_PFP_UCODE_ADDR, 0);
3985
3986         /* CE */
3987         fw_data = (const __be32 *)rdev->ce_fw->data;
3988         WREG32(CP_CE_UCODE_ADDR, 0);
3989         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3990                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3991         WREG32(CP_CE_UCODE_ADDR, 0);
3992
3993         /* ME */
3994         fw_data = (const __be32 *)rdev->me_fw->data;
3995         WREG32(CP_ME_RAM_WADDR, 0);
3996         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3997                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3998         WREG32(CP_ME_RAM_WADDR, 0);
3999
4000         WREG32(CP_PFP_UCODE_ADDR, 0);
4001         WREG32(CP_CE_UCODE_ADDR, 0);
4002         WREG32(CP_ME_RAM_WADDR, 0);
4003         WREG32(CP_ME_RAM_RADDR, 0);
4004         return 0;
4005 }
4006
4007 /**
4008  * cik_cp_gfx_start - start the gfx ring
4009  *
4010  * @rdev: radeon_device pointer
4011  *
4012  * Enables the ring and loads the clear state context and other
4013  * packets required to init the ring.
4014  * Returns 0 for success, error for failure.
4015  */
4016 static int cik_cp_gfx_start(struct radeon_device *rdev)
4017 {
4018         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4019         int r, i;
4020
4021         /* init the CP */
4022         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4023         WREG32(CP_ENDIAN_SWAP, 0);
4024         WREG32(CP_DEVICE_ID, 1);
4025
4026         cik_cp_gfx_enable(rdev, true);
4027
4028         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4029         if (r) {
4030                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4031                 return r;
4032         }
4033
4034         /* init the CE partitions.  CE only used for gfx on CIK */
4035         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4036         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4037         radeon_ring_write(ring, 0xc000);
4038         radeon_ring_write(ring, 0xc000);
4039
4040         /* setup clear context state */
4041         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4042         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4043
4044         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4045         radeon_ring_write(ring, 0x80000000);
4046         radeon_ring_write(ring, 0x80000000);
4047
4048         for (i = 0; i < cik_default_size; i++)
4049                 radeon_ring_write(ring, cik_default_state[i]);
4050
4051         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4052         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4053
4054         /* set clear context state */
4055         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4056         radeon_ring_write(ring, 0);
4057
4058         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4059         radeon_ring_write(ring, 0x00000316);
4060         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4061         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4062
4063         radeon_ring_unlock_commit(rdev, ring);
4064
4065         return 0;
4066 }
4067
4068 /**
4069  * cik_cp_gfx_fini - stop the gfx ring
4070  *
4071  * @rdev: radeon_device pointer
4072  *
4073  * Stop the gfx ring and tear down the driver ring
4074  * info.
4075  */
4076 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4077 {
4078         cik_cp_gfx_enable(rdev, false);
4079         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4080 }
4081
4082 /**
4083  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4084  *
4085  * @rdev: radeon_device pointer
4086  *
4087  * Program the location and size of the gfx ring buffer
4088  * and test it to make sure it's working.
4089  * Returns 0 for success, error for failure.
4090  */
4091 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4092 {
4093         struct radeon_ring *ring;
4094         u32 tmp;
4095         u32 rb_bufsz;
4096         u64 rb_addr;
4097         int r;
4098
4099         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4100         if (rdev->family != CHIP_HAWAII)
4101                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4102
4103         /* Set the write pointer delay */
4104         WREG32(CP_RB_WPTR_DELAY, 0);
4105
4106         /* set the RB to use vmid 0 */
4107         WREG32(CP_RB_VMID, 0);
4108
4109         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4110
4111         /* ring 0 - compute and gfx */
4112         /* Set ring buffer size */
4113         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4114         rb_bufsz = order_base_2(ring->ring_size / 8);
4115         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4116 #ifdef __BIG_ENDIAN
4117         tmp |= BUF_SWAP_32BIT;
4118 #endif
4119         WREG32(CP_RB0_CNTL, tmp);
4120
4121         /* Initialize the ring buffer's read and write pointers */
4122         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4123         ring->wptr = 0;
4124         WREG32(CP_RB0_WPTR, ring->wptr);
4125
4126         /* set the wb address wether it's enabled or not */
4127         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4128         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4129
4130         /* scratch register shadowing is no longer supported */
4131         WREG32(SCRATCH_UMSK, 0);
4132
4133         if (!rdev->wb.enabled)
4134                 tmp |= RB_NO_UPDATE;
4135
4136         mdelay(1);
4137         WREG32(CP_RB0_CNTL, tmp);
4138
4139         rb_addr = ring->gpu_addr >> 8;
4140         WREG32(CP_RB0_BASE, rb_addr);
4141         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4142
4143         /* start the ring */
4144         cik_cp_gfx_start(rdev);
4145         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4146         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4147         if (r) {
4148                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4149                 return r;
4150         }
4151
4152         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4153                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4154
4155         return 0;
4156 }
4157
4158 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4159                      struct radeon_ring *ring)
4160 {
4161         u32 rptr;
4162
4163         if (rdev->wb.enabled)
4164                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4165         else
4166                 rptr = RREG32(CP_RB0_RPTR);
4167
4168         return rptr;
4169 }
4170
4171 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4172                      struct radeon_ring *ring)
4173 {
4174         u32 wptr;
4175
4176         wptr = RREG32(CP_RB0_WPTR);
4177
4178         return wptr;
4179 }
4180
4181 void cik_gfx_set_wptr(struct radeon_device *rdev,
4182                       struct radeon_ring *ring)
4183 {
4184         WREG32(CP_RB0_WPTR, ring->wptr);
4185         (void)RREG32(CP_RB0_WPTR);
4186 }
4187
4188 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4189                          struct radeon_ring *ring)
4190 {
4191         u32 rptr;
4192
4193         if (rdev->wb.enabled) {
4194                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4195         } else {
4196                 mutex_lock(&rdev->srbm_mutex);
4197                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4198                 rptr = RREG32(CP_HQD_PQ_RPTR);
4199                 cik_srbm_select(rdev, 0, 0, 0, 0);
4200                 mutex_unlock(&rdev->srbm_mutex);
4201         }
4202
4203         return rptr;
4204 }
4205
4206 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4207                          struct radeon_ring *ring)
4208 {
4209         u32 wptr;
4210
4211         if (rdev->wb.enabled) {
4212                 /* XXX check if swapping is necessary on BE */
4213                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4214         } else {
4215                 mutex_lock(&rdev->srbm_mutex);
4216                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4217                 wptr = RREG32(CP_HQD_PQ_WPTR);
4218                 cik_srbm_select(rdev, 0, 0, 0, 0);
4219                 mutex_unlock(&rdev->srbm_mutex);
4220         }
4221
4222         return wptr;
4223 }
4224
4225 void cik_compute_set_wptr(struct radeon_device *rdev,
4226                           struct radeon_ring *ring)
4227 {
4228         /* XXX check if swapping is necessary on BE */
4229         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4230         WDOORBELL32(ring->doorbell_index, ring->wptr);
4231 }
4232
4233 /**
4234  * cik_cp_compute_enable - enable/disable the compute CP MEs
4235  *
4236  * @rdev: radeon_device pointer
4237  * @enable: enable or disable the MEs
4238  *
4239  * Halts or unhalts the compute MEs.
4240  */
4241 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4242 {
4243         if (enable)
4244                 WREG32(CP_MEC_CNTL, 0);
4245         else {
4246                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4247                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4248                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4249         }
4250         udelay(50);
4251 }
4252
4253 /**
4254  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4255  *
4256  * @rdev: radeon_device pointer
4257  *
4258  * Loads the compute MEC1&2 ucode.
4259  * Returns 0 for success, -EINVAL if the ucode is not available.
4260  */
4261 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4262 {
4263         const __be32 *fw_data;
4264         int i;
4265
4266         if (!rdev->mec_fw)
4267                 return -EINVAL;
4268
4269         cik_cp_compute_enable(rdev, false);
4270
4271         /* MEC1 */
4272         fw_data = (const __be32 *)rdev->mec_fw->data;
4273         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4274         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4275                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4276         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4277
4278         if (rdev->family == CHIP_KAVERI) {
4279                 /* MEC2 */
4280                 fw_data = (const __be32 *)rdev->mec_fw->data;
4281                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4282                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4283                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4284                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4285         }
4286
4287         return 0;
4288 }
4289
4290 /**
4291  * cik_cp_compute_start - start the compute queues
4292  *
4293  * @rdev: radeon_device pointer
4294  *
4295  * Enable the compute queues.
4296  * Returns 0 for success, error for failure.
4297  */
4298 static int cik_cp_compute_start(struct radeon_device *rdev)
4299 {
4300         cik_cp_compute_enable(rdev, true);
4301
4302         return 0;
4303 }
4304
4305 /**
4306  * cik_cp_compute_fini - stop the compute queues
4307  *
4308  * @rdev: radeon_device pointer
4309  *
4310  * Stop the compute queues and tear down the driver queue
4311  * info.
4312  */
4313 static void cik_cp_compute_fini(struct radeon_device *rdev)
4314 {
4315         int i, idx, r;
4316
4317         cik_cp_compute_enable(rdev, false);
4318
4319         for (i = 0; i < 2; i++) {
4320                 if (i == 0)
4321                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4322                 else
4323                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4324
4325                 if (rdev->ring[idx].mqd_obj) {
4326                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4327                         if (unlikely(r != 0))
4328                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4329
4330                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4331                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4332
4333                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4334                         rdev->ring[idx].mqd_obj = NULL;
4335                 }
4336         }
4337 }
4338
4339 static void cik_mec_fini(struct radeon_device *rdev)
4340 {
4341         int r;
4342
4343         if (rdev->mec.hpd_eop_obj) {
4344                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4345                 if (unlikely(r != 0))
4346                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4347                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4348                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4349
4350                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4351                 rdev->mec.hpd_eop_obj = NULL;
4352         }
4353 }
4354
4355 #define MEC_HPD_SIZE 2048
4356
4357 static int cik_mec_init(struct radeon_device *rdev)
4358 {
4359         int r;
4360         u32 *hpd;
4361
4362         /*
4363          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4364          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4365          */
4366         if (rdev->family == CHIP_KAVERI)
4367                 rdev->mec.num_mec = 2;
4368         else
4369                 rdev->mec.num_mec = 1;
4370         rdev->mec.num_pipe = 4;
4371         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4372
4373         if (rdev->mec.hpd_eop_obj == NULL) {
4374                 r = radeon_bo_create(rdev,
4375                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4376                                      PAGE_SIZE, true,
4377                                      RADEON_GEM_DOMAIN_GTT, NULL,
4378                                      &rdev->mec.hpd_eop_obj);
4379                 if (r) {
4380                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4381                         return r;
4382                 }
4383         }
4384
4385         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4386         if (unlikely(r != 0)) {
4387                 cik_mec_fini(rdev);
4388                 return r;
4389         }
4390         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4391                           &rdev->mec.hpd_eop_gpu_addr);
4392         if (r) {
4393                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4394                 cik_mec_fini(rdev);
4395                 return r;
4396         }
4397         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4398         if (r) {
4399                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4400                 cik_mec_fini(rdev);
4401                 return r;
4402         }
4403
4404         /* clear memory.  Not sure if this is required or not */
4405         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4406
4407         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4408         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4409
4410         return 0;
4411 }
4412
4413 struct hqd_registers
4414 {
4415         u32 cp_mqd_base_addr;
4416         u32 cp_mqd_base_addr_hi;
4417         u32 cp_hqd_active;
4418         u32 cp_hqd_vmid;
4419         u32 cp_hqd_persistent_state;
4420         u32 cp_hqd_pipe_priority;
4421         u32 cp_hqd_queue_priority;
4422         u32 cp_hqd_quantum;
4423         u32 cp_hqd_pq_base;
4424         u32 cp_hqd_pq_base_hi;
4425         u32 cp_hqd_pq_rptr;
4426         u32 cp_hqd_pq_rptr_report_addr;
4427         u32 cp_hqd_pq_rptr_report_addr_hi;
4428         u32 cp_hqd_pq_wptr_poll_addr;
4429         u32 cp_hqd_pq_wptr_poll_addr_hi;
4430         u32 cp_hqd_pq_doorbell_control;
4431         u32 cp_hqd_pq_wptr;
4432         u32 cp_hqd_pq_control;
4433         u32 cp_hqd_ib_base_addr;
4434         u32 cp_hqd_ib_base_addr_hi;
4435         u32 cp_hqd_ib_rptr;
4436         u32 cp_hqd_ib_control;
4437         u32 cp_hqd_iq_timer;
4438         u32 cp_hqd_iq_rptr;
4439         u32 cp_hqd_dequeue_request;
4440         u32 cp_hqd_dma_offload;
4441         u32 cp_hqd_sema_cmd;
4442         u32 cp_hqd_msg_type;
4443         u32 cp_hqd_atomic0_preop_lo;
4444         u32 cp_hqd_atomic0_preop_hi;
4445         u32 cp_hqd_atomic1_preop_lo;
4446         u32 cp_hqd_atomic1_preop_hi;
4447         u32 cp_hqd_hq_scheduler0;
4448         u32 cp_hqd_hq_scheduler1;
4449         u32 cp_mqd_control;
4450 };
4451
4452 struct bonaire_mqd
4453 {
4454         u32 header;
4455         u32 dispatch_initiator;
4456         u32 dimensions[3];
4457         u32 start_idx[3];
4458         u32 num_threads[3];
4459         u32 pipeline_stat_enable;
4460         u32 perf_counter_enable;
4461         u32 pgm[2];
4462         u32 tba[2];
4463         u32 tma[2];
4464         u32 pgm_rsrc[2];
4465         u32 vmid;
4466         u32 resource_limits;
4467         u32 static_thread_mgmt01[2];
4468         u32 tmp_ring_size;
4469         u32 static_thread_mgmt23[2];
4470         u32 restart[3];
4471         u32 thread_trace_enable;
4472         u32 reserved1;
4473         u32 user_data[16];
4474         u32 vgtcs_invoke_count[2];
4475         struct hqd_registers queue_state;
4476         u32 dequeue_cntr;
4477         u32 interrupt_queue[64];
4478 };
4479
4480 /**
4481  * cik_cp_compute_resume - setup the compute queue registers
4482  *
4483  * @rdev: radeon_device pointer
4484  *
4485  * Program the compute queues and test them to make sure they
4486  * are working.
4487  * Returns 0 for success, error for failure.
4488  */
4489 static int cik_cp_compute_resume(struct radeon_device *rdev)
4490 {
4491         int r, i, idx;
4492         u32 tmp;
4493         bool use_doorbell = true;
4494         u64 hqd_gpu_addr;
4495         u64 mqd_gpu_addr;
4496         u64 eop_gpu_addr;
4497         u64 wb_gpu_addr;
4498         u32 *buf;
4499         struct bonaire_mqd *mqd;
4500
4501         r = cik_cp_compute_start(rdev);
4502         if (r)
4503                 return r;
4504
4505         /* fix up chicken bits */
4506         tmp = RREG32(CP_CPF_DEBUG);
4507         tmp |= (1 << 23);
4508         WREG32(CP_CPF_DEBUG, tmp);
4509
4510         /* init the pipes */
4511         mutex_lock(&rdev->srbm_mutex);
4512         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4513                 int me = (i < 4) ? 1 : 2;
4514                 int pipe = (i < 4) ? i : (i - 4);
4515
4516                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4517
4518                 cik_srbm_select(rdev, me, pipe, 0, 0);
4519
4520                 /* write the EOP addr */
4521                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4522                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4523
4524                 /* set the VMID assigned */
4525                 WREG32(CP_HPD_EOP_VMID, 0);
4526
4527                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4528                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4529                 tmp &= ~EOP_SIZE_MASK;
4530                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4531                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4532         }
4533         cik_srbm_select(rdev, 0, 0, 0, 0);
4534         mutex_unlock(&rdev->srbm_mutex);
4535
4536         /* init the queues.  Just two for now. */
4537         for (i = 0; i < 2; i++) {
4538                 if (i == 0)
4539                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4540                 else
4541                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4542
4543                 if (rdev->ring[idx].mqd_obj == NULL) {
4544                         r = radeon_bo_create(rdev,
4545                                              sizeof(struct bonaire_mqd),
4546                                              PAGE_SIZE, true,
4547                                              RADEON_GEM_DOMAIN_GTT, NULL,
4548                                              &rdev->ring[idx].mqd_obj);
4549                         if (r) {
4550                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4551                                 return r;
4552                         }
4553                 }
4554
4555                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4556                 if (unlikely(r != 0)) {
4557                         cik_cp_compute_fini(rdev);
4558                         return r;
4559                 }
4560                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4561                                   &mqd_gpu_addr);
4562                 if (r) {
4563                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4564                         cik_cp_compute_fini(rdev);
4565                         return r;
4566                 }
4567                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4568                 if (r) {
4569                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4570                         cik_cp_compute_fini(rdev);
4571                         return r;
4572                 }
4573
4574                 /* init the mqd struct */
4575                 memset(buf, 0, sizeof(struct bonaire_mqd));
4576
4577                 mqd = (struct bonaire_mqd *)buf;
4578                 mqd->header = 0xC0310800;
4579                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4580                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4581                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4582                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4583
4584                 mutex_lock(&rdev->srbm_mutex);
4585                 cik_srbm_select(rdev, rdev->ring[idx].me,
4586                                 rdev->ring[idx].pipe,
4587                                 rdev->ring[idx].queue, 0);
4588
4589                 /* disable wptr polling */
4590                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4591                 tmp &= ~WPTR_POLL_EN;
4592                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4593
4594                 /* enable doorbell? */
4595                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4596                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4597                 if (use_doorbell)
4598                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4599                 else
4600                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4601                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4602                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4603
4604                 /* disable the queue if it's active */
4605                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4606                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4607                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4608                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4609                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4610                         for (i = 0; i < rdev->usec_timeout; i++) {
4611                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4612                                         break;
4613                                 udelay(1);
4614                         }
4615                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4616                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4617                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4618                 }
4619
4620                 /* set the pointer to the MQD */
4621                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4622                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4623                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4624                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4625                 /* set MQD vmid to 0 */
4626                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4627                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4628                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4629
4630                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4631                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4632                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4633                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4634                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4635                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4636
4637                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4638                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4639                 mqd->queue_state.cp_hqd_pq_control &=
4640                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4641
4642                 mqd->queue_state.cp_hqd_pq_control |=
4643                         order_base_2(rdev->ring[idx].ring_size / 8);
4644                 mqd->queue_state.cp_hqd_pq_control |=
4645                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4646 #ifdef __BIG_ENDIAN
4647                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4648 #endif
4649                 mqd->queue_state.cp_hqd_pq_control &=
4650                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4651                 mqd->queue_state.cp_hqd_pq_control |=
4652                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4653                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4654
4655                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4656                 if (i == 0)
4657                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4658                 else
4659                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4660                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4661                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4662                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4663                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4664                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4665
4666                 /* set the wb address wether it's enabled or not */
4667                 if (i == 0)
4668                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4669                 else
4670                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4671                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4672                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4673                         upper_32_bits(wb_gpu_addr) & 0xffff;
4674                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4675                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4676                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4677                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4678
4679                 /* enable the doorbell if requested */
4680                 if (use_doorbell) {
4681                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4682                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4683                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4684                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4685                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4686                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4687                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4688                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4689
4690                 } else {
4691                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4692                 }
4693                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4694                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4695
4696                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4697                 rdev->ring[idx].wptr = 0;
4698                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4699                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4700                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4701
4702                 /* set the vmid for the queue */
4703                 mqd->queue_state.cp_hqd_vmid = 0;
4704                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4705
4706                 /* activate the queue */
4707                 mqd->queue_state.cp_hqd_active = 1;
4708                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4709
4710                 cik_srbm_select(rdev, 0, 0, 0, 0);
4711                 mutex_unlock(&rdev->srbm_mutex);
4712
4713                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4714                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4715
4716                 rdev->ring[idx].ready = true;
4717                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4718                 if (r)
4719                         rdev->ring[idx].ready = false;
4720         }
4721
4722         return 0;
4723 }
4724
4725 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4726 {
4727         cik_cp_gfx_enable(rdev, enable);
4728         cik_cp_compute_enable(rdev, enable);
4729 }
4730
4731 static int cik_cp_load_microcode(struct radeon_device *rdev)
4732 {
4733         int r;
4734
4735         r = cik_cp_gfx_load_microcode(rdev);
4736         if (r)
4737                 return r;
4738         r = cik_cp_compute_load_microcode(rdev);
4739         if (r)
4740                 return r;
4741
4742         return 0;
4743 }
4744
4745 static void cik_cp_fini(struct radeon_device *rdev)
4746 {
4747         cik_cp_gfx_fini(rdev);
4748         cik_cp_compute_fini(rdev);
4749 }
4750
4751 static int cik_cp_resume(struct radeon_device *rdev)
4752 {
4753         int r;
4754
4755         cik_enable_gui_idle_interrupt(rdev, false);
4756
4757         r = cik_cp_load_microcode(rdev);
4758         if (r)
4759                 return r;
4760
4761         r = cik_cp_gfx_resume(rdev);
4762         if (r)
4763                 return r;
4764         r = cik_cp_compute_resume(rdev);
4765         if (r)
4766                 return r;
4767
4768         cik_enable_gui_idle_interrupt(rdev, true);
4769
4770         return 0;
4771 }
4772
4773 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4774 {
4775         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4776                 RREG32(GRBM_STATUS));
4777         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4778                 RREG32(GRBM_STATUS2));
4779         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4780                 RREG32(GRBM_STATUS_SE0));
4781         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4782                 RREG32(GRBM_STATUS_SE1));
4783         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4784                 RREG32(GRBM_STATUS_SE2));
4785         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4786                 RREG32(GRBM_STATUS_SE3));
4787         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4788                 RREG32(SRBM_STATUS));
4789         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4790                 RREG32(SRBM_STATUS2));
4791         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4792                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4793         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4794                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4795         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4796         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4797                  RREG32(CP_STALLED_STAT1));
4798         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4799                  RREG32(CP_STALLED_STAT2));
4800         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4801                  RREG32(CP_STALLED_STAT3));
4802         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4803                  RREG32(CP_CPF_BUSY_STAT));
4804         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4805                  RREG32(CP_CPF_STALLED_STAT1));
4806         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4807         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4808         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4809                  RREG32(CP_CPC_STALLED_STAT1));
4810         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4811 }
4812
4813 /**
4814  * cik_gpu_check_soft_reset - check which blocks are busy
4815  *
4816  * @rdev: radeon_device pointer
4817  *
4818  * Check which blocks are busy and return the relevant reset
4819  * mask to be used by cik_gpu_soft_reset().
4820  * Returns a mask of the blocks to be reset.
4821  */
4822 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4823 {
4824         u32 reset_mask = 0;
4825         u32 tmp;
4826
4827         /* GRBM_STATUS */
4828         tmp = RREG32(GRBM_STATUS);
4829         if (tmp & (PA_BUSY | SC_BUSY |
4830                    BCI_BUSY | SX_BUSY |
4831                    TA_BUSY | VGT_BUSY |
4832                    DB_BUSY | CB_BUSY |
4833                    GDS_BUSY | SPI_BUSY |
4834                    IA_BUSY | IA_BUSY_NO_DMA))
4835                 reset_mask |= RADEON_RESET_GFX;
4836
4837         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4838                 reset_mask |= RADEON_RESET_CP;
4839
4840         /* GRBM_STATUS2 */
4841         tmp = RREG32(GRBM_STATUS2);
4842         if (tmp & RLC_BUSY)
4843                 reset_mask |= RADEON_RESET_RLC;
4844
4845         /* SDMA0_STATUS_REG */
4846         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4847         if (!(tmp & SDMA_IDLE))
4848                 reset_mask |= RADEON_RESET_DMA;
4849
4850         /* SDMA1_STATUS_REG */
4851         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4852         if (!(tmp & SDMA_IDLE))
4853                 reset_mask |= RADEON_RESET_DMA1;
4854
4855         /* SRBM_STATUS2 */
4856         tmp = RREG32(SRBM_STATUS2);
4857         if (tmp & SDMA_BUSY)
4858                 reset_mask |= RADEON_RESET_DMA;
4859
4860         if (tmp & SDMA1_BUSY)
4861                 reset_mask |= RADEON_RESET_DMA1;
4862
4863         /* SRBM_STATUS */
4864         tmp = RREG32(SRBM_STATUS);
4865
4866         if (tmp & IH_BUSY)
4867                 reset_mask |= RADEON_RESET_IH;
4868
4869         if (tmp & SEM_BUSY)
4870                 reset_mask |= RADEON_RESET_SEM;
4871
4872         if (tmp & GRBM_RQ_PENDING)
4873                 reset_mask |= RADEON_RESET_GRBM;
4874
4875         if (tmp & VMC_BUSY)
4876                 reset_mask |= RADEON_RESET_VMC;
4877
4878         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4879                    MCC_BUSY | MCD_BUSY))
4880                 reset_mask |= RADEON_RESET_MC;
4881
4882         if (evergreen_is_display_hung(rdev))
4883                 reset_mask |= RADEON_RESET_DISPLAY;
4884
4885         /* Skip MC reset as it's mostly likely not hung, just busy */
4886         if (reset_mask & RADEON_RESET_MC) {
4887                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4888                 reset_mask &= ~RADEON_RESET_MC;
4889         }
4890
4891         return reset_mask;
4892 }
4893
4894 /**
4895  * cik_gpu_soft_reset - soft reset GPU
4896  *
4897  * @rdev: radeon_device pointer
4898  * @reset_mask: mask of which blocks to reset
4899  *
4900  * Soft reset the blocks specified in @reset_mask.
4901  */
4902 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4903 {
4904         struct evergreen_mc_save save;
4905         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4906         u32 tmp;
4907
4908         if (reset_mask == 0)
4909                 return;
4910
4911         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4912
4913         cik_print_gpu_status_regs(rdev);
4914         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4915                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4916         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4917                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4918
4919         /* disable CG/PG */
4920         cik_fini_pg(rdev);
4921         cik_fini_cg(rdev);
4922
4923         /* stop the rlc */
4924         cik_rlc_stop(rdev);
4925
4926         /* Disable GFX parsing/prefetching */
4927         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4928
4929         /* Disable MEC parsing/prefetching */
4930         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4931
4932         if (reset_mask & RADEON_RESET_DMA) {
4933                 /* sdma0 */
4934                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4935                 tmp |= SDMA_HALT;
4936                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4937         }
4938         if (reset_mask & RADEON_RESET_DMA1) {
4939                 /* sdma1 */
4940                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4941                 tmp |= SDMA_HALT;
4942                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4943         }
4944
4945         evergreen_mc_stop(rdev, &save);
4946         if (evergreen_mc_wait_for_idle(rdev)) {
4947                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4948         }
4949
4950         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4951                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4952
4953         if (reset_mask & RADEON_RESET_CP) {
4954                 grbm_soft_reset |= SOFT_RESET_CP;
4955
4956                 srbm_soft_reset |= SOFT_RESET_GRBM;
4957         }
4958
4959         if (reset_mask & RADEON_RESET_DMA)
4960                 srbm_soft_reset |= SOFT_RESET_SDMA;
4961
4962         if (reset_mask & RADEON_RESET_DMA1)
4963                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4964
4965         if (reset_mask & RADEON_RESET_DISPLAY)
4966                 srbm_soft_reset |= SOFT_RESET_DC;
4967
4968         if (reset_mask & RADEON_RESET_RLC)
4969                 grbm_soft_reset |= SOFT_RESET_RLC;
4970
4971         if (reset_mask & RADEON_RESET_SEM)
4972                 srbm_soft_reset |= SOFT_RESET_SEM;
4973
4974         if (reset_mask & RADEON_RESET_IH)
4975                 srbm_soft_reset |= SOFT_RESET_IH;
4976
4977         if (reset_mask & RADEON_RESET_GRBM)
4978                 srbm_soft_reset |= SOFT_RESET_GRBM;
4979
4980         if (reset_mask & RADEON_RESET_VMC)
4981                 srbm_soft_reset |= SOFT_RESET_VMC;
4982
4983         if (!(rdev->flags & RADEON_IS_IGP)) {
4984                 if (reset_mask & RADEON_RESET_MC)
4985                         srbm_soft_reset |= SOFT_RESET_MC;
4986         }
4987
4988         if (grbm_soft_reset) {
4989                 tmp = RREG32(GRBM_SOFT_RESET);
4990                 tmp |= grbm_soft_reset;
4991                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4992                 WREG32(GRBM_SOFT_RESET, tmp);
4993                 tmp = RREG32(GRBM_SOFT_RESET);
4994
4995                 udelay(50);
4996
4997                 tmp &= ~grbm_soft_reset;
4998                 WREG32(GRBM_SOFT_RESET, tmp);
4999                 tmp = RREG32(GRBM_SOFT_RESET);
5000         }
5001
5002         if (srbm_soft_reset) {
5003                 tmp = RREG32(SRBM_SOFT_RESET);
5004                 tmp |= srbm_soft_reset;
5005                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5006                 WREG32(SRBM_SOFT_RESET, tmp);
5007                 tmp = RREG32(SRBM_SOFT_RESET);
5008
5009                 udelay(50);
5010
5011                 tmp &= ~srbm_soft_reset;
5012                 WREG32(SRBM_SOFT_RESET, tmp);
5013                 tmp = RREG32(SRBM_SOFT_RESET);
5014         }
5015
5016         /* Wait a little for things to settle down */
5017         udelay(50);
5018
5019         evergreen_mc_resume(rdev, &save);
5020         udelay(50);
5021
5022         cik_print_gpu_status_regs(rdev);
5023 }
5024
5025 struct kv_reset_save_regs {
5026         u32 gmcon_reng_execute;
5027         u32 gmcon_misc;
5028         u32 gmcon_misc3;
5029 };
5030
5031 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5032                                    struct kv_reset_save_regs *save)
5033 {
5034         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5035         save->gmcon_misc = RREG32(GMCON_MISC);
5036         save->gmcon_misc3 = RREG32(GMCON_MISC3);
5037
5038         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5039         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5040                                                 STCTRL_STUTTER_EN));
5041 }
5042
5043 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5044                                       struct kv_reset_save_regs *save)
5045 {
5046         int i;
5047
5048         WREG32(GMCON_PGFSM_WRITE, 0);
5049         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5050
5051         for (i = 0; i < 5; i++)
5052                 WREG32(GMCON_PGFSM_WRITE, 0);
5053
5054         WREG32(GMCON_PGFSM_WRITE, 0);
5055         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5056
5057         for (i = 0; i < 5; i++)
5058                 WREG32(GMCON_PGFSM_WRITE, 0);
5059
5060         WREG32(GMCON_PGFSM_WRITE, 0x210000);
5061         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5062
5063         for (i = 0; i < 5; i++)
5064                 WREG32(GMCON_PGFSM_WRITE, 0);
5065
5066         WREG32(GMCON_PGFSM_WRITE, 0x21003);
5067         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5068
5069         for (i = 0; i < 5; i++)
5070                 WREG32(GMCON_PGFSM_WRITE, 0);
5071
5072         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5073         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5074
5075         for (i = 0; i < 5; i++)
5076                 WREG32(GMCON_PGFSM_WRITE, 0);
5077
5078         WREG32(GMCON_PGFSM_WRITE, 0);
5079         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5080
5081         for (i = 0; i < 5; i++)
5082                 WREG32(GMCON_PGFSM_WRITE, 0);
5083
5084         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5085         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5086
5087         for (i = 0; i < 5; i++)
5088                 WREG32(GMCON_PGFSM_WRITE, 0);
5089
5090         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5091         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5092
5093         for (i = 0; i < 5; i++)
5094                 WREG32(GMCON_PGFSM_WRITE, 0);
5095
5096         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5097         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5098
5099         for (i = 0; i < 5; i++)
5100                 WREG32(GMCON_PGFSM_WRITE, 0);
5101
5102         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5103         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5104
5105         for (i = 0; i < 5; i++)
5106                 WREG32(GMCON_PGFSM_WRITE, 0);
5107
5108         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5109         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5110
5111         WREG32(GMCON_MISC3, save->gmcon_misc3);
5112         WREG32(GMCON_MISC, save->gmcon_misc);
5113         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5114 }
5115
5116 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5117 {
5118         struct evergreen_mc_save save;
5119         struct kv_reset_save_regs kv_save = { 0 };
5120         u32 tmp, i;
5121
5122         dev_info(rdev->dev, "GPU pci config reset\n");
5123
5124         /* disable dpm? */
5125
5126         /* disable cg/pg */
5127         cik_fini_pg(rdev);
5128         cik_fini_cg(rdev);
5129
5130         /* Disable GFX parsing/prefetching */
5131         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5132
5133         /* Disable MEC parsing/prefetching */
5134         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5135
5136         /* sdma0 */
5137         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5138         tmp |= SDMA_HALT;
5139         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5140         /* sdma1 */
5141         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5142         tmp |= SDMA_HALT;
5143         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5144         /* XXX other engines? */
5145
5146         /* halt the rlc, disable cp internal ints */
5147         cik_rlc_stop(rdev);
5148
5149         udelay(50);
5150
5151         /* disable mem access */
5152         evergreen_mc_stop(rdev, &save);
5153         if (evergreen_mc_wait_for_idle(rdev)) {
5154                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5155         }
5156
5157         if (rdev->flags & RADEON_IS_IGP)
5158                 kv_save_regs_for_reset(rdev, &kv_save);
5159
5160         /* disable BM */
5161         pci_clear_master(rdev->pdev);
5162         /* reset */
5163         radeon_pci_config_reset(rdev);
5164
5165         udelay(100);
5166
5167         /* wait for asic to come out of reset */
5168         for (i = 0; i < rdev->usec_timeout; i++) {
5169                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5170                         break;
5171                 udelay(1);
5172         }
5173
5174         /* does asic init need to be run first??? */
5175         if (rdev->flags & RADEON_IS_IGP)
5176                 kv_restore_regs_for_reset(rdev, &kv_save);
5177 }
5178
5179 /**
5180  * cik_asic_reset - soft reset GPU
5181  *
5182  * @rdev: radeon_device pointer
5183  *
5184  * Look up which blocks are hung and attempt
5185  * to reset them.
5186  * Returns 0 for success.
5187  */
5188 int cik_asic_reset(struct radeon_device *rdev)
5189 {
5190         u32 reset_mask;
5191
5192         reset_mask = cik_gpu_check_soft_reset(rdev);
5193
5194         if (reset_mask)
5195                 r600_set_bios_scratch_engine_hung(rdev, true);
5196
5197         /* try soft reset */
5198         cik_gpu_soft_reset(rdev, reset_mask);
5199
5200         reset_mask = cik_gpu_check_soft_reset(rdev);
5201
5202         /* try pci config reset */
5203         if (reset_mask && radeon_hard_reset)
5204                 cik_gpu_pci_config_reset(rdev);
5205
5206         reset_mask = cik_gpu_check_soft_reset(rdev);
5207
5208         if (!reset_mask)
5209                 r600_set_bios_scratch_engine_hung(rdev, false);
5210
5211         return 0;
5212 }
5213
5214 /**
5215  * cik_gfx_is_lockup - check if the 3D engine is locked up
5216  *
5217  * @rdev: radeon_device pointer
5218  * @ring: radeon_ring structure holding ring information
5219  *
5220  * Check if the 3D engine is locked up (CIK).
5221  * Returns true if the engine is locked, false if not.
5222  */
5223 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5224 {
5225         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5226
5227         if (!(reset_mask & (RADEON_RESET_GFX |
5228                             RADEON_RESET_COMPUTE |
5229                             RADEON_RESET_CP))) {
5230                 radeon_ring_lockup_update(rdev, ring);
5231                 return false;
5232         }
5233         return radeon_ring_test_lockup(rdev, ring);
5234 }
5235
5236 /* MC */
5237 /**
5238  * cik_mc_program - program the GPU memory controller
5239  *
5240  * @rdev: radeon_device pointer
5241  *
5242  * Set the location of vram, gart, and AGP in the GPU's
5243  * physical address space (CIK).
5244  */
5245 static void cik_mc_program(struct radeon_device *rdev)
5246 {
5247         struct evergreen_mc_save save;
5248         u32 tmp;
5249         int i, j;
5250
5251         /* Initialize HDP */
5252         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5253                 WREG32((0x2c14 + j), 0x00000000);
5254                 WREG32((0x2c18 + j), 0x00000000);
5255                 WREG32((0x2c1c + j), 0x00000000);
5256                 WREG32((0x2c20 + j), 0x00000000);
5257                 WREG32((0x2c24 + j), 0x00000000);
5258         }
5259         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5260
5261         evergreen_mc_stop(rdev, &save);
5262         if (radeon_mc_wait_for_idle(rdev)) {
5263                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5264         }
5265         /* Lockout access through VGA aperture*/
5266         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5267         /* Update configuration */
5268         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5269                rdev->mc.vram_start >> 12);
5270         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5271                rdev->mc.vram_end >> 12);
5272         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5273                rdev->vram_scratch.gpu_addr >> 12);
5274         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5275         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5276         WREG32(MC_VM_FB_LOCATION, tmp);
5277         /* XXX double check these! */
5278         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5279         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5280         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5281         WREG32(MC_VM_AGP_BASE, 0);
5282         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5283         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5284         if (radeon_mc_wait_for_idle(rdev)) {
5285                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5286         }
5287         evergreen_mc_resume(rdev, &save);
5288         /* we need to own VRAM, so turn off the VGA renderer here
5289          * to stop it overwriting our objects */
5290         rv515_vga_render_disable(rdev);
5291 }
5292
5293 /**
5294  * cik_mc_init - initialize the memory controller driver params
5295  *
5296  * @rdev: radeon_device pointer
5297  *
5298  * Look up the amount of vram, vram width, and decide how to place
5299  * vram and gart within the GPU's physical address space (CIK).
5300  * Returns 0 for success.
5301  */
5302 static int cik_mc_init(struct radeon_device *rdev)
5303 {
5304         u32 tmp;
5305         int chansize, numchan;
5306
5307         /* Get VRAM informations */
5308         rdev->mc.vram_is_ddr = true;
5309         tmp = RREG32(MC_ARB_RAMCFG);
5310         if (tmp & CHANSIZE_MASK) {
5311                 chansize = 64;
5312         } else {
5313                 chansize = 32;
5314         }
5315         tmp = RREG32(MC_SHARED_CHMAP);
5316         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5317         case 0:
5318         default:
5319                 numchan = 1;
5320                 break;
5321         case 1:
5322                 numchan = 2;
5323                 break;
5324         case 2:
5325                 numchan = 4;
5326                 break;
5327         case 3:
5328                 numchan = 8;
5329                 break;
5330         case 4:
5331                 numchan = 3;
5332                 break;
5333         case 5:
5334                 numchan = 6;
5335                 break;
5336         case 6:
5337                 numchan = 10;
5338                 break;
5339         case 7:
5340                 numchan = 12;
5341                 break;
5342         case 8:
5343                 numchan = 16;
5344                 break;
5345         }
5346         rdev->mc.vram_width = numchan * chansize;
5347         /* Could aper size report 0 ? */
5348         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5349         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5350         /* size in MB on si */
5351         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5352         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5353         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5354         si_vram_gtt_location(rdev, &rdev->mc);
5355         radeon_update_bandwidth_info(rdev);
5356
5357         return 0;
5358 }
5359
5360 /*
5361  * GART
5362  * VMID 0 is the physical GPU addresses as used by the kernel.
5363  * VMIDs 1-15 are used for userspace clients and are handled
5364  * by the radeon vm/hsa code.
5365  */
5366 /**
5367  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5368  *
5369  * @rdev: radeon_device pointer
5370  *
5371  * Flush the TLB for the VMID 0 page table (CIK).
5372  */
5373 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5374 {
5375         /* flush hdp cache */
5376         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5377
5378         /* bits 0-15 are the VM contexts0-15 */
5379         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5380 }
5381
5382 /**
5383  * cik_pcie_gart_enable - gart enable
5384  *
5385  * @rdev: radeon_device pointer
5386  *
5387  * This sets up the TLBs, programs the page tables for VMID0,
5388  * sets up the hw for VMIDs 1-15 which are allocated on
5389  * demand, and sets up the global locations for the LDS, GDS,
5390  * and GPUVM for FSA64 clients (CIK).
5391  * Returns 0 for success, errors for failure.
5392  */
5393 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5394 {
5395         int r, i;
5396
5397         if (rdev->gart.robj == NULL) {
5398                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5399                 return -EINVAL;
5400         }
5401         r = radeon_gart_table_vram_pin(rdev);
5402         if (r)
5403                 return r;
5404         radeon_gart_restore(rdev);
5405         /* Setup TLB control */
5406         WREG32(MC_VM_MX_L1_TLB_CNTL,
5407                (0xA << 7) |
5408                ENABLE_L1_TLB |
5409                ENABLE_L1_FRAGMENT_PROCESSING |
5410                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5411                ENABLE_ADVANCED_DRIVER_MODEL |
5412                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5413         /* Setup L2 cache */
5414         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5415                ENABLE_L2_FRAGMENT_PROCESSING |
5416                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5417                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5418                EFFECTIVE_L2_QUEUE_SIZE(7) |
5419                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5420         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5421         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5422                BANK_SELECT(4) |
5423                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5424         /* setup context0 */
5425         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5426         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5427         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5428         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5429                         (u32)(rdev->dummy_page.addr >> 12));
5430         WREG32(VM_CONTEXT0_CNTL2, 0);
5431         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5432                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5433
5434         WREG32(0x15D4, 0);
5435         WREG32(0x15D8, 0);
5436         WREG32(0x15DC, 0);
5437
5438         /* empty context1-15 */
5439         /* FIXME start with 4G, once using 2 level pt switch to full
5440          * vm size space
5441          */
5442         /* set vm size, must be a multiple of 4 */
5443         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5444         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5445         for (i = 1; i < 16; i++) {
5446                 if (i < 8)
5447                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5448                                rdev->gart.table_addr >> 12);
5449                 else
5450                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5451                                rdev->gart.table_addr >> 12);
5452         }
5453
5454         /* enable context1-15 */
5455         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5456                (u32)(rdev->dummy_page.addr >> 12));
5457         WREG32(VM_CONTEXT1_CNTL2, 4);
5458         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5459                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5460                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5461                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5462                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5463                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5464                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5465                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5466                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5467                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5468                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5469                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5470                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5471                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5472
5473         if (rdev->family == CHIP_KAVERI) {
5474                 u32 tmp = RREG32(CHUB_CONTROL);
5475                 tmp &= ~BYPASS_VM;
5476                 WREG32(CHUB_CONTROL, tmp);
5477         }
5478
5479         /* XXX SH_MEM regs */
5480         /* where to put LDS, scratch, GPUVM in FSA64 space */
5481         mutex_lock(&rdev->srbm_mutex);
5482         for (i = 0; i < 16; i++) {
5483                 cik_srbm_select(rdev, 0, 0, 0, i);
5484                 /* CP and shaders */
5485                 WREG32(SH_MEM_CONFIG, 0);
5486                 WREG32(SH_MEM_APE1_BASE, 1);
5487                 WREG32(SH_MEM_APE1_LIMIT, 0);
5488                 WREG32(SH_MEM_BASES, 0);
5489                 /* SDMA GFX */
5490                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5491                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5492                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5493                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5494                 /* XXX SDMA RLC - todo */
5495         }
5496         cik_srbm_select(rdev, 0, 0, 0, 0);
5497         mutex_unlock(&rdev->srbm_mutex);
5498
5499         cik_pcie_gart_tlb_flush(rdev);
5500         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5501                  (unsigned)(rdev->mc.gtt_size >> 20),
5502                  (unsigned long long)rdev->gart.table_addr);
5503         rdev->gart.ready = true;
5504         return 0;
5505 }
5506
5507 /**
5508  * cik_pcie_gart_disable - gart disable
5509  *
5510  * @rdev: radeon_device pointer
5511  *
5512  * This disables all VM page table (CIK).
5513  */
5514 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5515 {
5516         /* Disable all tables */
5517         WREG32(VM_CONTEXT0_CNTL, 0);
5518         WREG32(VM_CONTEXT1_CNTL, 0);
5519         /* Setup TLB control */
5520         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5521                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5522         /* Setup L2 cache */
5523         WREG32(VM_L2_CNTL,
5524                ENABLE_L2_FRAGMENT_PROCESSING |
5525                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5526                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5527                EFFECTIVE_L2_QUEUE_SIZE(7) |
5528                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5529         WREG32(VM_L2_CNTL2, 0);
5530         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5531                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5532         radeon_gart_table_vram_unpin(rdev);
5533 }
5534
5535 /**
5536  * cik_pcie_gart_fini - vm fini callback
5537  *
5538  * @rdev: radeon_device pointer
5539  *
5540  * Tears down the driver GART/VM setup (CIK).
5541  */
5542 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5543 {
5544         cik_pcie_gart_disable(rdev);
5545         radeon_gart_table_vram_free(rdev);
5546         radeon_gart_fini(rdev);
5547 }
5548
5549 /* vm parser */
5550 /**
5551  * cik_ib_parse - vm ib_parse callback
5552  *
5553  * @rdev: radeon_device pointer
5554  * @ib: indirect buffer pointer
5555  *
5556  * CIK uses hw IB checking so this is a nop (CIK).
5557  */
5558 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5559 {
5560         return 0;
5561 }
5562
5563 /*
5564  * vm
5565  * VMID 0 is the physical GPU addresses as used by the kernel.
5566  * VMIDs 1-15 are used for userspace clients and are handled
5567  * by the radeon vm/hsa code.
5568  */
5569 /**
5570  * cik_vm_init - cik vm init callback
5571  *
5572  * @rdev: radeon_device pointer
5573  *
5574  * Inits cik specific vm parameters (number of VMs, base of vram for
5575  * VMIDs 1-15) (CIK).
5576  * Returns 0 for success.
5577  */
5578 int cik_vm_init(struct radeon_device *rdev)
5579 {
5580         /* number of VMs */
5581         rdev->vm_manager.nvm = 16;
5582         /* base offset of vram pages */
5583         if (rdev->flags & RADEON_IS_IGP) {
5584                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5585                 tmp <<= 22;
5586                 rdev->vm_manager.vram_base_offset = tmp;
5587         } else
5588                 rdev->vm_manager.vram_base_offset = 0;
5589
5590         return 0;
5591 }
5592
5593 /**
5594  * cik_vm_fini - cik vm fini callback
5595  *
5596  * @rdev: radeon_device pointer
5597  *
5598  * Tear down any asic specific VM setup (CIK).
5599  */
5600 void cik_vm_fini(struct radeon_device *rdev)
5601 {
5602 }
5603
5604 /**
5605  * cik_vm_decode_fault - print human readable fault info
5606  *
5607  * @rdev: radeon_device pointer
5608  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5609  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5610  *
5611  * Print human readable fault information (CIK).
5612  */
5613 static void cik_vm_decode_fault(struct radeon_device *rdev,
5614                                 u32 status, u32 addr, u32 mc_client)
5615 {
5616         u32 mc_id;
5617         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5618         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5619         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5620                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5621
5622         if (rdev->family == CHIP_HAWAII)
5623                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5624         else
5625                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5626
5627         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5628                protections, vmid, addr,
5629                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5630                block, mc_client, mc_id);
5631 }
5632
5633 /**
5634  * cik_vm_flush - cik vm flush using the CP
5635  *
5636  * @rdev: radeon_device pointer
5637  *
5638  * Update the page table base and flush the VM TLB
5639  * using the CP (CIK).
5640  */
5641 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5642 {
5643         struct radeon_ring *ring = &rdev->ring[ridx];
5644
5645         if (vm == NULL)
5646                 return;
5647
5648         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5649         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5650                                  WRITE_DATA_DST_SEL(0)));
5651         if (vm->id < 8) {
5652                 radeon_ring_write(ring,
5653                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5654         } else {
5655                 radeon_ring_write(ring,
5656                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5657         }
5658         radeon_ring_write(ring, 0);
5659         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5660
5661         /* update SH_MEM_* regs */
5662         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5663         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5664                                  WRITE_DATA_DST_SEL(0)));
5665         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5666         radeon_ring_write(ring, 0);
5667         radeon_ring_write(ring, VMID(vm->id));
5668
5669         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5670         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5671                                  WRITE_DATA_DST_SEL(0)));
5672         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5673         radeon_ring_write(ring, 0);
5674
5675         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5676         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5677         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5678         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5679
5680         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5681         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5682                                  WRITE_DATA_DST_SEL(0)));
5683         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5684         radeon_ring_write(ring, 0);
5685         radeon_ring_write(ring, VMID(0));
5686
5687         /* HDP flush */
5688         cik_hdp_flush_cp_ring_emit(rdev, ridx);
5689
5690         /* bits 0-15 are the VM contexts0-15 */
5691         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5692         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5693                                  WRITE_DATA_DST_SEL(0)));
5694         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5695         radeon_ring_write(ring, 0);
5696         radeon_ring_write(ring, 1 << vm->id);
5697
5698         /* compute doesn't have PFP */
5699         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5700                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5701                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5702                 radeon_ring_write(ring, 0x0);
5703         }
5704 }
5705
5706 /*
5707  * RLC
5708  * The RLC is a multi-purpose microengine that handles a
5709  * variety of functions, the most important of which is
5710  * the interrupt controller.
5711  */
5712 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5713                                           bool enable)
5714 {
5715         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5716
5717         if (enable)
5718                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5719         else
5720                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5721         WREG32(CP_INT_CNTL_RING0, tmp);
5722 }
5723
5724 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5725 {
5726         u32 tmp;
5727
5728         tmp = RREG32(RLC_LB_CNTL);
5729         if (enable)
5730                 tmp |= LOAD_BALANCE_ENABLE;
5731         else
5732                 tmp &= ~LOAD_BALANCE_ENABLE;
5733         WREG32(RLC_LB_CNTL, tmp);
5734 }
5735
5736 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5737 {
5738         u32 i, j, k;
5739         u32 mask;
5740
5741         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5742                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5743                         cik_select_se_sh(rdev, i, j);
5744                         for (k = 0; k < rdev->usec_timeout; k++) {
5745                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5746                                         break;
5747                                 udelay(1);
5748                         }
5749                 }
5750         }
5751         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5752
5753         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5754         for (k = 0; k < rdev->usec_timeout; k++) {
5755                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5756                         break;
5757                 udelay(1);
5758         }
5759 }
5760
5761 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5762 {
5763         u32 tmp;
5764
5765         tmp = RREG32(RLC_CNTL);
5766         if (tmp != rlc)
5767                 WREG32(RLC_CNTL, rlc);
5768 }
5769
5770 static u32 cik_halt_rlc(struct radeon_device *rdev)
5771 {
5772         u32 data, orig;
5773
5774         orig = data = RREG32(RLC_CNTL);
5775
5776         if (data & RLC_ENABLE) {
5777                 u32 i;
5778
5779                 data &= ~RLC_ENABLE;
5780                 WREG32(RLC_CNTL, data);
5781
5782                 for (i = 0; i < rdev->usec_timeout; i++) {
5783                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5784                                 break;
5785                         udelay(1);
5786                 }
5787
5788                 cik_wait_for_rlc_serdes(rdev);
5789         }
5790
5791         return orig;
5792 }
5793
5794 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5795 {
5796         u32 tmp, i, mask;
5797
5798         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5799         WREG32(RLC_GPR_REG2, tmp);
5800
5801         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5802         for (i = 0; i < rdev->usec_timeout; i++) {
5803                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5804                         break;
5805                 udelay(1);
5806         }
5807
5808         for (i = 0; i < rdev->usec_timeout; i++) {
5809                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5810                         break;
5811                 udelay(1);
5812         }
5813 }
5814
5815 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5816 {
5817         u32 tmp;
5818
5819         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5820         WREG32(RLC_GPR_REG2, tmp);
5821 }
5822
5823 /**
5824  * cik_rlc_stop - stop the RLC ME
5825  *
5826  * @rdev: radeon_device pointer
5827  *
5828  * Halt the RLC ME (MicroEngine) (CIK).
5829  */
5830 static void cik_rlc_stop(struct radeon_device *rdev)
5831 {
5832         WREG32(RLC_CNTL, 0);
5833
5834         cik_enable_gui_idle_interrupt(rdev, false);
5835
5836         cik_wait_for_rlc_serdes(rdev);
5837 }
5838
5839 /**
5840  * cik_rlc_start - start the RLC ME
5841  *
5842  * @rdev: radeon_device pointer
5843  *
5844  * Unhalt the RLC ME (MicroEngine) (CIK).
5845  */
5846 static void cik_rlc_start(struct radeon_device *rdev)
5847 {
5848         WREG32(RLC_CNTL, RLC_ENABLE);
5849
5850         cik_enable_gui_idle_interrupt(rdev, true);
5851
5852         udelay(50);
5853 }
5854
5855 /**
5856  * cik_rlc_resume - setup the RLC hw
5857  *
5858  * @rdev: radeon_device pointer
5859  *
5860  * Initialize the RLC registers, load the ucode,
5861  * and start the RLC (CIK).
5862  * Returns 0 for success, -EINVAL if the ucode is not available.
5863  */
5864 static int cik_rlc_resume(struct radeon_device *rdev)
5865 {
5866         u32 i, size, tmp;
5867         const __be32 *fw_data;
5868
5869         if (!rdev->rlc_fw)
5870                 return -EINVAL;
5871
5872         switch (rdev->family) {
5873         case CHIP_BONAIRE:
5874         case CHIP_HAWAII:
5875         default:
5876                 size = BONAIRE_RLC_UCODE_SIZE;
5877                 break;
5878         case CHIP_KAVERI:
5879                 size = KV_RLC_UCODE_SIZE;
5880                 break;
5881         case CHIP_KABINI:
5882                 size = KB_RLC_UCODE_SIZE;
5883                 break;
5884         case CHIP_MULLINS:
5885                 size = ML_RLC_UCODE_SIZE;
5886                 break;
5887         }
5888
5889         cik_rlc_stop(rdev);
5890
5891         /* disable CG */
5892         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5893         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5894
5895         si_rlc_reset(rdev);
5896
5897         cik_init_pg(rdev);
5898
5899         cik_init_cg(rdev);
5900
5901         WREG32(RLC_LB_CNTR_INIT, 0);
5902         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5903
5904         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5905         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5906         WREG32(RLC_LB_PARAMS, 0x00600408);
5907         WREG32(RLC_LB_CNTL, 0x80000004);
5908
5909         WREG32(RLC_MC_CNTL, 0);
5910         WREG32(RLC_UCODE_CNTL, 0);
5911
5912         fw_data = (const __be32 *)rdev->rlc_fw->data;
5913                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5914         for (i = 0; i < size; i++)
5915                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5916         WREG32(RLC_GPM_UCODE_ADDR, 0);
5917
5918         /* XXX - find out what chips support lbpw */
5919         cik_enable_lbpw(rdev, false);
5920
5921         if (rdev->family == CHIP_BONAIRE)
5922                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5923
5924         cik_rlc_start(rdev);
5925
5926         return 0;
5927 }
5928
5929 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5930 {
5931         u32 data, orig, tmp, tmp2;
5932
5933         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5934
5935         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5936                 cik_enable_gui_idle_interrupt(rdev, true);
5937
5938                 tmp = cik_halt_rlc(rdev);
5939
5940                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5941                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5942                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5943                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5944                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5945
5946                 cik_update_rlc(rdev, tmp);
5947
5948                 data |= CGCG_EN | CGLS_EN;
5949         } else {
5950                 cik_enable_gui_idle_interrupt(rdev, false);
5951
5952                 RREG32(CB_CGTT_SCLK_CTRL);
5953                 RREG32(CB_CGTT_SCLK_CTRL);
5954                 RREG32(CB_CGTT_SCLK_CTRL);
5955                 RREG32(CB_CGTT_SCLK_CTRL);
5956
5957                 data &= ~(CGCG_EN | CGLS_EN);
5958         }
5959
5960         if (orig != data)
5961                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5962
5963 }
5964
5965 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5966 {
5967         u32 data, orig, tmp = 0;
5968
5969         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5970                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5971                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5972                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5973                                 data |= CP_MEM_LS_EN;
5974                                 if (orig != data)
5975                                         WREG32(CP_MEM_SLP_CNTL, data);
5976                         }
5977                 }
5978
5979                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5980                 data &= 0xfffffffd;
5981                 if (orig != data)
5982                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5983
5984                 tmp = cik_halt_rlc(rdev);
5985
5986                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5987                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5988                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5989                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5990                 WREG32(RLC_SERDES_WR_CTRL, data);
5991
5992                 cik_update_rlc(rdev, tmp);
5993
5994                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5995                         orig = data = RREG32(CGTS_SM_CTRL_REG);
5996                         data &= ~SM_MODE_MASK;
5997                         data |= SM_MODE(0x2);
5998                         data |= SM_MODE_ENABLE;
5999                         data &= ~CGTS_OVERRIDE;
6000                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6001                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6002                                 data &= ~CGTS_LS_OVERRIDE;
6003                         data &= ~ON_MONITOR_ADD_MASK;
6004                         data |= ON_MONITOR_ADD_EN;
6005                         data |= ON_MONITOR_ADD(0x96);
6006                         if (orig != data)
6007                                 WREG32(CGTS_SM_CTRL_REG, data);
6008                 }
6009         } else {
6010                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6011                 data |= 0x00000002;
6012                 if (orig != data)
6013                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6014
6015                 data = RREG32(RLC_MEM_SLP_CNTL);
6016                 if (data & RLC_MEM_LS_EN) {
6017                         data &= ~RLC_MEM_LS_EN;
6018                         WREG32(RLC_MEM_SLP_CNTL, data);
6019                 }
6020
6021                 data = RREG32(CP_MEM_SLP_CNTL);
6022                 if (data & CP_MEM_LS_EN) {
6023                         data &= ~CP_MEM_LS_EN;
6024                         WREG32(CP_MEM_SLP_CNTL, data);
6025                 }
6026
6027                 orig = data = RREG32(CGTS_SM_CTRL_REG);
6028                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6029                 if (orig != data)
6030                         WREG32(CGTS_SM_CTRL_REG, data);
6031
6032                 tmp = cik_halt_rlc(rdev);
6033
6034                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6035                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6036                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6037                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6038                 WREG32(RLC_SERDES_WR_CTRL, data);
6039
6040                 cik_update_rlc(rdev, tmp);
6041         }
6042 }
6043
6044 static const u32 mc_cg_registers[] =
6045 {
6046         MC_HUB_MISC_HUB_CG,
6047         MC_HUB_MISC_SIP_CG,
6048         MC_HUB_MISC_VM_CG,
6049         MC_XPB_CLK_GAT,
6050         ATC_MISC_CG,
6051         MC_CITF_MISC_WR_CG,
6052         MC_CITF_MISC_RD_CG,
6053         MC_CITF_MISC_VM_CG,
6054         VM_L2_CG,
6055 };
6056
6057 static void cik_enable_mc_ls(struct radeon_device *rdev,
6058                              bool enable)
6059 {
6060         int i;
6061         u32 orig, data;
6062
6063         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6064                 orig = data = RREG32(mc_cg_registers[i]);
6065                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6066                         data |= MC_LS_ENABLE;
6067                 else
6068                         data &= ~MC_LS_ENABLE;
6069                 if (data != orig)
6070                         WREG32(mc_cg_registers[i], data);
6071         }
6072 }
6073
6074 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6075                                bool enable)
6076 {
6077         int i;
6078         u32 orig, data;
6079
6080         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6081                 orig = data = RREG32(mc_cg_registers[i]);
6082                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6083                         data |= MC_CG_ENABLE;
6084                 else
6085                         data &= ~MC_CG_ENABLE;
6086                 if (data != orig)
6087                         WREG32(mc_cg_registers[i], data);
6088         }
6089 }
6090
6091 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6092                                  bool enable)
6093 {
6094         u32 orig, data;
6095
6096         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6097                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6098                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6099         } else {
6100                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6101                 data |= 0xff000000;
6102                 if (data != orig)
6103                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6104
6105                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6106                 data |= 0xff000000;
6107                 if (data != orig)
6108                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6109         }
6110 }
6111
6112 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6113                                  bool enable)
6114 {
6115         u32 orig, data;
6116
6117         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6118                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6119                 data |= 0x100;
6120                 if (orig != data)
6121                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6122
6123                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6124                 data |= 0x100;
6125                 if (orig != data)
6126                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6127         } else {
6128                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6129                 data &= ~0x100;
6130                 if (orig != data)
6131                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6132
6133                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6134                 data &= ~0x100;
6135                 if (orig != data)
6136                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6137         }
6138 }
6139
6140 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6141                                 bool enable)
6142 {
6143         u32 orig, data;
6144
6145         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6146                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6147                 data = 0xfff;
6148                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6149
6150                 orig = data = RREG32(UVD_CGC_CTRL);
6151                 data |= DCM;
6152                 if (orig != data)
6153                         WREG32(UVD_CGC_CTRL, data);
6154         } else {
6155                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6156                 data &= ~0xfff;
6157                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6158
6159                 orig = data = RREG32(UVD_CGC_CTRL);
6160                 data &= ~DCM;
6161                 if (orig != data)
6162                         WREG32(UVD_CGC_CTRL, data);
6163         }
6164 }
6165
6166 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6167                                bool enable)
6168 {
6169         u32 orig, data;
6170
6171         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6172
6173         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6174                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6175                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6176         else
6177                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6178                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6179
6180         if (orig != data)
6181                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6182 }
6183
6184 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6185                                 bool enable)
6186 {
6187         u32 orig, data;
6188
6189         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6190
6191         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6192                 data &= ~CLOCK_GATING_DIS;
6193         else
6194                 data |= CLOCK_GATING_DIS;
6195
6196         if (orig != data)
6197                 WREG32(HDP_HOST_PATH_CNTL, data);
6198 }
6199
6200 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6201                               bool enable)
6202 {
6203         u32 orig, data;
6204
6205         orig = data = RREG32(HDP_MEM_POWER_LS);
6206
6207         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6208                 data |= HDP_LS_ENABLE;
6209         else
6210                 data &= ~HDP_LS_ENABLE;
6211
6212         if (orig != data)
6213                 WREG32(HDP_MEM_POWER_LS, data);
6214 }
6215
6216 void cik_update_cg(struct radeon_device *rdev,
6217                    u32 block, bool enable)
6218 {
6219
6220         if (block & RADEON_CG_BLOCK_GFX) {
6221                 cik_enable_gui_idle_interrupt(rdev, false);
6222                 /* order matters! */
6223                 if (enable) {
6224                         cik_enable_mgcg(rdev, true);
6225                         cik_enable_cgcg(rdev, true);
6226                 } else {
6227                         cik_enable_cgcg(rdev, false);
6228                         cik_enable_mgcg(rdev, false);
6229                 }
6230                 cik_enable_gui_idle_interrupt(rdev, true);
6231         }
6232
6233         if (block & RADEON_CG_BLOCK_MC) {
6234                 if (!(rdev->flags & RADEON_IS_IGP)) {
6235                         cik_enable_mc_mgcg(rdev, enable);
6236                         cik_enable_mc_ls(rdev, enable);
6237                 }
6238         }
6239
6240         if (block & RADEON_CG_BLOCK_SDMA) {
6241                 cik_enable_sdma_mgcg(rdev, enable);
6242                 cik_enable_sdma_mgls(rdev, enable);
6243         }
6244
6245         if (block & RADEON_CG_BLOCK_BIF) {
6246                 cik_enable_bif_mgls(rdev, enable);
6247         }
6248
6249         if (block & RADEON_CG_BLOCK_UVD) {
6250                 if (rdev->has_uvd)
6251                         cik_enable_uvd_mgcg(rdev, enable);
6252         }
6253
6254         if (block & RADEON_CG_BLOCK_HDP) {
6255                 cik_enable_hdp_mgcg(rdev, enable);
6256                 cik_enable_hdp_ls(rdev, enable);
6257         }
6258
6259         if (block & RADEON_CG_BLOCK_VCE) {
6260                 vce_v2_0_enable_mgcg(rdev, enable);
6261         }
6262 }
6263
6264 static void cik_init_cg(struct radeon_device *rdev)
6265 {
6266
6267         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6268
6269         if (rdev->has_uvd)
6270                 si_init_uvd_internal_cg(rdev);
6271
6272         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6273                              RADEON_CG_BLOCK_SDMA |
6274                              RADEON_CG_BLOCK_BIF |
6275                              RADEON_CG_BLOCK_UVD |
6276                              RADEON_CG_BLOCK_HDP), true);
6277 }
6278
6279 static void cik_fini_cg(struct radeon_device *rdev)
6280 {
6281         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6282                              RADEON_CG_BLOCK_SDMA |
6283                              RADEON_CG_BLOCK_BIF |
6284                              RADEON_CG_BLOCK_UVD |
6285                              RADEON_CG_BLOCK_HDP), false);
6286
6287         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6288 }
6289
6290 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6291                                           bool enable)
6292 {
6293         u32 data, orig;
6294
6295         orig = data = RREG32(RLC_PG_CNTL);
6296         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6297                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6298         else
6299                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6300         if (orig != data)
6301                 WREG32(RLC_PG_CNTL, data);
6302 }
6303
6304 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6305                                           bool enable)
6306 {
6307         u32 data, orig;
6308
6309         orig = data = RREG32(RLC_PG_CNTL);
6310         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6311                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6312         else
6313                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6314         if (orig != data)
6315                 WREG32(RLC_PG_CNTL, data);
6316 }
6317
6318 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6319 {
6320         u32 data, orig;
6321
6322         orig = data = RREG32(RLC_PG_CNTL);
6323         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6324                 data &= ~DISABLE_CP_PG;
6325         else
6326                 data |= DISABLE_CP_PG;
6327         if (orig != data)
6328                 WREG32(RLC_PG_CNTL, data);
6329 }
6330
6331 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6332 {
6333         u32 data, orig;
6334
6335         orig = data = RREG32(RLC_PG_CNTL);
6336         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6337                 data &= ~DISABLE_GDS_PG;
6338         else
6339                 data |= DISABLE_GDS_PG;
6340         if (orig != data)
6341                 WREG32(RLC_PG_CNTL, data);
6342 }
6343
6344 #define CP_ME_TABLE_SIZE    96
6345 #define CP_ME_TABLE_OFFSET  2048
6346 #define CP_MEC_TABLE_OFFSET 4096
6347
6348 void cik_init_cp_pg_table(struct radeon_device *rdev)
6349 {
6350         const __be32 *fw_data;
6351         volatile u32 *dst_ptr;
6352         int me, i, max_me = 4;
6353         u32 bo_offset = 0;
6354         u32 table_offset;
6355
6356         if (rdev->family == CHIP_KAVERI)
6357                 max_me = 5;
6358
6359         if (rdev->rlc.cp_table_ptr == NULL)
6360                 return;
6361
6362         /* write the cp table buffer */
6363         dst_ptr = rdev->rlc.cp_table_ptr;
6364         for (me = 0; me < max_me; me++) {
6365                 if (me == 0) {
6366                         fw_data = (const __be32 *)rdev->ce_fw->data;
6367                         table_offset = CP_ME_TABLE_OFFSET;
6368                 } else if (me == 1) {
6369                         fw_data = (const __be32 *)rdev->pfp_fw->data;
6370                         table_offset = CP_ME_TABLE_OFFSET;
6371                 } else if (me == 2) {
6372                         fw_data = (const __be32 *)rdev->me_fw->data;
6373                         table_offset = CP_ME_TABLE_OFFSET;
6374                 } else {
6375                         fw_data = (const __be32 *)rdev->mec_fw->data;
6376                         table_offset = CP_MEC_TABLE_OFFSET;
6377                 }
6378
6379                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6380                         dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6381                 }
6382                 bo_offset += CP_ME_TABLE_SIZE;
6383         }
6384 }
6385
6386 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6387                                 bool enable)
6388 {
6389         u32 data, orig;
6390
6391         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6392                 orig = data = RREG32(RLC_PG_CNTL);
6393                 data |= GFX_PG_ENABLE;
6394                 if (orig != data)
6395                         WREG32(RLC_PG_CNTL, data);
6396
6397                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6398                 data |= AUTO_PG_EN;
6399                 if (orig != data)
6400                         WREG32(RLC_AUTO_PG_CTRL, data);
6401         } else {
6402                 orig = data = RREG32(RLC_PG_CNTL);
6403                 data &= ~GFX_PG_ENABLE;
6404                 if (orig != data)
6405                         WREG32(RLC_PG_CNTL, data);
6406
6407                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6408                 data &= ~AUTO_PG_EN;
6409                 if (orig != data)
6410                         WREG32(RLC_AUTO_PG_CTRL, data);
6411
6412                 data = RREG32(DB_RENDER_CONTROL);
6413         }
6414 }
6415
6416 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6417 {
6418         u32 mask = 0, tmp, tmp1;
6419         int i;
6420
6421         cik_select_se_sh(rdev, se, sh);
6422         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6423         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6424         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6425
6426         tmp &= 0xffff0000;
6427
6428         tmp |= tmp1;
6429         tmp >>= 16;
6430
6431         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6432                 mask <<= 1;
6433                 mask |= 1;
6434         }
6435
6436         return (~tmp) & mask;
6437 }
6438
6439 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6440 {
6441         u32 i, j, k, active_cu_number = 0;
6442         u32 mask, counter, cu_bitmap;
6443         u32 tmp = 0;
6444
6445         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6446                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6447                         mask = 1;
6448                         cu_bitmap = 0;
6449                         counter = 0;
6450                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6451                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6452                                         if (counter < 2)
6453                                                 cu_bitmap |= mask;
6454                                         counter ++;
6455                                 }
6456                                 mask <<= 1;
6457                         }
6458
6459                         active_cu_number += counter;
6460                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6461                 }
6462         }
6463
6464         WREG32(RLC_PG_AO_CU_MASK, tmp);
6465
6466         tmp = RREG32(RLC_MAX_PG_CU);
6467         tmp &= ~MAX_PU_CU_MASK;
6468         tmp |= MAX_PU_CU(active_cu_number);
6469         WREG32(RLC_MAX_PG_CU, tmp);
6470 }
6471
6472 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6473                                        bool enable)
6474 {
6475         u32 data, orig;
6476
6477         orig = data = RREG32(RLC_PG_CNTL);
6478         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6479                 data |= STATIC_PER_CU_PG_ENABLE;
6480         else
6481                 data &= ~STATIC_PER_CU_PG_ENABLE;
6482         if (orig != data)
6483                 WREG32(RLC_PG_CNTL, data);
6484 }
6485
6486 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6487                                         bool enable)
6488 {
6489         u32 data, orig;
6490
6491         orig = data = RREG32(RLC_PG_CNTL);
6492         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6493                 data |= DYN_PER_CU_PG_ENABLE;
6494         else
6495                 data &= ~DYN_PER_CU_PG_ENABLE;
6496         if (orig != data)
6497                 WREG32(RLC_PG_CNTL, data);
6498 }
6499
6500 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6501 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6502
6503 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6504 {
6505         u32 data, orig;
6506         u32 i;
6507
6508         if (rdev->rlc.cs_data) {
6509                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6510                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6511                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6512                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6513         } else {
6514                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6515                 for (i = 0; i < 3; i++)
6516                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6517         }
6518         if (rdev->rlc.reg_list) {
6519                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6520                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6521                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6522         }
6523
6524         orig = data = RREG32(RLC_PG_CNTL);
6525         data |= GFX_PG_SRC;
6526         if (orig != data)
6527                 WREG32(RLC_PG_CNTL, data);
6528
6529         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6530         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6531
6532         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6533         data &= ~IDLE_POLL_COUNT_MASK;
6534         data |= IDLE_POLL_COUNT(0x60);
6535         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6536
6537         data = 0x10101010;
6538         WREG32(RLC_PG_DELAY, data);
6539
6540         data = RREG32(RLC_PG_DELAY_2);
6541         data &= ~0xff;
6542         data |= 0x3;
6543         WREG32(RLC_PG_DELAY_2, data);
6544
6545         data = RREG32(RLC_AUTO_PG_CTRL);
6546         data &= ~GRBM_REG_SGIT_MASK;
6547         data |= GRBM_REG_SGIT(0x700);
6548         WREG32(RLC_AUTO_PG_CTRL, data);
6549
6550 }
6551
6552 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6553 {
6554         cik_enable_gfx_cgpg(rdev, enable);
6555         cik_enable_gfx_static_mgpg(rdev, enable);
6556         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6557 }
6558
6559 u32 cik_get_csb_size(struct radeon_device *rdev)
6560 {
6561         u32 count = 0;
6562         const struct cs_section_def *sect = NULL;
6563         const struct cs_extent_def *ext = NULL;
6564
6565         if (rdev->rlc.cs_data == NULL)
6566                 return 0;
6567
6568         /* begin clear state */
6569         count += 2;
6570         /* context control state */
6571         count += 3;
6572
6573         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6574                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6575                         if (sect->id == SECT_CONTEXT)
6576                                 count += 2 + ext->reg_count;
6577                         else
6578                                 return 0;
6579                 }
6580         }
6581         /* pa_sc_raster_config/pa_sc_raster_config1 */
6582         count += 4;
6583         /* end clear state */
6584         count += 2;
6585         /* clear state */
6586         count += 2;
6587
6588         return count;
6589 }
6590
6591 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6592 {
6593         u32 count = 0, i;
6594         const struct cs_section_def *sect = NULL;
6595         const struct cs_extent_def *ext = NULL;
6596
6597         if (rdev->rlc.cs_data == NULL)
6598                 return;
6599         if (buffer == NULL)
6600                 return;
6601
6602         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6603         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6604
6605         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6606         buffer[count++] = cpu_to_le32(0x80000000);
6607         buffer[count++] = cpu_to_le32(0x80000000);
6608
6609         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6610                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6611                         if (sect->id == SECT_CONTEXT) {
6612                                 buffer[count++] =
6613                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6614                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6615                                 for (i = 0; i < ext->reg_count; i++)
6616                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6617                         } else {
6618                                 return;
6619                         }
6620                 }
6621         }
6622
6623         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6624         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6625         switch (rdev->family) {
6626         case CHIP_BONAIRE:
6627                 buffer[count++] = cpu_to_le32(0x16000012);
6628                 buffer[count++] = cpu_to_le32(0x00000000);
6629                 break;
6630         case CHIP_KAVERI:
6631                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6632                 buffer[count++] = cpu_to_le32(0x00000000);
6633                 break;
6634         case CHIP_KABINI:
6635         case CHIP_MULLINS:
6636                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6637                 buffer[count++] = cpu_to_le32(0x00000000);
6638                 break;
6639         case CHIP_HAWAII:
6640                 buffer[count++] = cpu_to_le32(0x3a00161a);
6641                 buffer[count++] = cpu_to_le32(0x0000002e);
6642                 break;
6643         default:
6644                 buffer[count++] = cpu_to_le32(0x00000000);
6645                 buffer[count++] = cpu_to_le32(0x00000000);
6646                 break;
6647         }
6648
6649         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6650         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6651
6652         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6653         buffer[count++] = cpu_to_le32(0);
6654 }
6655
6656 static void cik_init_pg(struct radeon_device *rdev)
6657 {
6658         if (rdev->pg_flags) {
6659                 cik_enable_sck_slowdown_on_pu(rdev, true);
6660                 cik_enable_sck_slowdown_on_pd(rdev, true);
6661                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6662                         cik_init_gfx_cgpg(rdev);
6663                         cik_enable_cp_pg(rdev, true);
6664                         cik_enable_gds_pg(rdev, true);
6665                 }
6666                 cik_init_ao_cu_mask(rdev);
6667                 cik_update_gfx_pg(rdev, true);
6668         }
6669 }
6670
6671 static void cik_fini_pg(struct radeon_device *rdev)
6672 {
6673         if (rdev->pg_flags) {
6674                 cik_update_gfx_pg(rdev, false);
6675                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6676                         cik_enable_cp_pg(rdev, false);
6677                         cik_enable_gds_pg(rdev, false);
6678                 }
6679         }
6680 }
6681
6682 /*
6683  * Interrupts
6684  * Starting with r6xx, interrupts are handled via a ring buffer.
6685  * Ring buffers are areas of GPU accessible memory that the GPU
6686  * writes interrupt vectors into and the host reads vectors out of.
6687  * There is a rptr (read pointer) that determines where the
6688  * host is currently reading, and a wptr (write pointer)
6689  * which determines where the GPU has written.  When the
6690  * pointers are equal, the ring is idle.  When the GPU
6691  * writes vectors to the ring buffer, it increments the
6692  * wptr.  When there is an interrupt, the host then starts
6693  * fetching commands and processing them until the pointers are
6694  * equal again at which point it updates the rptr.
6695  */
6696
6697 /**
6698  * cik_enable_interrupts - Enable the interrupt ring buffer
6699  *
6700  * @rdev: radeon_device pointer
6701  *
6702  * Enable the interrupt ring buffer (CIK).
6703  */
6704 static void cik_enable_interrupts(struct radeon_device *rdev)
6705 {
6706         u32 ih_cntl = RREG32(IH_CNTL);
6707         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6708
6709         ih_cntl |= ENABLE_INTR;
6710         ih_rb_cntl |= IH_RB_ENABLE;
6711         WREG32(IH_CNTL, ih_cntl);
6712         WREG32(IH_RB_CNTL, ih_rb_cntl);
6713         rdev->ih.enabled = true;
6714 }
6715
6716 /**
6717  * cik_disable_interrupts - Disable the interrupt ring buffer
6718  *
6719  * @rdev: radeon_device pointer
6720  *
6721  * Disable the interrupt ring buffer (CIK).
6722  */
6723 static void cik_disable_interrupts(struct radeon_device *rdev)
6724 {
6725         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6726         u32 ih_cntl = RREG32(IH_CNTL);
6727
6728         ih_rb_cntl &= ~IH_RB_ENABLE;
6729         ih_cntl &= ~ENABLE_INTR;
6730         WREG32(IH_RB_CNTL, ih_rb_cntl);
6731         WREG32(IH_CNTL, ih_cntl);
6732         /* set rptr, wptr to 0 */
6733         WREG32(IH_RB_RPTR, 0);
6734         WREG32(IH_RB_WPTR, 0);
6735         rdev->ih.enabled = false;
6736         rdev->ih.rptr = 0;
6737 }
6738
6739 /**
6740  * cik_disable_interrupt_state - Disable all interrupt sources
6741  *
6742  * @rdev: radeon_device pointer
6743  *
6744  * Clear all interrupt enable bits used by the driver (CIK).
6745  */
6746 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6747 {
6748         u32 tmp;
6749
6750         /* gfx ring */
6751         tmp = RREG32(CP_INT_CNTL_RING0) &
6752                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6753         WREG32(CP_INT_CNTL_RING0, tmp);
6754         /* sdma */
6755         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6756         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6757         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6758         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6759         /* compute queues */
6760         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6761         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6762         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6763         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6764         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6765         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6766         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6767         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6768         /* grbm */
6769         WREG32(GRBM_INT_CNTL, 0);
6770         /* vline/vblank, etc. */
6771         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6772         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6773         if (rdev->num_crtc >= 4) {
6774                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6775                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6776         }
6777         if (rdev->num_crtc >= 6) {
6778                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6779                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6780         }
6781         /* pflip */
6782         if (rdev->num_crtc >= 2) {
6783                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6784                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6785         }
6786         if (rdev->num_crtc >= 4) {
6787                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6788                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6789         }
6790         if (rdev->num_crtc >= 6) {
6791                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6792                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6793         }
6794
6795         /* dac hotplug */
6796         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6797
6798         /* digital hotplug */
6799         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6800         WREG32(DC_HPD1_INT_CONTROL, tmp);
6801         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6802         WREG32(DC_HPD2_INT_CONTROL, tmp);
6803         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6804         WREG32(DC_HPD3_INT_CONTROL, tmp);
6805         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6806         WREG32(DC_HPD4_INT_CONTROL, tmp);
6807         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6808         WREG32(DC_HPD5_INT_CONTROL, tmp);
6809         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6810         WREG32(DC_HPD6_INT_CONTROL, tmp);
6811
6812 }
6813
6814 /**
6815  * cik_irq_init - init and enable the interrupt ring
6816  *
6817  * @rdev: radeon_device pointer
6818  *
6819  * Allocate a ring buffer for the interrupt controller,
6820  * enable the RLC, disable interrupts, enable the IH
6821  * ring buffer and enable it (CIK).
6822  * Called at device load and reume.
6823  * Returns 0 for success, errors for failure.
6824  */
6825 static int cik_irq_init(struct radeon_device *rdev)
6826 {
6827         int ret = 0;
6828         int rb_bufsz;
6829         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6830
6831         /* allocate ring */
6832         ret = r600_ih_ring_alloc(rdev);
6833         if (ret)
6834                 return ret;
6835
6836         /* disable irqs */
6837         cik_disable_interrupts(rdev);
6838
6839         /* init rlc */
6840         ret = cik_rlc_resume(rdev);
6841         if (ret) {
6842                 r600_ih_ring_fini(rdev);
6843                 return ret;
6844         }
6845
6846         /* setup interrupt control */
6847         /* XXX this should actually be a bus address, not an MC address. same on older asics */
6848         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6849         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6850         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6851          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6852          */
6853         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6854         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6855         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6856         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6857
6858         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6859         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6860
6861         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6862                       IH_WPTR_OVERFLOW_CLEAR |
6863                       (rb_bufsz << 1));
6864
6865         if (rdev->wb.enabled)
6866                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6867
6868         /* set the writeback address whether it's enabled or not */
6869         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6870         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6871
6872         WREG32(IH_RB_CNTL, ih_rb_cntl);
6873
6874         /* set rptr, wptr to 0 */
6875         WREG32(IH_RB_RPTR, 0);
6876         WREG32(IH_RB_WPTR, 0);
6877
6878         /* Default settings for IH_CNTL (disabled at first) */
6879         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6880         /* RPTR_REARM only works if msi's are enabled */
6881         if (rdev->msi_enabled)
6882                 ih_cntl |= RPTR_REARM;
6883         WREG32(IH_CNTL, ih_cntl);
6884
6885         /* force the active interrupt state to all disabled */
6886         cik_disable_interrupt_state(rdev);
6887
6888         pci_set_master(rdev->pdev);
6889
6890         /* enable irqs */
6891         cik_enable_interrupts(rdev);
6892
6893         return ret;
6894 }
6895
6896 /**
6897  * cik_irq_set - enable/disable interrupt sources
6898  *
6899  * @rdev: radeon_device pointer
6900  *
6901  * Enable interrupt sources on the GPU (vblanks, hpd,
6902  * etc.) (CIK).
6903  * Returns 0 for success, errors for failure.
6904  */
6905 int cik_irq_set(struct radeon_device *rdev)
6906 {
6907         u32 cp_int_cntl;
6908         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6909         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6910         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6911         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6912         u32 grbm_int_cntl = 0;
6913         u32 dma_cntl, dma_cntl1;
6914         u32 thermal_int;
6915
6916         if (!rdev->irq.installed) {
6917                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6918                 return -EINVAL;
6919         }
6920         /* don't enable anything if the ih is disabled */
6921         if (!rdev->ih.enabled) {
6922                 cik_disable_interrupts(rdev);
6923                 /* force the active interrupt state to all disabled */
6924                 cik_disable_interrupt_state(rdev);
6925                 return 0;
6926         }
6927
6928         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6929                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6930         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6931
6932         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6933         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6934         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6935         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6936         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6937         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6938
6939         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6940         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6941
6942         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6943         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6944         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6945         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6946         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6947         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6948         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6949         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6950
6951         if (rdev->flags & RADEON_IS_IGP)
6952                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6953                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6954         else
6955                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6956                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6957
6958         /* enable CP interrupts on all rings */
6959         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6960                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6961                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6962         }
6963         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6964                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6965                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6966                 if (ring->me == 1) {
6967                         switch (ring->pipe) {
6968                         case 0:
6969                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6970                                 break;
6971                         case 1:
6972                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6973                                 break;
6974                         case 2:
6975                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6976                                 break;
6977                         case 3:
6978                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6979                                 break;
6980                         default:
6981                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6982                                 break;
6983                         }
6984                 } else if (ring->me == 2) {
6985                         switch (ring->pipe) {
6986                         case 0:
6987                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6988                                 break;
6989                         case 1:
6990                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6991                                 break;
6992                         case 2:
6993                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6994                                 break;
6995                         case 3:
6996                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6997                                 break;
6998                         default:
6999                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7000                                 break;
7001                         }
7002                 } else {
7003                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7004                 }
7005         }
7006         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7007                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7008                 DRM_DEBUG("si_irq_set: sw int cp2\n");
7009                 if (ring->me == 1) {
7010                         switch (ring->pipe) {
7011                         case 0:
7012                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7013                                 break;
7014                         case 1:
7015                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7016                                 break;
7017                         case 2:
7018                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7019                                 break;
7020                         case 3:
7021                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7022                                 break;
7023                         default:
7024                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7025                                 break;
7026                         }
7027                 } else if (ring->me == 2) {
7028                         switch (ring->pipe) {
7029                         case 0:
7030                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7031                                 break;
7032                         case 1:
7033                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7034                                 break;
7035                         case 2:
7036                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7037                                 break;
7038                         case 3:
7039                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7040                                 break;
7041                         default:
7042                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7043                                 break;
7044                         }
7045                 } else {
7046                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7047                 }
7048         }
7049
7050         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7051                 DRM_DEBUG("cik_irq_set: sw int dma\n");
7052                 dma_cntl |= TRAP_ENABLE;
7053         }
7054
7055         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7056                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7057                 dma_cntl1 |= TRAP_ENABLE;
7058         }
7059
7060         if (rdev->irq.crtc_vblank_int[0] ||
7061             atomic_read(&rdev->irq.pflip[0])) {
7062                 DRM_DEBUG("cik_irq_set: vblank 0\n");
7063                 crtc1 |= VBLANK_INTERRUPT_MASK;
7064         }
7065         if (rdev->irq.crtc_vblank_int[1] ||
7066             atomic_read(&rdev->irq.pflip[1])) {
7067                 DRM_DEBUG("cik_irq_set: vblank 1\n");
7068                 crtc2 |= VBLANK_INTERRUPT_MASK;
7069         }
7070         if (rdev->irq.crtc_vblank_int[2] ||
7071             atomic_read(&rdev->irq.pflip[2])) {
7072                 DRM_DEBUG("cik_irq_set: vblank 2\n");
7073                 crtc3 |= VBLANK_INTERRUPT_MASK;
7074         }
7075         if (rdev->irq.crtc_vblank_int[3] ||
7076             atomic_read(&rdev->irq.pflip[3])) {
7077                 DRM_DEBUG("cik_irq_set: vblank 3\n");
7078                 crtc4 |= VBLANK_INTERRUPT_MASK;
7079         }
7080         if (rdev->irq.crtc_vblank_int[4] ||
7081             atomic_read(&rdev->irq.pflip[4])) {
7082                 DRM_DEBUG("cik_irq_set: vblank 4\n");
7083                 crtc5 |= VBLANK_INTERRUPT_MASK;
7084         }
7085         if (rdev->irq.crtc_vblank_int[5] ||
7086             atomic_read(&rdev->irq.pflip[5])) {
7087                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7088                 crtc6 |= VBLANK_INTERRUPT_MASK;
7089         }
7090         if (rdev->irq.hpd[0]) {
7091                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7092                 hpd1 |= DC_HPDx_INT_EN;
7093         }
7094         if (rdev->irq.hpd[1]) {
7095                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7096                 hpd2 |= DC_HPDx_INT_EN;
7097         }
7098         if (rdev->irq.hpd[2]) {
7099                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7100                 hpd3 |= DC_HPDx_INT_EN;
7101         }
7102         if (rdev->irq.hpd[3]) {
7103                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7104                 hpd4 |= DC_HPDx_INT_EN;
7105         }
7106         if (rdev->irq.hpd[4]) {
7107                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7108                 hpd5 |= DC_HPDx_INT_EN;
7109         }
7110         if (rdev->irq.hpd[5]) {
7111                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7112                 hpd6 |= DC_HPDx_INT_EN;
7113         }
7114
7115         if (rdev->irq.dpm_thermal) {
7116                 DRM_DEBUG("dpm thermal\n");
7117                 if (rdev->flags & RADEON_IS_IGP)
7118                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7119                 else
7120                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7121         }
7122
7123         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7124
7125         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7126         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7127
7128         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7129         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7130         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7131         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7132         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7133         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7134         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7135         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7136
7137         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7138
7139         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7140         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7141         if (rdev->num_crtc >= 4) {
7142                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7143                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7144         }
7145         if (rdev->num_crtc >= 6) {
7146                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7147                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7148         }
7149
7150         if (rdev->num_crtc >= 2) {
7151                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7152                        GRPH_PFLIP_INT_MASK);
7153                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7154                        GRPH_PFLIP_INT_MASK);
7155         }
7156         if (rdev->num_crtc >= 4) {
7157                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7158                        GRPH_PFLIP_INT_MASK);
7159                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7160                        GRPH_PFLIP_INT_MASK);
7161         }
7162         if (rdev->num_crtc >= 6) {
7163                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7164                        GRPH_PFLIP_INT_MASK);
7165                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7166                        GRPH_PFLIP_INT_MASK);
7167         }
7168
7169         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7170         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7171         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7172         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7173         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7174         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7175
7176         if (rdev->flags & RADEON_IS_IGP)
7177                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7178         else
7179                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7180
7181         return 0;
7182 }
7183
7184 /**
7185  * cik_irq_ack - ack interrupt sources
7186  *
7187  * @rdev: radeon_device pointer
7188  *
7189  * Ack interrupt sources on the GPU (vblanks, hpd,
7190  * etc.) (CIK).  Certain interrupts sources are sw
7191  * generated and do not require an explicit ack.
7192  */
7193 static inline void cik_irq_ack(struct radeon_device *rdev)
7194 {
7195         u32 tmp;
7196
7197         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7198         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7199         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7200         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7201         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7202         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7203         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7204
7205         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7206                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7207         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7208                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7209         if (rdev->num_crtc >= 4) {
7210                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7211                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7212                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7213                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7214         }
7215         if (rdev->num_crtc >= 6) {
7216                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7217                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7218                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7219                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7220         }
7221
7222         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7223                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7224                        GRPH_PFLIP_INT_CLEAR);
7225         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7226                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7227                        GRPH_PFLIP_INT_CLEAR);
7228         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7229                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7230         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7231                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7232         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7233                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7234         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7235                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7236
7237         if (rdev->num_crtc >= 4) {
7238                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7239                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7240                                GRPH_PFLIP_INT_CLEAR);
7241                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7242                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7243                                GRPH_PFLIP_INT_CLEAR);
7244                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7245                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7246                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7247                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7248                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7249                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7250                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7251                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7252         }
7253
7254         if (rdev->num_crtc >= 6) {
7255                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7256                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7257                                GRPH_PFLIP_INT_CLEAR);
7258                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7259                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7260                                GRPH_PFLIP_INT_CLEAR);
7261                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7262                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7263                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7264                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7265                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7266                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7267                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7268                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7269         }
7270
7271         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7272                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7273                 tmp |= DC_HPDx_INT_ACK;
7274                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7275         }
7276         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7277                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7278                 tmp |= DC_HPDx_INT_ACK;
7279                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7280         }
7281         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7282                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7283                 tmp |= DC_HPDx_INT_ACK;
7284                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7285         }
7286         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7287                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7288                 tmp |= DC_HPDx_INT_ACK;
7289                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7290         }
7291         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7292                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7293                 tmp |= DC_HPDx_INT_ACK;
7294                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7295         }
7296         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7297                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7298                 tmp |= DC_HPDx_INT_ACK;
7299                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7300         }
7301 }
7302
7303 /**
7304  * cik_irq_disable - disable interrupts
7305  *
7306  * @rdev: radeon_device pointer
7307  *
7308  * Disable interrupts on the hw (CIK).
7309  */
7310 static void cik_irq_disable(struct radeon_device *rdev)
7311 {
7312         cik_disable_interrupts(rdev);
7313         /* Wait and acknowledge irq */
7314         mdelay(1);
7315         cik_irq_ack(rdev);
7316         cik_disable_interrupt_state(rdev);
7317 }
7318
7319 /**
7320  * cik_irq_disable - disable interrupts for suspend
7321  *
7322  * @rdev: radeon_device pointer
7323  *
7324  * Disable interrupts and stop the RLC (CIK).
7325  * Used for suspend.
7326  */
7327 static void cik_irq_suspend(struct radeon_device *rdev)
7328 {
7329         cik_irq_disable(rdev);
7330         cik_rlc_stop(rdev);
7331 }
7332
7333 /**
7334  * cik_irq_fini - tear down interrupt support
7335  *
7336  * @rdev: radeon_device pointer
7337  *
7338  * Disable interrupts on the hw and free the IH ring
7339  * buffer (CIK).
7340  * Used for driver unload.
7341  */
7342 static void cik_irq_fini(struct radeon_device *rdev)
7343 {
7344         cik_irq_suspend(rdev);
7345         r600_ih_ring_fini(rdev);
7346 }
7347
7348 /**
7349  * cik_get_ih_wptr - get the IH ring buffer wptr
7350  *
7351  * @rdev: radeon_device pointer
7352  *
7353  * Get the IH ring buffer wptr from either the register
7354  * or the writeback memory buffer (CIK).  Also check for
7355  * ring buffer overflow and deal with it.
7356  * Used by cik_irq_process().
7357  * Returns the value of the wptr.
7358  */
7359 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7360 {
7361         u32 wptr, tmp;
7362
7363         if (rdev->wb.enabled)
7364                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7365         else
7366                 wptr = RREG32(IH_RB_WPTR);
7367
7368         if (wptr & RB_OVERFLOW) {
7369                 /* When a ring buffer overflow happen start parsing interrupt
7370                  * from the last not overwritten vector (wptr + 16). Hopefully
7371                  * this should allow us to catchup.
7372                  */
7373                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7374                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7375                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7376                 tmp = RREG32(IH_RB_CNTL);
7377                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7378                 WREG32(IH_RB_CNTL, tmp);
7379         }
7380         return (wptr & rdev->ih.ptr_mask);
7381 }
7382
7383 /*        CIK IV Ring
7384  * Each IV ring entry is 128 bits:
7385  * [7:0]    - interrupt source id
7386  * [31:8]   - reserved
7387  * [59:32]  - interrupt source data
7388  * [63:60]  - reserved
7389  * [71:64]  - RINGID
7390  *            CP:
7391  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7392  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7393  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7394  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7395  *            PIPE_ID - ME0 0=3D
7396  *                    - ME1&2 compute dispatcher (4 pipes each)
7397  *            SDMA:
7398  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7399  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7400  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7401  * [79:72]  - VMID
7402  * [95:80]  - PASID
7403  * [127:96] - reserved
7404  */
7405 /**
7406  * cik_irq_process - interrupt handler
7407  *
7408  * @rdev: radeon_device pointer
7409  *
7410  * Interrupt hander (CIK).  Walk the IH ring,
7411  * ack interrupts and schedule work to handle
7412  * interrupt events.
7413  * Returns irq process return code.
7414  */
7415 int cik_irq_process(struct radeon_device *rdev)
7416 {
7417         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7418         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7419         u32 wptr;
7420         u32 rptr;
7421         u32 src_id, src_data, ring_id;
7422         u8 me_id, pipe_id, queue_id;
7423         u32 ring_index;
7424         bool queue_hotplug = false;
7425         bool queue_reset = false;
7426         u32 addr, status, mc_client;
7427         bool queue_thermal = false;
7428
7429         if (!rdev->ih.enabled || rdev->shutdown)
7430                 return IRQ_NONE;
7431
7432         wptr = cik_get_ih_wptr(rdev);
7433
7434 restart_ih:
7435         /* is somebody else already processing irqs? */
7436         if (atomic_xchg(&rdev->ih.lock, 1))
7437                 return IRQ_NONE;
7438
7439         rptr = rdev->ih.rptr;
7440         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7441
7442         /* Order reading of wptr vs. reading of IH ring data */
7443         rmb();
7444
7445         /* display interrupts */
7446         cik_irq_ack(rdev);
7447
7448         while (rptr != wptr) {
7449                 /* wptr/rptr are in bytes! */
7450                 ring_index = rptr / 4;
7451                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7452                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7453                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7454
7455                 switch (src_id) {
7456                 case 1: /* D1 vblank/vline */
7457                         switch (src_data) {
7458                         case 0: /* D1 vblank */
7459                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7460                                         if (rdev->irq.crtc_vblank_int[0]) {
7461                                                 drm_handle_vblank(rdev->ddev, 0);
7462                                                 rdev->pm.vblank_sync = true;
7463                                                 wake_up(&rdev->irq.vblank_queue);
7464                                         }
7465                                         if (atomic_read(&rdev->irq.pflip[0]))
7466                                                 radeon_crtc_handle_vblank(rdev, 0);
7467                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7468                                         DRM_DEBUG("IH: D1 vblank\n");
7469                                 }
7470                                 break;
7471                         case 1: /* D1 vline */
7472                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7473                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7474                                         DRM_DEBUG("IH: D1 vline\n");
7475                                 }
7476                                 break;
7477                         default:
7478                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7479                                 break;
7480                         }
7481                         break;
7482                 case 2: /* D2 vblank/vline */
7483                         switch (src_data) {
7484                         case 0: /* D2 vblank */
7485                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7486                                         if (rdev->irq.crtc_vblank_int[1]) {
7487                                                 drm_handle_vblank(rdev->ddev, 1);
7488                                                 rdev->pm.vblank_sync = true;
7489                                                 wake_up(&rdev->irq.vblank_queue);
7490                                         }
7491                                         if (atomic_read(&rdev->irq.pflip[1]))
7492                                                 radeon_crtc_handle_vblank(rdev, 1);
7493                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7494                                         DRM_DEBUG("IH: D2 vblank\n");
7495                                 }
7496                                 break;
7497                         case 1: /* D2 vline */
7498                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7499                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7500                                         DRM_DEBUG("IH: D2 vline\n");
7501                                 }
7502                                 break;
7503                         default:
7504                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7505                                 break;
7506                         }
7507                         break;
7508                 case 3: /* D3 vblank/vline */
7509                         switch (src_data) {
7510                         case 0: /* D3 vblank */
7511                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7512                                         if (rdev->irq.crtc_vblank_int[2]) {
7513                                                 drm_handle_vblank(rdev->ddev, 2);
7514                                                 rdev->pm.vblank_sync = true;
7515                                                 wake_up(&rdev->irq.vblank_queue);
7516                                         }
7517                                         if (atomic_read(&rdev->irq.pflip[2]))
7518                                                 radeon_crtc_handle_vblank(rdev, 2);
7519                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7520                                         DRM_DEBUG("IH: D3 vblank\n");
7521                                 }
7522                                 break;
7523                         case 1: /* D3 vline */
7524                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7525                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7526                                         DRM_DEBUG("IH: D3 vline\n");
7527                                 }
7528                                 break;
7529                         default:
7530                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7531                                 break;
7532                         }
7533                         break;
7534                 case 4: /* D4 vblank/vline */
7535                         switch (src_data) {
7536                         case 0: /* D4 vblank */
7537                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7538                                         if (rdev->irq.crtc_vblank_int[3]) {
7539                                                 drm_handle_vblank(rdev->ddev, 3);
7540                                                 rdev->pm.vblank_sync = true;
7541                                                 wake_up(&rdev->irq.vblank_queue);
7542                                         }
7543                                         if (atomic_read(&rdev->irq.pflip[3]))
7544                                                 radeon_crtc_handle_vblank(rdev, 3);
7545                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7546                                         DRM_DEBUG("IH: D4 vblank\n");
7547                                 }
7548                                 break;
7549                         case 1: /* D4 vline */
7550                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7551                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7552                                         DRM_DEBUG("IH: D4 vline\n");
7553                                 }
7554                                 break;
7555                         default:
7556                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7557                                 break;
7558                         }
7559                         break;
7560                 case 5: /* D5 vblank/vline */
7561                         switch (src_data) {
7562                         case 0: /* D5 vblank */
7563                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7564                                         if (rdev->irq.crtc_vblank_int[4]) {
7565                                                 drm_handle_vblank(rdev->ddev, 4);
7566                                                 rdev->pm.vblank_sync = true;
7567                                                 wake_up(&rdev->irq.vblank_queue);
7568                                         }
7569                                         if (atomic_read(&rdev->irq.pflip[4]))
7570                                                 radeon_crtc_handle_vblank(rdev, 4);
7571                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7572                                         DRM_DEBUG("IH: D5 vblank\n");
7573                                 }
7574                                 break;
7575                         case 1: /* D5 vline */
7576                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7577                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7578                                         DRM_DEBUG("IH: D5 vline\n");
7579                                 }
7580                                 break;
7581                         default:
7582                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7583                                 break;
7584                         }
7585                         break;
7586                 case 6: /* D6 vblank/vline */
7587                         switch (src_data) {
7588                         case 0: /* D6 vblank */
7589                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7590                                         if (rdev->irq.crtc_vblank_int[5]) {
7591                                                 drm_handle_vblank(rdev->ddev, 5);
7592                                                 rdev->pm.vblank_sync = true;
7593                                                 wake_up(&rdev->irq.vblank_queue);
7594                                         }
7595                                         if (atomic_read(&rdev->irq.pflip[5]))
7596                                                 radeon_crtc_handle_vblank(rdev, 5);
7597                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7598                                         DRM_DEBUG("IH: D6 vblank\n");
7599                                 }
7600                                 break;
7601                         case 1: /* D6 vline */
7602                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7603                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7604                                         DRM_DEBUG("IH: D6 vline\n");
7605                                 }
7606                                 break;
7607                         default:
7608                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7609                                 break;
7610                         }
7611                         break;
7612                 case 8: /* D1 page flip */
7613                 case 10: /* D2 page flip */
7614                 case 12: /* D3 page flip */
7615                 case 14: /* D4 page flip */
7616                 case 16: /* D5 page flip */
7617                 case 18: /* D6 page flip */
7618                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7619                         radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7620                         break;
7621                 case 42: /* HPD hotplug */
7622                         switch (src_data) {
7623                         case 0:
7624                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7625                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7626                                         queue_hotplug = true;
7627                                         DRM_DEBUG("IH: HPD1\n");
7628                                 }
7629                                 break;
7630                         case 1:
7631                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7632                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7633                                         queue_hotplug = true;
7634                                         DRM_DEBUG("IH: HPD2\n");
7635                                 }
7636                                 break;
7637                         case 2:
7638                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7639                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7640                                         queue_hotplug = true;
7641                                         DRM_DEBUG("IH: HPD3\n");
7642                                 }
7643                                 break;
7644                         case 3:
7645                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7646                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7647                                         queue_hotplug = true;
7648                                         DRM_DEBUG("IH: HPD4\n");
7649                                 }
7650                                 break;
7651                         case 4:
7652                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7653                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7654                                         queue_hotplug = true;
7655                                         DRM_DEBUG("IH: HPD5\n");
7656                                 }
7657                                 break;
7658                         case 5:
7659                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7660                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7661                                         queue_hotplug = true;
7662                                         DRM_DEBUG("IH: HPD6\n");
7663                                 }
7664                                 break;
7665                         default:
7666                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7667                                 break;
7668                         }
7669                         break;
7670                 case 124: /* UVD */
7671                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7672                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7673                         break;
7674                 case 146:
7675                 case 147:
7676                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7677                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7678                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7679                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7680                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7681                                 addr);
7682                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7683                                 status);
7684                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7685                         /* reset addr and status */
7686                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7687                         break;
7688                 case 167: /* VCE */
7689                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7690                         switch (src_data) {
7691                         case 0:
7692                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7693                                 break;
7694                         case 1:
7695                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7696                                 break;
7697                         default:
7698                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7699                                 break;
7700                         }
7701                         break;
7702                 case 176: /* GFX RB CP_INT */
7703                 case 177: /* GFX IB CP_INT */
7704                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7705                         break;
7706                 case 181: /* CP EOP event */
7707                         DRM_DEBUG("IH: CP EOP\n");
7708                         /* XXX check the bitfield order! */
7709                         me_id = (ring_id & 0x60) >> 5;
7710                         pipe_id = (ring_id & 0x18) >> 3;
7711                         queue_id = (ring_id & 0x7) >> 0;
7712                         switch (me_id) {
7713                         case 0:
7714                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7715                                 break;
7716                         case 1:
7717                         case 2:
7718                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7719                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7720                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7721                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7722                                 break;
7723                         }
7724                         break;
7725                 case 184: /* CP Privileged reg access */
7726                         DRM_ERROR("Illegal register access in command stream\n");
7727                         /* XXX check the bitfield order! */
7728                         me_id = (ring_id & 0x60) >> 5;
7729                         pipe_id = (ring_id & 0x18) >> 3;
7730                         queue_id = (ring_id & 0x7) >> 0;
7731                         switch (me_id) {
7732                         case 0:
7733                                 /* This results in a full GPU reset, but all we need to do is soft
7734                                  * reset the CP for gfx
7735                                  */
7736                                 queue_reset = true;
7737                                 break;
7738                         case 1:
7739                                 /* XXX compute */
7740                                 queue_reset = true;
7741                                 break;
7742                         case 2:
7743                                 /* XXX compute */
7744                                 queue_reset = true;
7745                                 break;
7746                         }
7747                         break;
7748                 case 185: /* CP Privileged inst */
7749                         DRM_ERROR("Illegal instruction in command stream\n");
7750                         /* XXX check the bitfield order! */
7751                         me_id = (ring_id & 0x60) >> 5;
7752                         pipe_id = (ring_id & 0x18) >> 3;
7753                         queue_id = (ring_id & 0x7) >> 0;
7754                         switch (me_id) {
7755                         case 0:
7756                                 /* This results in a full GPU reset, but all we need to do is soft
7757                                  * reset the CP for gfx
7758                                  */
7759                                 queue_reset = true;
7760                                 break;
7761                         case 1:
7762                                 /* XXX compute */
7763                                 queue_reset = true;
7764                                 break;
7765                         case 2:
7766                                 /* XXX compute */
7767                                 queue_reset = true;
7768                                 break;
7769                         }
7770                         break;
7771                 case 224: /* SDMA trap event */
7772                         /* XXX check the bitfield order! */
7773                         me_id = (ring_id & 0x3) >> 0;
7774                         queue_id = (ring_id & 0xc) >> 2;
7775                         DRM_DEBUG("IH: SDMA trap\n");
7776                         switch (me_id) {
7777                         case 0:
7778                                 switch (queue_id) {
7779                                 case 0:
7780                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7781                                         break;
7782                                 case 1:
7783                                         /* XXX compute */
7784                                         break;
7785                                 case 2:
7786                                         /* XXX compute */
7787                                         break;
7788                                 }
7789                                 break;
7790                         case 1:
7791                                 switch (queue_id) {
7792                                 case 0:
7793                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7794                                         break;
7795                                 case 1:
7796                                         /* XXX compute */
7797                                         break;
7798                                 case 2:
7799                                         /* XXX compute */
7800                                         break;
7801                                 }
7802                                 break;
7803                         }
7804                         break;
7805                 case 230: /* thermal low to high */
7806                         DRM_DEBUG("IH: thermal low to high\n");
7807                         rdev->pm.dpm.thermal.high_to_low = false;
7808                         queue_thermal = true;
7809                         break;
7810                 case 231: /* thermal high to low */
7811                         DRM_DEBUG("IH: thermal high to low\n");
7812                         rdev->pm.dpm.thermal.high_to_low = true;
7813                         queue_thermal = true;
7814                         break;
7815                 case 233: /* GUI IDLE */
7816                         DRM_DEBUG("IH: GUI idle\n");
7817                         break;
7818                 case 241: /* SDMA Privileged inst */
7819                 case 247: /* SDMA Privileged inst */
7820                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
7821                         /* XXX check the bitfield order! */
7822                         me_id = (ring_id & 0x3) >> 0;
7823                         queue_id = (ring_id & 0xc) >> 2;
7824                         switch (me_id) {
7825                         case 0:
7826                                 switch (queue_id) {
7827                                 case 0:
7828                                         queue_reset = true;
7829                                         break;
7830                                 case 1:
7831                                         /* XXX compute */
7832                                         queue_reset = true;
7833                                         break;
7834                                 case 2:
7835                                         /* XXX compute */
7836                                         queue_reset = true;
7837                                         break;
7838                                 }
7839                                 break;
7840                         case 1:
7841                                 switch (queue_id) {
7842                                 case 0:
7843                                         queue_reset = true;
7844                                         break;
7845                                 case 1:
7846                                         /* XXX compute */
7847                                         queue_reset = true;
7848                                         break;
7849                                 case 2:
7850                                         /* XXX compute */
7851                                         queue_reset = true;
7852                                         break;
7853                                 }
7854                                 break;
7855                         }
7856                         break;
7857                 default:
7858                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7859                         break;
7860                 }
7861
7862                 /* wptr/rptr are in bytes! */
7863                 rptr += 16;
7864                 rptr &= rdev->ih.ptr_mask;
7865         }
7866         if (queue_hotplug)
7867                 schedule_work(&rdev->hotplug_work);
7868         if (queue_reset)
7869                 schedule_work(&rdev->reset_work);
7870         if (queue_thermal)
7871                 schedule_work(&rdev->pm.dpm.thermal.work);
7872         rdev->ih.rptr = rptr;
7873         WREG32(IH_RB_RPTR, rdev->ih.rptr);
7874         atomic_set(&rdev->ih.lock, 0);
7875
7876         /* make sure wptr hasn't changed while processing */
7877         wptr = cik_get_ih_wptr(rdev);
7878         if (wptr != rptr)
7879                 goto restart_ih;
7880
7881         return IRQ_HANDLED;
7882 }
7883
7884 /*
7885  * startup/shutdown callbacks
7886  */
7887 /**
7888  * cik_startup - program the asic to a functional state
7889  *
7890  * @rdev: radeon_device pointer
7891  *
7892  * Programs the asic to a functional state (CIK).
7893  * Called by cik_init() and cik_resume().
7894  * Returns 0 for success, error for failure.
7895  */
7896 static int cik_startup(struct radeon_device *rdev)
7897 {
7898         struct radeon_ring *ring;
7899         int r;
7900
7901         /* enable pcie gen2/3 link */
7902         cik_pcie_gen3_enable(rdev);
7903         /* enable aspm */
7904         cik_program_aspm(rdev);
7905
7906         /* scratch needs to be initialized before MC */
7907         r = r600_vram_scratch_init(rdev);
7908         if (r)
7909                 return r;
7910
7911         cik_mc_program(rdev);
7912
7913         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7914                 r = ci_mc_load_microcode(rdev);
7915                 if (r) {
7916                         DRM_ERROR("Failed to load MC firmware!\n");
7917                         return r;
7918                 }
7919         }
7920
7921         r = cik_pcie_gart_enable(rdev);
7922         if (r)
7923                 return r;
7924         cik_gpu_init(rdev);
7925
7926         /* allocate rlc buffers */
7927         if (rdev->flags & RADEON_IS_IGP) {
7928                 if (rdev->family == CHIP_KAVERI) {
7929                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7930                         rdev->rlc.reg_list_size =
7931                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7932                 } else {
7933                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7934                         rdev->rlc.reg_list_size =
7935                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7936                 }
7937         }
7938         rdev->rlc.cs_data = ci_cs_data;
7939         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7940         r = sumo_rlc_init(rdev);
7941         if (r) {
7942                 DRM_ERROR("Failed to init rlc BOs!\n");
7943                 return r;
7944         }
7945
7946         /* allocate wb buffer */
7947         r = radeon_wb_init(rdev);
7948         if (r)
7949                 return r;
7950
7951         /* allocate mec buffers */
7952         r = cik_mec_init(rdev);
7953         if (r) {
7954                 DRM_ERROR("Failed to init MEC BOs!\n");
7955                 return r;
7956         }
7957
7958         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7959         if (r) {
7960                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7961                 return r;
7962         }
7963
7964         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7965         if (r) {
7966                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7967                 return r;
7968         }
7969
7970         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7971         if (r) {
7972                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7973                 return r;
7974         }
7975
7976         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7977         if (r) {
7978                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7979                 return r;
7980         }
7981
7982         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7983         if (r) {
7984                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7985                 return r;
7986         }
7987
7988         r = radeon_uvd_resume(rdev);
7989         if (!r) {
7990                 r = uvd_v4_2_resume(rdev);
7991                 if (!r) {
7992                         r = radeon_fence_driver_start_ring(rdev,
7993                                                            R600_RING_TYPE_UVD_INDEX);
7994                         if (r)
7995                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7996                 }
7997         }
7998         if (r)
7999                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8000
8001         r = radeon_vce_resume(rdev);
8002         if (!r) {
8003                 r = vce_v2_0_resume(rdev);
8004                 if (!r)
8005                         r = radeon_fence_driver_start_ring(rdev,
8006                                                            TN_RING_TYPE_VCE1_INDEX);
8007                 if (!r)
8008                         r = radeon_fence_driver_start_ring(rdev,
8009                                                            TN_RING_TYPE_VCE2_INDEX);
8010         }
8011         if (r) {
8012                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8013                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8014                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8015         }
8016
8017         /* Enable IRQ */
8018         if (!rdev->irq.installed) {
8019                 r = radeon_irq_kms_init(rdev);
8020                 if (r)
8021                         return r;
8022         }
8023
8024         r = cik_irq_init(rdev);
8025         if (r) {
8026                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8027                 radeon_irq_kms_fini(rdev);
8028                 return r;
8029         }
8030         cik_irq_set(rdev);
8031
8032         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8033         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8034                              PACKET3(PACKET3_NOP, 0x3FFF));
8035         if (r)
8036                 return r;
8037
8038         /* set up the compute queues */
8039         /* type-2 packets are deprecated on MEC, use type-3 instead */
8040         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8041         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8042                              PACKET3(PACKET3_NOP, 0x3FFF));
8043         if (r)
8044                 return r;
8045         ring->me = 1; /* first MEC */
8046         ring->pipe = 0; /* first pipe */
8047         ring->queue = 0; /* first queue */
8048         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8049
8050         /* type-2 packets are deprecated on MEC, use type-3 instead */
8051         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8052         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8053                              PACKET3(PACKET3_NOP, 0x3FFF));
8054         if (r)
8055                 return r;
8056         /* dGPU only have 1 MEC */
8057         ring->me = 1; /* first MEC */
8058         ring->pipe = 0; /* first pipe */
8059         ring->queue = 1; /* second queue */
8060         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8061
8062         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8063         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8064                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8065         if (r)
8066                 return r;
8067
8068         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8069         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8070                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8071         if (r)
8072                 return r;
8073
8074         r = cik_cp_resume(rdev);
8075         if (r)
8076                 return r;
8077
8078         r = cik_sdma_resume(rdev);
8079         if (r)
8080                 return r;
8081
8082         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8083         if (ring->ring_size) {
8084                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8085                                      RADEON_CP_PACKET2);
8086                 if (!r)
8087                         r = uvd_v1_0_init(rdev);
8088                 if (r)
8089                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8090         }
8091
8092         r = -ENOENT;
8093
8094         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8095         if (ring->ring_size)
8096                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8097                                      VCE_CMD_NO_OP);
8098
8099         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8100         if (ring->ring_size)
8101                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8102                                      VCE_CMD_NO_OP);
8103
8104         if (!r)
8105                 r = vce_v1_0_init(rdev);
8106         else if (r != -ENOENT)
8107                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8108
8109         r = radeon_ib_pool_init(rdev);
8110         if (r) {
8111                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8112                 return r;
8113         }
8114
8115         r = radeon_vm_manager_init(rdev);
8116         if (r) {
8117                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8118                 return r;
8119         }
8120
8121         r = dce6_audio_init(rdev);
8122         if (r)
8123                 return r;
8124
8125         return 0;
8126 }
8127
8128 /**
8129  * cik_resume - resume the asic to a functional state
8130  *
8131  * @rdev: radeon_device pointer
8132  *
8133  * Programs the asic to a functional state (CIK).
8134  * Called at resume.
8135  * Returns 0 for success, error for failure.
8136  */
8137 int cik_resume(struct radeon_device *rdev)
8138 {
8139         int r;
8140
8141         /* post card */
8142         atom_asic_init(rdev->mode_info.atom_context);
8143
8144         /* init golden registers */
8145         cik_init_golden_registers(rdev);
8146
8147         if (rdev->pm.pm_method == PM_METHOD_DPM)
8148                 radeon_pm_resume(rdev);
8149
8150         rdev->accel_working = true;
8151         r = cik_startup(rdev);
8152         if (r) {
8153                 DRM_ERROR("cik startup failed on resume\n");
8154                 rdev->accel_working = false;
8155                 return r;
8156         }
8157
8158         return r;
8159
8160 }
8161
8162 /**
8163  * cik_suspend - suspend the asic
8164  *
8165  * @rdev: radeon_device pointer
8166  *
8167  * Bring the chip into a state suitable for suspend (CIK).
8168  * Called at suspend.
8169  * Returns 0 for success.
8170  */
8171 int cik_suspend(struct radeon_device *rdev)
8172 {
8173         radeon_pm_suspend(rdev);
8174         dce6_audio_fini(rdev);
8175         radeon_vm_manager_fini(rdev);
8176         cik_cp_enable(rdev, false);
8177         cik_sdma_enable(rdev, false);
8178         uvd_v1_0_fini(rdev);
8179         radeon_uvd_suspend(rdev);
8180         radeon_vce_suspend(rdev);
8181         cik_fini_pg(rdev);
8182         cik_fini_cg(rdev);
8183         cik_irq_suspend(rdev);
8184         radeon_wb_disable(rdev);
8185         cik_pcie_gart_disable(rdev);
8186         return 0;
8187 }
8188
8189 /* Plan is to move initialization in that function and use
8190  * helper function so that radeon_device_init pretty much
8191  * do nothing more than calling asic specific function. This
8192  * should also allow to remove a bunch of callback function
8193  * like vram_info.
8194  */
8195 /**
8196  * cik_init - asic specific driver and hw init
8197  *
8198  * @rdev: radeon_device pointer
8199  *
8200  * Setup asic specific driver variables and program the hw
8201  * to a functional state (CIK).
8202  * Called at driver startup.
8203  * Returns 0 for success, errors for failure.
8204  */
8205 int cik_init(struct radeon_device *rdev)
8206 {
8207         struct radeon_ring *ring;
8208         int r;
8209
8210         /* Read BIOS */
8211         if (!radeon_get_bios(rdev)) {
8212                 if (ASIC_IS_AVIVO(rdev))
8213                         return -EINVAL;
8214         }
8215         /* Must be an ATOMBIOS */
8216         if (!rdev->is_atom_bios) {
8217                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8218                 return -EINVAL;
8219         }
8220         r = radeon_atombios_init(rdev);
8221         if (r)
8222                 return r;
8223
8224         /* Post card if necessary */
8225         if (!radeon_card_posted(rdev)) {
8226                 if (!rdev->bios) {
8227                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8228                         return -EINVAL;
8229                 }
8230                 DRM_INFO("GPU not posted. posting now...\n");
8231                 atom_asic_init(rdev->mode_info.atom_context);
8232         }
8233         /* init golden registers */
8234         cik_init_golden_registers(rdev);
8235         /* Initialize scratch registers */
8236         cik_scratch_init(rdev);
8237         /* Initialize surface registers */
8238         radeon_surface_init(rdev);
8239         /* Initialize clocks */
8240         radeon_get_clock_info(rdev->ddev);
8241
8242         /* Fence driver */
8243         r = radeon_fence_driver_init(rdev);
8244         if (r)
8245                 return r;
8246
8247         /* initialize memory controller */
8248         r = cik_mc_init(rdev);
8249         if (r)
8250                 return r;
8251         /* Memory manager */
8252         r = radeon_bo_init(rdev);
8253         if (r)
8254                 return r;
8255
8256         if (rdev->flags & RADEON_IS_IGP) {
8257                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8258                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8259                         r = cik_init_microcode(rdev);
8260                         if (r) {
8261                                 DRM_ERROR("Failed to load firmware!\n");
8262                                 return r;
8263                         }
8264                 }
8265         } else {
8266                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8267                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8268                     !rdev->mc_fw) {
8269                         r = cik_init_microcode(rdev);
8270                         if (r) {
8271                                 DRM_ERROR("Failed to load firmware!\n");
8272                                 return r;
8273                         }
8274                 }
8275         }
8276
8277         /* Initialize power management */
8278         radeon_pm_init(rdev);
8279
8280         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8281         ring->ring_obj = NULL;
8282         r600_ring_init(rdev, ring, 1024 * 1024);
8283
8284         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8285         ring->ring_obj = NULL;
8286         r600_ring_init(rdev, ring, 1024 * 1024);
8287         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8288         if (r)
8289                 return r;
8290
8291         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8292         ring->ring_obj = NULL;
8293         r600_ring_init(rdev, ring, 1024 * 1024);
8294         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8295         if (r)
8296                 return r;
8297
8298         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8299         ring->ring_obj = NULL;
8300         r600_ring_init(rdev, ring, 256 * 1024);
8301
8302         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8303         ring->ring_obj = NULL;
8304         r600_ring_init(rdev, ring, 256 * 1024);
8305
8306         r = radeon_uvd_init(rdev);
8307         if (!r) {
8308                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8309                 ring->ring_obj = NULL;
8310                 r600_ring_init(rdev, ring, 4096);
8311         }
8312
8313         r = radeon_vce_init(rdev);
8314         if (!r) {
8315                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8316                 ring->ring_obj = NULL;
8317                 r600_ring_init(rdev, ring, 4096);
8318
8319                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8320                 ring->ring_obj = NULL;
8321                 r600_ring_init(rdev, ring, 4096);
8322         }
8323
8324         rdev->ih.ring_obj = NULL;
8325         r600_ih_ring_init(rdev, 64 * 1024);
8326
8327         r = r600_pcie_gart_init(rdev);
8328         if (r)
8329                 return r;
8330
8331         rdev->accel_working = true;
8332         r = cik_startup(rdev);
8333         if (r) {
8334                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8335                 cik_cp_fini(rdev);
8336                 cik_sdma_fini(rdev);
8337                 cik_irq_fini(rdev);
8338                 sumo_rlc_fini(rdev);
8339                 cik_mec_fini(rdev);
8340                 radeon_wb_fini(rdev);
8341                 radeon_ib_pool_fini(rdev);
8342                 radeon_vm_manager_fini(rdev);
8343                 radeon_irq_kms_fini(rdev);
8344                 cik_pcie_gart_fini(rdev);
8345                 rdev->accel_working = false;
8346         }
8347
8348         /* Don't start up if the MC ucode is missing.
8349          * The default clocks and voltages before the MC ucode
8350          * is loaded are not suffient for advanced operations.
8351          */
8352         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8353                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8354                 return -EINVAL;
8355         }
8356
8357         return 0;
8358 }
8359
8360 /**
8361  * cik_fini - asic specific driver and hw fini
8362  *
8363  * @rdev: radeon_device pointer
8364  *
8365  * Tear down the asic specific driver variables and program the hw
8366  * to an idle state (CIK).
8367  * Called at driver unload.
8368  */
8369 void cik_fini(struct radeon_device *rdev)
8370 {
8371         radeon_pm_fini(rdev);
8372         cik_cp_fini(rdev);
8373         cik_sdma_fini(rdev);
8374         cik_fini_pg(rdev);
8375         cik_fini_cg(rdev);
8376         cik_irq_fini(rdev);
8377         sumo_rlc_fini(rdev);
8378         cik_mec_fini(rdev);
8379         radeon_wb_fini(rdev);
8380         radeon_vm_manager_fini(rdev);
8381         radeon_ib_pool_fini(rdev);
8382         radeon_irq_kms_fini(rdev);
8383         uvd_v1_0_fini(rdev);
8384         radeon_uvd_fini(rdev);
8385         radeon_vce_fini(rdev);
8386         cik_pcie_gart_fini(rdev);
8387         r600_vram_scratch_fini(rdev);
8388         radeon_gem_fini(rdev);
8389         radeon_fence_driver_fini(rdev);
8390         radeon_bo_fini(rdev);
8391         radeon_atombios_fini(rdev);
8392         kfree(rdev->bios);
8393         rdev->bios = NULL;
8394 }
8395
8396 void dce8_program_fmt(struct drm_encoder *encoder)
8397 {
8398         struct drm_device *dev = encoder->dev;
8399         struct radeon_device *rdev = dev->dev_private;
8400         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8401         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8402         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8403         int bpc = 0;
8404         u32 tmp = 0;
8405         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8406
8407         if (connector) {
8408                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8409                 bpc = radeon_get_monitor_bpc(connector);
8410                 dither = radeon_connector->dither;
8411         }
8412
8413         /* LVDS/eDP FMT is set up by atom */
8414         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8415                 return;
8416
8417         /* not needed for analog */
8418         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8419             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8420                 return;
8421
8422         if (bpc == 0)
8423                 return;
8424
8425         switch (bpc) {
8426         case 6:
8427                 if (dither == RADEON_FMT_DITHER_ENABLE)
8428                         /* XXX sort out optimal dither settings */
8429                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8430                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8431                 else
8432                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8433                 break;
8434         case 8:
8435                 if (dither == RADEON_FMT_DITHER_ENABLE)
8436                         /* XXX sort out optimal dither settings */
8437                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8438                                 FMT_RGB_RANDOM_ENABLE |
8439                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8440                 else
8441                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8442                 break;
8443         case 10:
8444                 if (dither == RADEON_FMT_DITHER_ENABLE)
8445                         /* XXX sort out optimal dither settings */
8446                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8447                                 FMT_RGB_RANDOM_ENABLE |
8448                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8449                 else
8450                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8451                 break;
8452         default:
8453                 /* not needed */
8454                 break;
8455         }
8456
8457         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8458 }
8459
8460 /* display watermark setup */
8461 /**
8462  * dce8_line_buffer_adjust - Set up the line buffer
8463  *
8464  * @rdev: radeon_device pointer
8465  * @radeon_crtc: the selected display controller
8466  * @mode: the current display mode on the selected display
8467  * controller
8468  *
8469  * Setup up the line buffer allocation for
8470  * the selected display controller (CIK).
8471  * Returns the line buffer size in pixels.
8472  */
8473 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8474                                    struct radeon_crtc *radeon_crtc,
8475                                    struct drm_display_mode *mode)
8476 {
8477         u32 tmp, buffer_alloc, i;
8478         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8479         /*
8480          * Line Buffer Setup
8481          * There are 6 line buffers, one for each display controllers.
8482          * There are 3 partitions per LB. Select the number of partitions
8483          * to enable based on the display width.  For display widths larger
8484          * than 4096, you need use to use 2 display controllers and combine
8485          * them using the stereo blender.
8486          */
8487         if (radeon_crtc->base.enabled && mode) {
8488                 if (mode->crtc_hdisplay < 1920) {
8489                         tmp = 1;
8490                         buffer_alloc = 2;
8491                 } else if (mode->crtc_hdisplay < 2560) {
8492                         tmp = 2;
8493                         buffer_alloc = 2;
8494                 } else if (mode->crtc_hdisplay < 4096) {
8495                         tmp = 0;
8496                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8497                 } else {
8498                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8499                         tmp = 0;
8500                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8501                 }
8502         } else {
8503                 tmp = 1;
8504                 buffer_alloc = 0;
8505         }
8506
8507         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8508                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8509
8510         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8511                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8512         for (i = 0; i < rdev->usec_timeout; i++) {
8513                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8514                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8515                         break;
8516                 udelay(1);
8517         }
8518
8519         if (radeon_crtc->base.enabled && mode) {
8520                 switch (tmp) {
8521                 case 0:
8522                 default:
8523                         return 4096 * 2;
8524                 case 1:
8525                         return 1920 * 2;
8526                 case 2:
8527                         return 2560 * 2;
8528                 }
8529         }
8530
8531         /* controller not enabled, so no lb used */
8532         return 0;
8533 }
8534
8535 /**
8536  * cik_get_number_of_dram_channels - get the number of dram channels
8537  *
8538  * @rdev: radeon_device pointer
8539  *
8540  * Look up the number of video ram channels (CIK).
8541  * Used for display watermark bandwidth calculations
8542  * Returns the number of dram channels
8543  */
8544 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8545 {
8546         u32 tmp = RREG32(MC_SHARED_CHMAP);
8547
8548         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8549         case 0:
8550         default:
8551                 return 1;
8552         case 1:
8553                 return 2;
8554         case 2:
8555                 return 4;
8556         case 3:
8557                 return 8;
8558         case 4:
8559                 return 3;
8560         case 5:
8561                 return 6;
8562         case 6:
8563                 return 10;
8564         case 7:
8565                 return 12;
8566         case 8:
8567                 return 16;
8568         }
8569 }
8570
8571 struct dce8_wm_params {
8572         u32 dram_channels; /* number of dram channels */
8573         u32 yclk;          /* bandwidth per dram data pin in kHz */
8574         u32 sclk;          /* engine clock in kHz */
8575         u32 disp_clk;      /* display clock in kHz */
8576         u32 src_width;     /* viewport width */
8577         u32 active_time;   /* active display time in ns */
8578         u32 blank_time;    /* blank time in ns */
8579         bool interlaced;    /* mode is interlaced */
8580         fixed20_12 vsc;    /* vertical scale ratio */
8581         u32 num_heads;     /* number of active crtcs */
8582         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8583         u32 lb_size;       /* line buffer allocated to pipe */
8584         u32 vtaps;         /* vertical scaler taps */
8585 };
8586
8587 /**
8588  * dce8_dram_bandwidth - get the dram bandwidth
8589  *
8590  * @wm: watermark calculation data
8591  *
8592  * Calculate the raw dram bandwidth (CIK).
8593  * Used for display watermark bandwidth calculations
8594  * Returns the dram bandwidth in MBytes/s
8595  */
8596 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8597 {
8598         /* Calculate raw DRAM Bandwidth */
8599         fixed20_12 dram_efficiency; /* 0.7 */
8600         fixed20_12 yclk, dram_channels, bandwidth;
8601         fixed20_12 a;
8602
8603         a.full = dfixed_const(1000);
8604         yclk.full = dfixed_const(wm->yclk);
8605         yclk.full = dfixed_div(yclk, a);
8606         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8607         a.full = dfixed_const(10);
8608         dram_efficiency.full = dfixed_const(7);
8609         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8610         bandwidth.full = dfixed_mul(dram_channels, yclk);
8611         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8612
8613         return dfixed_trunc(bandwidth);
8614 }
8615
8616 /**
8617  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8618  *
8619  * @wm: watermark calculation data
8620  *
8621  * Calculate the dram bandwidth used for display (CIK).
8622  * Used for display watermark bandwidth calculations
8623  * Returns the dram bandwidth for display in MBytes/s
8624  */
8625 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8626 {
8627         /* Calculate DRAM Bandwidth and the part allocated to display. */
8628         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8629         fixed20_12 yclk, dram_channels, bandwidth;
8630         fixed20_12 a;
8631
8632         a.full = dfixed_const(1000);
8633         yclk.full = dfixed_const(wm->yclk);
8634         yclk.full = dfixed_div(yclk, a);
8635         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8636         a.full = dfixed_const(10);
8637         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8638         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8639         bandwidth.full = dfixed_mul(dram_channels, yclk);
8640         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8641
8642         return dfixed_trunc(bandwidth);
8643 }
8644
8645 /**
8646  * dce8_data_return_bandwidth - get the data return bandwidth
8647  *
8648  * @wm: watermark calculation data
8649  *
8650  * Calculate the data return bandwidth used for display (CIK).
8651  * Used for display watermark bandwidth calculations
8652  * Returns the data return bandwidth in MBytes/s
8653  */
8654 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8655 {
8656         /* Calculate the display Data return Bandwidth */
8657         fixed20_12 return_efficiency; /* 0.8 */
8658         fixed20_12 sclk, bandwidth;
8659         fixed20_12 a;
8660
8661         a.full = dfixed_const(1000);
8662         sclk.full = dfixed_const(wm->sclk);
8663         sclk.full = dfixed_div(sclk, a);
8664         a.full = dfixed_const(10);
8665         return_efficiency.full = dfixed_const(8);
8666         return_efficiency.full = dfixed_div(return_efficiency, a);
8667         a.full = dfixed_const(32);
8668         bandwidth.full = dfixed_mul(a, sclk);
8669         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8670
8671         return dfixed_trunc(bandwidth);
8672 }
8673
8674 /**
8675  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8676  *
8677  * @wm: watermark calculation data
8678  *
8679  * Calculate the dmif bandwidth used for display (CIK).
8680  * Used for display watermark bandwidth calculations
8681  * Returns the dmif bandwidth in MBytes/s
8682  */
8683 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8684 {
8685         /* Calculate the DMIF Request Bandwidth */
8686         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8687         fixed20_12 disp_clk, bandwidth;
8688         fixed20_12 a, b;
8689
8690         a.full = dfixed_const(1000);
8691         disp_clk.full = dfixed_const(wm->disp_clk);
8692         disp_clk.full = dfixed_div(disp_clk, a);
8693         a.full = dfixed_const(32);
8694         b.full = dfixed_mul(a, disp_clk);
8695
8696         a.full = dfixed_const(10);
8697         disp_clk_request_efficiency.full = dfixed_const(8);
8698         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8699
8700         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8701
8702         return dfixed_trunc(bandwidth);
8703 }
8704
8705 /**
8706  * dce8_available_bandwidth - get the min available bandwidth
8707  *
8708  * @wm: watermark calculation data
8709  *
8710  * Calculate the min available bandwidth used for display (CIK).
8711  * Used for display watermark bandwidth calculations
8712  * Returns the min available bandwidth in MBytes/s
8713  */
8714 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8715 {
8716         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8717         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8718         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8719         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8720
8721         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8722 }
8723
8724 /**
8725  * dce8_average_bandwidth - get the average available bandwidth
8726  *
8727  * @wm: watermark calculation data
8728  *
8729  * Calculate the average available bandwidth used for display (CIK).
8730  * Used for display watermark bandwidth calculations
8731  * Returns the average available bandwidth in MBytes/s
8732  */
8733 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8734 {
8735         /* Calculate the display mode Average Bandwidth
8736          * DisplayMode should contain the source and destination dimensions,
8737          * timing, etc.
8738          */
8739         fixed20_12 bpp;
8740         fixed20_12 line_time;
8741         fixed20_12 src_width;
8742         fixed20_12 bandwidth;
8743         fixed20_12 a;
8744
8745         a.full = dfixed_const(1000);
8746         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8747         line_time.full = dfixed_div(line_time, a);
8748         bpp.full = dfixed_const(wm->bytes_per_pixel);
8749         src_width.full = dfixed_const(wm->src_width);
8750         bandwidth.full = dfixed_mul(src_width, bpp);
8751         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8752         bandwidth.full = dfixed_div(bandwidth, line_time);
8753
8754         return dfixed_trunc(bandwidth);
8755 }
8756
8757 /**
8758  * dce8_latency_watermark - get the latency watermark
8759  *
8760  * @wm: watermark calculation data
8761  *
8762  * Calculate the latency watermark (CIK).
8763  * Used for display watermark bandwidth calculations
8764  * Returns the latency watermark in ns
8765  */
8766 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8767 {
8768         /* First calculate the latency in ns */
8769         u32 mc_latency = 2000; /* 2000 ns. */
8770         u32 available_bandwidth = dce8_available_bandwidth(wm);
8771         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8772         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8773         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8774         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8775                 (wm->num_heads * cursor_line_pair_return_time);
8776         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8777         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8778         u32 tmp, dmif_size = 12288;
8779         fixed20_12 a, b, c;
8780
8781         if (wm->num_heads == 0)
8782                 return 0;
8783
8784         a.full = dfixed_const(2);
8785         b.full = dfixed_const(1);
8786         if ((wm->vsc.full > a.full) ||
8787             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8788             (wm->vtaps >= 5) ||
8789             ((wm->vsc.full >= a.full) && wm->interlaced))
8790                 max_src_lines_per_dst_line = 4;
8791         else
8792                 max_src_lines_per_dst_line = 2;
8793
8794         a.full = dfixed_const(available_bandwidth);
8795         b.full = dfixed_const(wm->num_heads);
8796         a.full = dfixed_div(a, b);
8797
8798         b.full = dfixed_const(mc_latency + 512);
8799         c.full = dfixed_const(wm->disp_clk);
8800         b.full = dfixed_div(b, c);
8801
8802         c.full = dfixed_const(dmif_size);
8803         b.full = dfixed_div(c, b);
8804
8805         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8806
8807         b.full = dfixed_const(1000);
8808         c.full = dfixed_const(wm->disp_clk);
8809         b.full = dfixed_div(c, b);
8810         c.full = dfixed_const(wm->bytes_per_pixel);
8811         b.full = dfixed_mul(b, c);
8812
8813         lb_fill_bw = min(tmp, dfixed_trunc(b));
8814
8815         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8816         b.full = dfixed_const(1000);
8817         c.full = dfixed_const(lb_fill_bw);
8818         b.full = dfixed_div(c, b);
8819         a.full = dfixed_div(a, b);
8820         line_fill_time = dfixed_trunc(a);
8821
8822         if (line_fill_time < wm->active_time)
8823                 return latency;
8824         else
8825                 return latency + (line_fill_time - wm->active_time);
8826
8827 }
8828
8829 /**
8830  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8831  * average and available dram bandwidth
8832  *
8833  * @wm: watermark calculation data
8834  *
8835  * Check if the display average bandwidth fits in the display
8836  * dram bandwidth (CIK).
8837  * Used for display watermark bandwidth calculations
8838  * Returns true if the display fits, false if not.
8839  */
8840 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8841 {
8842         if (dce8_average_bandwidth(wm) <=
8843             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8844                 return true;
8845         else
8846                 return false;
8847 }
8848
8849 /**
8850  * dce8_average_bandwidth_vs_available_bandwidth - check
8851  * average and available bandwidth
8852  *
8853  * @wm: watermark calculation data
8854  *
8855  * Check if the display average bandwidth fits in the display
8856  * available bandwidth (CIK).
8857  * Used for display watermark bandwidth calculations
8858  * Returns true if the display fits, false if not.
8859  */
8860 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8861 {
8862         if (dce8_average_bandwidth(wm) <=
8863             (dce8_available_bandwidth(wm) / wm->num_heads))
8864                 return true;
8865         else
8866                 return false;
8867 }
8868
8869 /**
8870  * dce8_check_latency_hiding - check latency hiding
8871  *
8872  * @wm: watermark calculation data
8873  *
8874  * Check latency hiding (CIK).
8875  * Used for display watermark bandwidth calculations
8876  * Returns true if the display fits, false if not.
8877  */
8878 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8879 {
8880         u32 lb_partitions = wm->lb_size / wm->src_width;
8881         u32 line_time = wm->active_time + wm->blank_time;
8882         u32 latency_tolerant_lines;
8883         u32 latency_hiding;
8884         fixed20_12 a;
8885
8886         a.full = dfixed_const(1);
8887         if (wm->vsc.full > a.full)
8888                 latency_tolerant_lines = 1;
8889         else {
8890                 if (lb_partitions <= (wm->vtaps + 1))
8891                         latency_tolerant_lines = 1;
8892                 else
8893                         latency_tolerant_lines = 2;
8894         }
8895
8896         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8897
8898         if (dce8_latency_watermark(wm) <= latency_hiding)
8899                 return true;
8900         else
8901                 return false;
8902 }
8903
8904 /**
8905  * dce8_program_watermarks - program display watermarks
8906  *
8907  * @rdev: radeon_device pointer
8908  * @radeon_crtc: the selected display controller
8909  * @lb_size: line buffer size
8910  * @num_heads: number of display controllers in use
8911  *
8912  * Calculate and program the display watermarks for the
8913  * selected display controller (CIK).
8914  */
8915 static void dce8_program_watermarks(struct radeon_device *rdev,
8916                                     struct radeon_crtc *radeon_crtc,
8917                                     u32 lb_size, u32 num_heads)
8918 {
8919         struct drm_display_mode *mode = &radeon_crtc->base.mode;
8920         struct dce8_wm_params wm_low, wm_high;
8921         u32 pixel_period;
8922         u32 line_time = 0;
8923         u32 latency_watermark_a = 0, latency_watermark_b = 0;
8924         u32 tmp, wm_mask;
8925
8926         if (radeon_crtc->base.enabled && num_heads && mode) {
8927                 pixel_period = 1000000 / (u32)mode->clock;
8928                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8929
8930                 /* watermark for high clocks */
8931                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8932                     rdev->pm.dpm_enabled) {
8933                         wm_high.yclk =
8934                                 radeon_dpm_get_mclk(rdev, false) * 10;
8935                         wm_high.sclk =
8936                                 radeon_dpm_get_sclk(rdev, false) * 10;
8937                 } else {
8938                         wm_high.yclk = rdev->pm.current_mclk * 10;
8939                         wm_high.sclk = rdev->pm.current_sclk * 10;
8940                 }
8941
8942                 wm_high.disp_clk = mode->clock;
8943                 wm_high.src_width = mode->crtc_hdisplay;
8944                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8945                 wm_high.blank_time = line_time - wm_high.active_time;
8946                 wm_high.interlaced = false;
8947                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8948                         wm_high.interlaced = true;
8949                 wm_high.vsc = radeon_crtc->vsc;
8950                 wm_high.vtaps = 1;
8951                 if (radeon_crtc->rmx_type != RMX_OFF)
8952                         wm_high.vtaps = 2;
8953                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8954                 wm_high.lb_size = lb_size;
8955                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8956                 wm_high.num_heads = num_heads;
8957
8958                 /* set for high clocks */
8959                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8960
8961                 /* possibly force display priority to high */
8962                 /* should really do this at mode validation time... */
8963                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8964                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8965                     !dce8_check_latency_hiding(&wm_high) ||
8966                     (rdev->disp_priority == 2)) {
8967                         DRM_DEBUG_KMS("force priority to high\n");
8968                 }
8969
8970                 /* watermark for low clocks */
8971                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8972                     rdev->pm.dpm_enabled) {
8973                         wm_low.yclk =
8974                                 radeon_dpm_get_mclk(rdev, true) * 10;
8975                         wm_low.sclk =
8976                                 radeon_dpm_get_sclk(rdev, true) * 10;
8977                 } else {
8978                         wm_low.yclk = rdev->pm.current_mclk * 10;
8979                         wm_low.sclk = rdev->pm.current_sclk * 10;
8980                 }
8981
8982                 wm_low.disp_clk = mode->clock;
8983                 wm_low.src_width = mode->crtc_hdisplay;
8984                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8985                 wm_low.blank_time = line_time - wm_low.active_time;
8986                 wm_low.interlaced = false;
8987                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8988                         wm_low.interlaced = true;
8989                 wm_low.vsc = radeon_crtc->vsc;
8990                 wm_low.vtaps = 1;
8991                 if (radeon_crtc->rmx_type != RMX_OFF)
8992                         wm_low.vtaps = 2;
8993                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8994                 wm_low.lb_size = lb_size;
8995                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8996                 wm_low.num_heads = num_heads;
8997
8998                 /* set for low clocks */
8999                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9000
9001                 /* possibly force display priority to high */
9002                 /* should really do this at mode validation time... */
9003                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9004                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9005                     !dce8_check_latency_hiding(&wm_low) ||
9006                     (rdev->disp_priority == 2)) {
9007                         DRM_DEBUG_KMS("force priority to high\n");
9008                 }
9009         }
9010
9011         /* select wm A */
9012         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9013         tmp = wm_mask;
9014         tmp &= ~LATENCY_WATERMARK_MASK(3);
9015         tmp |= LATENCY_WATERMARK_MASK(1);
9016         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9017         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9018                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9019                 LATENCY_HIGH_WATERMARK(line_time)));
9020         /* select wm B */
9021         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9022         tmp &= ~LATENCY_WATERMARK_MASK(3);
9023         tmp |= LATENCY_WATERMARK_MASK(2);
9024         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9025         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9026                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9027                 LATENCY_HIGH_WATERMARK(line_time)));
9028         /* restore original selection */
9029         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9030
9031         /* save values for DPM */
9032         radeon_crtc->line_time = line_time;
9033         radeon_crtc->wm_high = latency_watermark_a;
9034         radeon_crtc->wm_low = latency_watermark_b;
9035 }
9036
9037 /**
9038  * dce8_bandwidth_update - program display watermarks
9039  *
9040  * @rdev: radeon_device pointer
9041  *
9042  * Calculate and program the display watermarks and line
9043  * buffer allocation (CIK).
9044  */
9045 void dce8_bandwidth_update(struct radeon_device *rdev)
9046 {
9047         struct drm_display_mode *mode = NULL;
9048         u32 num_heads = 0, lb_size;
9049         int i;
9050
9051         radeon_update_display_priority(rdev);
9052
9053         for (i = 0; i < rdev->num_crtc; i++) {
9054                 if (rdev->mode_info.crtcs[i]->base.enabled)
9055                         num_heads++;
9056         }
9057         for (i = 0; i < rdev->num_crtc; i++) {
9058                 mode = &rdev->mode_info.crtcs[i]->base.mode;
9059                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9060                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9061         }
9062 }
9063
9064 /**
9065  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9066  *
9067  * @rdev: radeon_device pointer
9068  *
9069  * Fetches a GPU clock counter snapshot (SI).
9070  * Returns the 64 bit clock counter snapshot.
9071  */
9072 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9073 {
9074         uint64_t clock;
9075
9076         mutex_lock(&rdev->gpu_clock_mutex);
9077         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9078         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9079                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9080         mutex_unlock(&rdev->gpu_clock_mutex);
9081         return clock;
9082 }
9083
9084 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9085                               u32 cntl_reg, u32 status_reg)
9086 {
9087         int r, i;
9088         struct atom_clock_dividers dividers;
9089         uint32_t tmp;
9090
9091         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9092                                            clock, false, &dividers);
9093         if (r)
9094                 return r;
9095
9096         tmp = RREG32_SMC(cntl_reg);
9097         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9098         tmp |= dividers.post_divider;
9099         WREG32_SMC(cntl_reg, tmp);
9100
9101         for (i = 0; i < 100; i++) {
9102                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9103                         break;
9104                 mdelay(10);
9105         }
9106         if (i == 100)
9107                 return -ETIMEDOUT;
9108
9109         return 0;
9110 }
9111
9112 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9113 {
9114         int r = 0;
9115
9116         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9117         if (r)
9118                 return r;
9119
9120         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9121         return r;
9122 }
9123
9124 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9125 {
9126         int r, i;
9127         struct atom_clock_dividers dividers;
9128         u32 tmp;
9129
9130         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9131                                            ecclk, false, &dividers);
9132         if (r)
9133                 return r;
9134
9135         for (i = 0; i < 100; i++) {
9136                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9137                         break;
9138                 mdelay(10);
9139         }
9140         if (i == 100)
9141                 return -ETIMEDOUT;
9142
9143         tmp = RREG32_SMC(CG_ECLK_CNTL);
9144         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9145         tmp |= dividers.post_divider;
9146         WREG32_SMC(CG_ECLK_CNTL, tmp);
9147
9148         for (i = 0; i < 100; i++) {
9149                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9150                         break;
9151                 mdelay(10);
9152         }
9153         if (i == 100)
9154                 return -ETIMEDOUT;
9155
9156         return 0;
9157 }
9158
9159 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9160 {
9161         struct pci_dev *root = rdev->pdev->bus->self;
9162         int bridge_pos, gpu_pos;
9163         u32 speed_cntl, mask, current_data_rate;
9164         int ret, i;
9165         u16 tmp16;
9166
9167         if (radeon_pcie_gen2 == 0)
9168                 return;
9169
9170         if (rdev->flags & RADEON_IS_IGP)
9171                 return;
9172
9173         if (!(rdev->flags & RADEON_IS_PCIE))
9174                 return;
9175
9176         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9177         if (ret != 0)
9178                 return;
9179
9180         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9181                 return;
9182
9183         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9184         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9185                 LC_CURRENT_DATA_RATE_SHIFT;
9186         if (mask & DRM_PCIE_SPEED_80) {
9187                 if (current_data_rate == 2) {
9188                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9189                         return;
9190                 }
9191                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9192         } else if (mask & DRM_PCIE_SPEED_50) {
9193                 if (current_data_rate == 1) {
9194                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9195                         return;
9196                 }
9197                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9198         }
9199
9200         bridge_pos = pci_pcie_cap(root);
9201         if (!bridge_pos)
9202                 return;
9203
9204         gpu_pos = pci_pcie_cap(rdev->pdev);
9205         if (!gpu_pos)
9206                 return;
9207
9208         if (mask & DRM_PCIE_SPEED_80) {
9209                 /* re-try equalization if gen3 is not already enabled */
9210                 if (current_data_rate != 2) {
9211                         u16 bridge_cfg, gpu_cfg;
9212                         u16 bridge_cfg2, gpu_cfg2;
9213                         u32 max_lw, current_lw, tmp;
9214
9215                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9216                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9217
9218                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9219                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9220
9221                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9222                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9223
9224                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9225                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9226                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9227
9228                         if (current_lw < max_lw) {
9229                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9230                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9231                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9232                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9233                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9234                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9235                                 }
9236                         }
9237
9238                         for (i = 0; i < 10; i++) {
9239                                 /* check status */
9240                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9241                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9242                                         break;
9243
9244                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9245                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9246
9247                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9248                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9249
9250                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9251                                 tmp |= LC_SET_QUIESCE;
9252                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9253
9254                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9255                                 tmp |= LC_REDO_EQ;
9256                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9257
9258                                 mdelay(100);
9259
9260                                 /* linkctl */
9261                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9262                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9263                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9264                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9265
9266                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9267                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9268                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9269                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9270
9271                                 /* linkctl2 */
9272                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9273                                 tmp16 &= ~((1 << 4) | (7 << 9));
9274                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9275                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9276
9277                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9278                                 tmp16 &= ~((1 << 4) | (7 << 9));
9279                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9280                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9281
9282                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9283                                 tmp &= ~LC_SET_QUIESCE;
9284                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9285                         }
9286                 }
9287         }
9288
9289         /* set the link speed */
9290         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9291         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9292         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9293
9294         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9295         tmp16 &= ~0xf;
9296         if (mask & DRM_PCIE_SPEED_80)
9297                 tmp16 |= 3; /* gen3 */
9298         else if (mask & DRM_PCIE_SPEED_50)
9299                 tmp16 |= 2; /* gen2 */
9300         else
9301                 tmp16 |= 1; /* gen1 */
9302         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9303
9304         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9305         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9306         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9307
9308         for (i = 0; i < rdev->usec_timeout; i++) {
9309                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9310                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9311                         break;
9312                 udelay(1);
9313         }
9314 }
9315
9316 static void cik_program_aspm(struct radeon_device *rdev)
9317 {
9318         u32 data, orig;
9319         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9320         bool disable_clkreq = false;
9321
9322         if (radeon_aspm == 0)
9323                 return;
9324
9325         /* XXX double check IGPs */
9326         if (rdev->flags & RADEON_IS_IGP)
9327                 return;
9328
9329         if (!(rdev->flags & RADEON_IS_PCIE))
9330                 return;
9331
9332         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9333         data &= ~LC_XMIT_N_FTS_MASK;
9334         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9335         if (orig != data)
9336                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9337
9338         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9339         data |= LC_GO_TO_RECOVERY;
9340         if (orig != data)
9341                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9342
9343         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9344         data |= P_IGNORE_EDB_ERR;
9345         if (orig != data)
9346                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9347
9348         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9349         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9350         data |= LC_PMI_TO_L1_DIS;
9351         if (!disable_l0s)
9352                 data |= LC_L0S_INACTIVITY(7);
9353
9354         if (!disable_l1) {
9355                 data |= LC_L1_INACTIVITY(7);
9356                 data &= ~LC_PMI_TO_L1_DIS;
9357                 if (orig != data)
9358                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9359
9360                 if (!disable_plloff_in_l1) {
9361                         bool clk_req_support;
9362
9363                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9364                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9365                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9366                         if (orig != data)
9367                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9368
9369                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9370                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9371                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9372                         if (orig != data)
9373                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9374
9375                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9376                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9377                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9378                         if (orig != data)
9379                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9380
9381                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9382                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9383                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9384                         if (orig != data)
9385                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9386
9387                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9388                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9389                         data |= LC_DYN_LANES_PWR_STATE(3);
9390                         if (orig != data)
9391                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9392
9393                         if (!disable_clkreq) {
9394                                 struct pci_dev *root = rdev->pdev->bus->self;
9395                                 u32 lnkcap;
9396
9397                                 clk_req_support = false;
9398                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9399                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9400                                         clk_req_support = true;
9401                         } else {
9402                                 clk_req_support = false;
9403                         }
9404
9405                         if (clk_req_support) {
9406                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9407                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9408                                 if (orig != data)
9409                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9410
9411                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9412                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9413                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9414                                 if (orig != data)
9415                                         WREG32_SMC(THM_CLK_CNTL, data);
9416
9417                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9418                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9419                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9420                                 if (orig != data)
9421                                         WREG32_SMC(MISC_CLK_CTRL, data);
9422
9423                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9424                                 data &= ~BCLK_AS_XCLK;
9425                                 if (orig != data)
9426                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9427
9428                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9429                                 data &= ~FORCE_BIF_REFCLK_EN;
9430                                 if (orig != data)
9431                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9432
9433                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9434                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9435                                 data |= MPLL_CLKOUT_SEL(4);
9436                                 if (orig != data)
9437                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9438                         }
9439                 }
9440         } else {
9441                 if (orig != data)
9442                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9443         }
9444
9445         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9446         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9447         if (orig != data)
9448                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9449
9450         if (!disable_l0s) {
9451                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9452                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9453                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9454                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9455                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9456                                 data &= ~LC_L0S_INACTIVITY_MASK;
9457                                 if (orig != data)
9458                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9459                         }
9460                 }
9461         }
9462 }