drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  66 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
  67 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
  68 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
  69 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
  70 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
  71 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
  72
  73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  75 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  76 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  77 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  78 extern void sumo_rlc_fini(struct radeon_device *rdev);
  79 extern int sumo_rlc_init(struct radeon_device *rdev);
  80 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  81 extern void si_rlc_reset(struct radeon_device *rdev);
  82 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
  83 extern int cik_sdma_resume(struct radeon_device *rdev);
  84 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
  85 extern void cik_sdma_fini(struct radeon_device *rdev);
  86 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
  87 static void cik_rlc_stop(struct radeon_device *rdev);
  88 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  89 static void cik_program_aspm(struct radeon_device *rdev);
  90 static void cik_init_pg(struct radeon_device *rdev);
  91 static void cik_init_cg(struct radeon_device *rdev);
  92 static void cik_fini_pg(struct radeon_device *rdev);
  93 static void cik_fini_cg(struct radeon_device *rdev);
  94 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
  95                                           bool enable);
  96
  97 /* get temperature in millidegrees */
  98 int ci_get_temp(struct radeon_device *rdev)
  99 {
 100         u32 temp;
 101         int actual_temp = 0;
 102
 103         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 104                 CTF_TEMP_SHIFT;
 105
 106         if (temp & 0x200)
 107                 actual_temp = 255;
 108         else
 109                 actual_temp = temp & 0x1ff;
 110
 111         actual_temp = actual_temp * 1000;
 112
 113         return actual_temp;
 114 }
 115
 116 /* get temperature in millidegrees */
 117 int kv_get_temp(struct radeon_device *rdev)
 118 {
 119         u32 temp;
 120         int actual_temp = 0;
 121
 122         temp = RREG32_SMC(0xC0300E0C);
 123
 124         if (temp)
 125                 actual_temp = (temp / 8) - 49;
 126         else
 127                 actual_temp = 0;
 128
 129         actual_temp = actual_temp * 1000;
 130
 131         return actual_temp;
 132 }
 133
 134 /*
 135  * Indirect registers accessor
 136  */
 137 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 138 {
 139         unsigned long flags;
 140         u32 r;
 141
 142         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 143         WREG32(PCIE_INDEX, reg);
 144         (void)RREG32(PCIE_INDEX);
 145         r = RREG32(PCIE_DATA);
 146         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 147         return r;
 148 }
 149
 150 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 151 {
 152         unsigned long flags;
 153
 154         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 155         WREG32(PCIE_INDEX, reg);
 156         (void)RREG32(PCIE_INDEX);
 157         WREG32(PCIE_DATA, v);
 158         (void)RREG32(PCIE_DATA);
 159         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 160 }
 161
 162 static const u32 spectre_rlc_save_restore_register_list[] =
 163 {
 164         (0x0e00 << 16) | (0xc12c >> 2),
 165         0x00000000,
 166         (0x0e00 << 16) | (0xc140 >> 2),
 167         0x00000000,
 168         (0x0e00 << 16) | (0xc150 >> 2),
 169         0x00000000,
 170         (0x0e00 << 16) | (0xc15c >> 2),
 171         0x00000000,
 172         (0x0e00 << 16) | (0xc168 >> 2),
 173         0x00000000,
 174         (0x0e00 << 16) | (0xc170 >> 2),
 175         0x00000000,
 176         (0x0e00 << 16) | (0xc178 >> 2),
 177         0x00000000,
 178         (0x0e00 << 16) | (0xc204 >> 2),
 179         0x00000000,
 180         (0x0e00 << 16) | (0xc2b4 >> 2),
 181         0x00000000,
 182         (0x0e00 << 16) | (0xc2b8 >> 2),
 183         0x00000000,
 184         (0x0e00 << 16) | (0xc2bc >> 2),
 185         0x00000000,
 186         (0x0e00 << 16) | (0xc2c0 >> 2),
 187         0x00000000,
 188         (0x0e00 << 16) | (0x8228 >> 2),
 189         0x00000000,
 190         (0x0e00 << 16) | (0x829c >> 2),
 191         0x00000000,
 192         (0x0e00 << 16) | (0x869c >> 2),
 193         0x00000000,
 194         (0x0600 << 16) | (0x98f4 >> 2),
 195         0x00000000,
 196         (0x0e00 << 16) | (0x98f8 >> 2),
 197         0x00000000,
 198         (0x0e00 << 16) | (0x9900 >> 2),
 199         0x00000000,
 200         (0x0e00 << 16) | (0xc260 >> 2),
 201         0x00000000,
 202         (0x0e00 << 16) | (0x90e8 >> 2),
 203         0x00000000,
 204         (0x0e00 << 16) | (0x3c000 >> 2),
 205         0x00000000,
 206         (0x0e00 << 16) | (0x3c00c >> 2),
 207         0x00000000,
 208         (0x0e00 << 16) | (0x8c1c >> 2),
 209         0x00000000,
 210         (0x0e00 << 16) | (0x9700 >> 2),
 211         0x00000000,
 212         (0x0e00 << 16) | (0xcd20 >> 2),
 213         0x00000000,
 214         (0x4e00 << 16) | (0xcd20 >> 2),
 215         0x00000000,
 216         (0x5e00 << 16) | (0xcd20 >> 2),
 217         0x00000000,
 218         (0x6e00 << 16) | (0xcd20 >> 2),
 219         0x00000000,
 220         (0x7e00 << 16) | (0xcd20 >> 2),
 221         0x00000000,
 222         (0x8e00 << 16) | (0xcd20 >> 2),
 223         0x00000000,
 224         (0x9e00 << 16) | (0xcd20 >> 2),
 225         0x00000000,
 226         (0xae00 << 16) | (0xcd20 >> 2),
 227         0x00000000,
 228         (0xbe00 << 16) | (0xcd20 >> 2),
 229         0x00000000,
 230         (0x0e00 << 16) | (0x89bc >> 2),
 231         0x00000000,
 232         (0x0e00 << 16) | (0x8900 >> 2),
 233         0x00000000,
 234         0x3,
 235         (0x0e00 << 16) | (0xc130 >> 2),
 236         0x00000000,
 237         (0x0e00 << 16) | (0xc134 >> 2),
 238         0x00000000,
 239         (0x0e00 << 16) | (0xc1fc >> 2),
 240         0x00000000,
 241         (0x0e00 << 16) | (0xc208 >> 2),
 242         0x00000000,
 243         (0x0e00 << 16) | (0xc264 >> 2),
 244         0x00000000,
 245         (0x0e00 << 16) | (0xc268 >> 2),
 246         0x00000000,
 247         (0x0e00 << 16) | (0xc26c >> 2),
 248         0x00000000,
 249         (0x0e00 << 16) | (0xc270 >> 2),
 250         0x00000000,
 251         (0x0e00 << 16) | (0xc274 >> 2),
 252         0x00000000,
 253         (0x0e00 << 16) | (0xc278 >> 2),
 254         0x00000000,
 255         (0x0e00 << 16) | (0xc27c >> 2),
 256         0x00000000,
 257         (0x0e00 << 16) | (0xc280 >> 2),
 258         0x00000000,
 259         (0x0e00 << 16) | (0xc284 >> 2),
 260         0x00000000,
 261         (0x0e00 << 16) | (0xc288 >> 2),
 262         0x00000000,
 263         (0x0e00 << 16) | (0xc28c >> 2),
 264         0x00000000,
 265         (0x0e00 << 16) | (0xc290 >> 2),
 266         0x00000000,
 267         (0x0e00 << 16) | (0xc294 >> 2),
 268         0x00000000,
 269         (0x0e00 << 16) | (0xc298 >> 2),
 270         0x00000000,
 271         (0x0e00 << 16) | (0xc29c >> 2),
 272         0x00000000,
 273         (0x0e00 << 16) | (0xc2a0 >> 2),
 274         0x00000000,
 275         (0x0e00 << 16) | (0xc2a4 >> 2),
 276         0x00000000,
 277         (0x0e00 << 16) | (0xc2a8 >> 2),
 278         0x00000000,
 279         (0x0e00 << 16) | (0xc2ac  >> 2),
 280         0x00000000,
 281         (0x0e00 << 16) | (0xc2b0 >> 2),
 282         0x00000000,
 283         (0x0e00 << 16) | (0x301d0 >> 2),
 284         0x00000000,
 285         (0x0e00 << 16) | (0x30238 >> 2),
 286         0x00000000,
 287         (0x0e00 << 16) | (0x30250 >> 2),
 288         0x00000000,
 289         (0x0e00 << 16) | (0x30254 >> 2),
 290         0x00000000,
 291         (0x0e00 << 16) | (0x30258 >> 2),
 292         0x00000000,
 293         (0x0e00 << 16) | (0x3025c >> 2),
 294         0x00000000,
 295         (0x4e00 << 16) | (0xc900 >> 2),
 296         0x00000000,
 297         (0x5e00 << 16) | (0xc900 >> 2),
 298         0x00000000,
 299         (0x6e00 << 16) | (0xc900 >> 2),
 300         0x00000000,
 301         (0x7e00 << 16) | (0xc900 >> 2),
 302         0x00000000,
 303         (0x8e00 << 16) | (0xc900 >> 2),
 304         0x00000000,
 305         (0x9e00 << 16) | (0xc900 >> 2),
 306         0x00000000,
 307         (0xae00 << 16) | (0xc900 >> 2),
 308         0x00000000,
 309         (0xbe00 << 16) | (0xc900 >> 2),
 310         0x00000000,
 311         (0x4e00 << 16) | (0xc904 >> 2),
 312         0x00000000,
 313         (0x5e00 << 16) | (0xc904 >> 2),
 314         0x00000000,
 315         (0x6e00 << 16) | (0xc904 >> 2),
 316         0x00000000,
 317         (0x7e00 << 16) | (0xc904 >> 2),
 318         0x00000000,
 319         (0x8e00 << 16) | (0xc904 >> 2),
 320         0x00000000,
 321         (0x9e00 << 16) | (0xc904 >> 2),
 322         0x00000000,
 323         (0xae00 << 16) | (0xc904 >> 2),
 324         0x00000000,
 325         (0xbe00 << 16) | (0xc904 >> 2),
 326         0x00000000,
 327         (0x4e00 << 16) | (0xc908 >> 2),
 328         0x00000000,
 329         (0x5e00 << 16) | (0xc908 >> 2),
 330         0x00000000,
 331         (0x6e00 << 16) | (0xc908 >> 2),
 332         0x00000000,
 333         (0x7e00 << 16) | (0xc908 >> 2),
 334         0x00000000,
 335         (0x8e00 << 16) | (0xc908 >> 2),
 336         0x00000000,
 337         (0x9e00 << 16) | (0xc908 >> 2),
 338         0x00000000,
 339         (0xae00 << 16) | (0xc908 >> 2),
 340         0x00000000,
 341         (0xbe00 << 16) | (0xc908 >> 2),
 342         0x00000000,
 343         (0x4e00 << 16) | (0xc90c >> 2),
 344         0x00000000,
 345         (0x5e00 << 16) | (0xc90c >> 2),
 346         0x00000000,
 347         (0x6e00 << 16) | (0xc90c >> 2),
 348         0x00000000,
 349         (0x7e00 << 16) | (0xc90c >> 2),
 350         0x00000000,
 351         (0x8e00 << 16) | (0xc90c >> 2),
 352         0x00000000,
 353         (0x9e00 << 16) | (0xc90c >> 2),
 354         0x00000000,
 355         (0xae00 << 16) | (0xc90c >> 2),
 356         0x00000000,
 357         (0xbe00 << 16) | (0xc90c >> 2),
 358         0x00000000,
 359         (0x4e00 << 16) | (0xc910 >> 2),
 360         0x00000000,
 361         (0x5e00 << 16) | (0xc910 >> 2),
 362         0x00000000,
 363         (0x6e00 << 16) | (0xc910 >> 2),
 364         0x00000000,
 365         (0x7e00 << 16) | (0xc910 >> 2),
 366         0x00000000,
 367         (0x8e00 << 16) | (0xc910 >> 2),
 368         0x00000000,
 369         (0x9e00 << 16) | (0xc910 >> 2),
 370         0x00000000,
 371         (0xae00 << 16) | (0xc910 >> 2),
 372         0x00000000,
 373         (0xbe00 << 16) | (0xc910 >> 2),
 374         0x00000000,
 375         (0x0e00 << 16) | (0xc99c >> 2),
 376         0x00000000,
 377         (0x0e00 << 16) | (0x9834 >> 2),
 378         0x00000000,
 379         (0x0000 << 16) | (0x30f00 >> 2),
 380         0x00000000,
 381         (0x0001 << 16) | (0x30f00 >> 2),
 382         0x00000000,
 383         (0x0000 << 16) | (0x30f04 >> 2),
 384         0x00000000,
 385         (0x0001 << 16) | (0x30f04 >> 2),
 386         0x00000000,
 387         (0x0000 << 16) | (0x30f08 >> 2),
 388         0x00000000,
 389         (0x0001 << 16) | (0x30f08 >> 2),
 390         0x00000000,
 391         (0x0000 << 16) | (0x30f0c >> 2),
 392         0x00000000,
 393         (0x0001 << 16) | (0x30f0c >> 2),
 394         0x00000000,
 395         (0x0600 << 16) | (0x9b7c >> 2),
 396         0x00000000,
 397         (0x0e00 << 16) | (0x8a14 >> 2),
 398         0x00000000,
 399         (0x0e00 << 16) | (0x8a18 >> 2),
 400         0x00000000,
 401         (0x0600 << 16) | (0x30a00 >> 2),
 402         0x00000000,
 403         (0x0e00 << 16) | (0x8bf0 >> 2),
 404         0x00000000,
 405         (0x0e00 << 16) | (0x8bcc >> 2),
 406         0x00000000,
 407         (0x0e00 << 16) | (0x8b24 >> 2),
 408         0x00000000,
 409         (0x0e00 << 16) | (0x30a04 >> 2),
 410         0x00000000,
 411         (0x0600 << 16) | (0x30a10 >> 2),
 412         0x00000000,
 413         (0x0600 << 16) | (0x30a14 >> 2),
 414         0x00000000,
 415         (0x0600 << 16) | (0x30a18 >> 2),
 416         0x00000000,
 417         (0x0600 << 16) | (0x30a2c >> 2),
 418         0x00000000,
 419         (0x0e00 << 16) | (0xc700 >> 2),
 420         0x00000000,
 421         (0x0e00 << 16) | (0xc704 >> 2),
 422         0x00000000,
 423         (0x0e00 << 16) | (0xc708 >> 2),
 424         0x00000000,
 425         (0x0e00 << 16) | (0xc768 >> 2),
 426         0x00000000,
 427         (0x0400 << 16) | (0xc770 >> 2),
 428         0x00000000,
 429         (0x0400 << 16) | (0xc774 >> 2),
 430         0x00000000,
 431         (0x0400 << 16) | (0xc778 >> 2),
 432         0x00000000,
 433         (0x0400 << 16) | (0xc77c >> 2),
 434         0x00000000,
 435         (0x0400 << 16) | (0xc780 >> 2),
 436         0x00000000,
 437         (0x0400 << 16) | (0xc784 >> 2),
 438         0x00000000,
 439         (0x0400 << 16) | (0xc788 >> 2),
 440         0x00000000,
 441         (0x0400 << 16) | (0xc78c >> 2),
 442         0x00000000,
 443         (0x0400 << 16) | (0xc798 >> 2),
 444         0x00000000,
 445         (0x0400 << 16) | (0xc79c >> 2),
 446         0x00000000,
 447         (0x0400 << 16) | (0xc7a0 >> 2),
 448         0x00000000,
 449         (0x0400 << 16) | (0xc7a4 >> 2),
 450         0x00000000,
 451         (0x0400 << 16) | (0xc7a8 >> 2),
 452         0x00000000,
 453         (0x0400 << 16) | (0xc7ac >> 2),
 454         0x00000000,
 455         (0x0400 << 16) | (0xc7b0 >> 2),
 456         0x00000000,
 457         (0x0400 << 16) | (0xc7b4 >> 2),
 458         0x00000000,
 459         (0x0e00 << 16) | (0x9100 >> 2),
 460         0x00000000,
 461         (0x0e00 << 16) | (0x3c010 >> 2),
 462         0x00000000,
 463         (0x0e00 << 16) | (0x92a8 >> 2),
 464         0x00000000,
 465         (0x0e00 << 16) | (0x92ac >> 2),
 466         0x00000000,
 467         (0x0e00 << 16) | (0x92b4 >> 2),
 468         0x00000000,
 469         (0x0e00 << 16) | (0x92b8 >> 2),
 470         0x00000000,
 471         (0x0e00 << 16) | (0x92bc >> 2),
 472         0x00000000,
 473         (0x0e00 << 16) | (0x92c0 >> 2),
 474         0x00000000,
 475         (0x0e00 << 16) | (0x92c4 >> 2),
 476         0x00000000,
 477         (0x0e00 << 16) | (0x92c8 >> 2),
 478         0x00000000,
 479         (0x0e00 << 16) | (0x92cc >> 2),
 480         0x00000000,
 481         (0x0e00 << 16) | (0x92d0 >> 2),
 482         0x00000000,
 483         (0x0e00 << 16) | (0x8c00 >> 2),
 484         0x00000000,
 485         (0x0e00 << 16) | (0x8c04 >> 2),
 486         0x00000000,
 487         (0x0e00 << 16) | (0x8c20 >> 2),
 488         0x00000000,
 489         (0x0e00 << 16) | (0x8c38 >> 2),
 490         0x00000000,
 491         (0x0e00 << 16) | (0x8c3c >> 2),
 492         0x00000000,
 493         (0x0e00 << 16) | (0xae00 >> 2),
 494         0x00000000,
 495         (0x0e00 << 16) | (0x9604 >> 2),
 496         0x00000000,
 497         (0x0e00 << 16) | (0xac08 >> 2),
 498         0x00000000,
 499         (0x0e00 << 16) | (0xac0c >> 2),
 500         0x00000000,
 501         (0x0e00 << 16) | (0xac10 >> 2),
 502         0x00000000,
 503         (0x0e00 << 16) | (0xac14 >> 2),
 504         0x00000000,
 505         (0x0e00 << 16) | (0xac58 >> 2),
 506         0x00000000,
 507         (0x0e00 << 16) | (0xac68 >> 2),
 508         0x00000000,
 509         (0x0e00 << 16) | (0xac6c >> 2),
 510         0x00000000,
 511         (0x0e00 << 16) | (0xac70 >> 2),
 512         0x00000000,
 513         (0x0e00 << 16) | (0xac74 >> 2),
 514         0x00000000,
 515         (0x0e00 << 16) | (0xac78 >> 2),
 516         0x00000000,
 517         (0x0e00 << 16) | (0xac7c >> 2),
 518         0x00000000,
 519         (0x0e00 << 16) | (0xac80 >> 2),
 520         0x00000000,
 521         (0x0e00 << 16) | (0xac84 >> 2),
 522         0x00000000,
 523         (0x0e00 << 16) | (0xac88 >> 2),
 524         0x00000000,
 525         (0x0e00 << 16) | (0xac8c >> 2),
 526         0x00000000,
 527         (0x0e00 << 16) | (0x970c >> 2),
 528         0x00000000,
 529         (0x0e00 << 16) | (0x9714 >> 2),
 530         0x00000000,
 531         (0x0e00 << 16) | (0x9718 >> 2),
 532         0x00000000,
 533         (0x0e00 << 16) | (0x971c >> 2),
 534         0x00000000,
 535         (0x0e00 << 16) | (0x31068 >> 2),
 536         0x00000000,
 537         (0x4e00 << 16) | (0x31068 >> 2),
 538         0x00000000,
 539         (0x5e00 << 16) | (0x31068 >> 2),
 540         0x00000000,
 541         (0x6e00 << 16) | (0x31068 >> 2),
 542         0x00000000,
 543         (0x7e00 << 16) | (0x31068 >> 2),
 544         0x00000000,
 545         (0x8e00 << 16) | (0x31068 >> 2),
 546         0x00000000,
 547         (0x9e00 << 16) | (0x31068 >> 2),
 548         0x00000000,
 549         (0xae00 << 16) | (0x31068 >> 2),
 550         0x00000000,
 551         (0xbe00 << 16) | (0x31068 >> 2),
 552         0x00000000,
 553         (0x0e00 << 16) | (0xcd10 >> 2),
 554         0x00000000,
 555         (0x0e00 << 16) | (0xcd14 >> 2),
 556         0x00000000,
 557         (0x0e00 << 16) | (0x88b0 >> 2),
 558         0x00000000,
 559         (0x0e00 << 16) | (0x88b4 >> 2),
 560         0x00000000,
 561         (0x0e00 << 16) | (0x88b8 >> 2),
 562         0x00000000,
 563         (0x0e00 << 16) | (0x88bc >> 2),
 564         0x00000000,
 565         (0x0400 << 16) | (0x89c0 >> 2),
 566         0x00000000,
 567         (0x0e00 << 16) | (0x88c4 >> 2),
 568         0x00000000,
 569         (0x0e00 << 16) | (0x88c8 >> 2),
 570         0x00000000,
 571         (0x0e00 << 16) | (0x88d0 >> 2),
 572         0x00000000,
 573         (0x0e00 << 16) | (0x88d4 >> 2),
 574         0x00000000,
 575         (0x0e00 << 16) | (0x88d8 >> 2),
 576         0x00000000,
 577         (0x0e00 << 16) | (0x8980 >> 2),
 578         0x00000000,
 579         (0x0e00 << 16) | (0x30938 >> 2),
 580         0x00000000,
 581         (0x0e00 << 16) | (0x3093c >> 2),
 582         0x00000000,
 583         (0x0e00 << 16) | (0x30940 >> 2),
 584         0x00000000,
 585         (0x0e00 << 16) | (0x89a0 >> 2),
 586         0x00000000,
 587         (0x0e00 << 16) | (0x30900 >> 2),
 588         0x00000000,
 589         (0x0e00 << 16) | (0x30904 >> 2),
 590         0x00000000,
 591         (0x0e00 << 16) | (0x89b4 >> 2),
 592         0x00000000,
 593         (0x0e00 << 16) | (0x3c210 >> 2),
 594         0x00000000,
 595         (0x0e00 << 16) | (0x3c214 >> 2),
 596         0x00000000,
 597         (0x0e00 << 16) | (0x3c218 >> 2),
 598         0x00000000,
 599         (0x0e00 << 16) | (0x8904 >> 2),
 600         0x00000000,
 601         0x5,
 602         (0x0e00 << 16) | (0x8c28 >> 2),
 603         (0x0e00 << 16) | (0x8c2c >> 2),
 604         (0x0e00 << 16) | (0x8c30 >> 2),
 605         (0x0e00 << 16) | (0x8c34 >> 2),
 606         (0x0e00 << 16) | (0x9600 >> 2),
 607 };
 608
 609 static const u32 kalindi_rlc_save_restore_register_list[] =
 610 {
 611         (0x0e00 << 16) | (0xc12c >> 2),
 612         0x00000000,
 613         (0x0e00 << 16) | (0xc140 >> 2),
 614         0x00000000,
 615         (0x0e00 << 16) | (0xc150 >> 2),
 616         0x00000000,
 617         (0x0e00 << 16) | (0xc15c >> 2),
 618         0x00000000,
 619         (0x0e00 << 16) | (0xc168 >> 2),
 620         0x00000000,
 621         (0x0e00 << 16) | (0xc170 >> 2),
 622         0x00000000,
 623         (0x0e00 << 16) | (0xc204 >> 2),
 624         0x00000000,
 625         (0x0e00 << 16) | (0xc2b4 >> 2),
 626         0x00000000,
 627         (0x0e00 << 16) | (0xc2b8 >> 2),
 628         0x00000000,
 629         (0x0e00 << 16) | (0xc2bc >> 2),
 630         0x00000000,
 631         (0x0e00 << 16) | (0xc2c0 >> 2),
 632         0x00000000,
 633         (0x0e00 << 16) | (0x8228 >> 2),
 634         0x00000000,
 635         (0x0e00 << 16) | (0x829c >> 2),
 636         0x00000000,
 637         (0x0e00 << 16) | (0x869c >> 2),
 638         0x00000000,
 639         (0x0600 << 16) | (0x98f4 >> 2),
 640         0x00000000,
 641         (0x0e00 << 16) | (0x98f8 >> 2),
 642         0x00000000,
 643         (0x0e00 << 16) | (0x9900 >> 2),
 644         0x00000000,
 645         (0x0e00 << 16) | (0xc260 >> 2),
 646         0x00000000,
 647         (0x0e00 << 16) | (0x90e8 >> 2),
 648         0x00000000,
 649         (0x0e00 << 16) | (0x3c000 >> 2),
 650         0x00000000,
 651         (0x0e00 << 16) | (0x3c00c >> 2),
 652         0x00000000,
 653         (0x0e00 << 16) | (0x8c1c >> 2),
 654         0x00000000,
 655         (0x0e00 << 16) | (0x9700 >> 2),
 656         0x00000000,
 657         (0x0e00 << 16) | (0xcd20 >> 2),
 658         0x00000000,
 659         (0x4e00 << 16) | (0xcd20 >> 2),
 660         0x00000000,
 661         (0x5e00 << 16) | (0xcd20 >> 2),
 662         0x00000000,
 663         (0x6e00 << 16) | (0xcd20 >> 2),
 664         0x00000000,
 665         (0x7e00 << 16) | (0xcd20 >> 2),
 666         0x00000000,
 667         (0x0e00 << 16) | (0x89bc >> 2),
 668         0x00000000,
 669         (0x0e00 << 16) | (0x8900 >> 2),
 670         0x00000000,
 671         0x3,
 672         (0x0e00 << 16) | (0xc130 >> 2),
 673         0x00000000,
 674         (0x0e00 << 16) | (0xc134 >> 2),
 675         0x00000000,
 676         (0x0e00 << 16) | (0xc1fc >> 2),
 677         0x00000000,
 678         (0x0e00 << 16) | (0xc208 >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0xc264 >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0xc268 >> 2),
 683         0x00000000,
 684         (0x0e00 << 16) | (0xc26c >> 2),
 685         0x00000000,
 686         (0x0e00 << 16) | (0xc270 >> 2),
 687         0x00000000,
 688         (0x0e00 << 16) | (0xc274 >> 2),
 689         0x00000000,
 690         (0x0e00 << 16) | (0xc28c >> 2),
 691         0x00000000,
 692         (0x0e00 << 16) | (0xc290 >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0xc294 >> 2),
 695         0x00000000,
 696         (0x0e00 << 16) | (0xc298 >> 2),
 697         0x00000000,
 698         (0x0e00 << 16) | (0xc2a0 >> 2),
 699         0x00000000,
 700         (0x0e00 << 16) | (0xc2a4 >> 2),
 701         0x00000000,
 702         (0x0e00 << 16) | (0xc2a8 >> 2),
 703         0x00000000,
 704         (0x0e00 << 16) | (0xc2ac >> 2),
 705         0x00000000,
 706         (0x0e00 << 16) | (0x301d0 >> 2),
 707         0x00000000,
 708         (0x0e00 << 16) | (0x30238 >> 2),
 709         0x00000000,
 710         (0x0e00 << 16) | (0x30250 >> 2),
 711         0x00000000,
 712         (0x0e00 << 16) | (0x30254 >> 2),
 713         0x00000000,
 714         (0x0e00 << 16) | (0x30258 >> 2),
 715         0x00000000,
 716         (0x0e00 << 16) | (0x3025c >> 2),
 717         0x00000000,
 718         (0x4e00 << 16) | (0xc900 >> 2),
 719         0x00000000,
 720         (0x5e00 << 16) | (0xc900 >> 2),
 721         0x00000000,
 722         (0x6e00 << 16) | (0xc900 >> 2),
 723         0x00000000,
 724         (0x7e00 << 16) | (0xc900 >> 2),
 725         0x00000000,
 726         (0x4e00 << 16) | (0xc904 >> 2),
 727         0x00000000,
 728         (0x5e00 << 16) | (0xc904 >> 2),
 729         0x00000000,
 730         (0x6e00 << 16) | (0xc904 >> 2),
 731         0x00000000,
 732         (0x7e00 << 16) | (0xc904 >> 2),
 733         0x00000000,
 734         (0x4e00 << 16) | (0xc908 >> 2),
 735         0x00000000,
 736         (0x5e00 << 16) | (0xc908 >> 2),
 737         0x00000000,
 738         (0x6e00 << 16) | (0xc908 >> 2),
 739         0x00000000,
 740         (0x7e00 << 16) | (0xc908 >> 2),
 741         0x00000000,
 742         (0x4e00 << 16) | (0xc90c >> 2),
 743         0x00000000,
 744         (0x5e00 << 16) | (0xc90c >> 2),
 745         0x00000000,
 746         (0x6e00 << 16) | (0xc90c >> 2),
 747         0x00000000,
 748         (0x7e00 << 16) | (0xc90c >> 2),
 749         0x00000000,
 750         (0x4e00 << 16) | (0xc910 >> 2),
 751         0x00000000,
 752         (0x5e00 << 16) | (0xc910 >> 2),
 753         0x00000000,
 754         (0x6e00 << 16) | (0xc910 >> 2),
 755         0x00000000,
 756         (0x7e00 << 16) | (0xc910 >> 2),
 757         0x00000000,
 758         (0x0e00 << 16) | (0xc99c >> 2),
 759         0x00000000,
 760         (0x0e00 << 16) | (0x9834 >> 2),
 761         0x00000000,
 762         (0x0000 << 16) | (0x30f00 >> 2),
 763         0x00000000,
 764         (0x0000 << 16) | (0x30f04 >> 2),
 765         0x00000000,
 766         (0x0000 << 16) | (0x30f08 >> 2),
 767         0x00000000,
 768         (0x0000 << 16) | (0x30f0c >> 2),
 769         0x00000000,
 770         (0x0600 << 16) | (0x9b7c >> 2),
 771         0x00000000,
 772         (0x0e00 << 16) | (0x8a14 >> 2),
 773         0x00000000,
 774         (0x0e00 << 16) | (0x8a18 >> 2),
 775         0x00000000,
 776         (0x0600 << 16) | (0x30a00 >> 2),
 777         0x00000000,
 778         (0x0e00 << 16) | (0x8bf0 >> 2),
 779         0x00000000,
 780         (0x0e00 << 16) | (0x8bcc >> 2),
 781         0x00000000,
 782         (0x0e00 << 16) | (0x8b24 >> 2),
 783         0x00000000,
 784         (0x0e00 << 16) | (0x30a04 >> 2),
 785         0x00000000,
 786         (0x0600 << 16) | (0x30a10 >> 2),
 787         0x00000000,
 788         (0x0600 << 16) | (0x30a14 >> 2),
 789         0x00000000,
 790         (0x0600 << 16) | (0x30a18 >> 2),
 791         0x00000000,
 792         (0x0600 << 16) | (0x30a2c >> 2),
 793         0x00000000,
 794         (0x0e00 << 16) | (0xc700 >> 2),
 795         0x00000000,
 796         (0x0e00 << 16) | (0xc704 >> 2),
 797         0x00000000,
 798         (0x0e00 << 16) | (0xc708 >> 2),
 799         0x00000000,
 800         (0x0e00 << 16) | (0xc768 >> 2),
 801         0x00000000,
 802         (0x0400 << 16) | (0xc770 >> 2),
 803         0x00000000,
 804         (0x0400 << 16) | (0xc774 >> 2),
 805         0x00000000,
 806         (0x0400 << 16) | (0xc798 >> 2),
 807         0x00000000,
 808         (0x0400 << 16) | (0xc79c >> 2),
 809         0x00000000,
 810         (0x0e00 << 16) | (0x9100 >> 2),
 811         0x00000000,
 812         (0x0e00 << 16) | (0x3c010 >> 2),
 813         0x00000000,
 814         (0x0e00 << 16) | (0x8c00 >> 2),
 815         0x00000000,
 816         (0x0e00 << 16) | (0x8c04 >> 2),
 817         0x00000000,
 818         (0x0e00 << 16) | (0x8c20 >> 2),
 819         0x00000000,
 820         (0x0e00 << 16) | (0x8c38 >> 2),
 821         0x00000000,
 822         (0x0e00 << 16) | (0x8c3c >> 2),
 823         0x00000000,
 824         (0x0e00 << 16) | (0xae00 >> 2),
 825         0x00000000,
 826         (0x0e00 << 16) | (0x9604 >> 2),
 827         0x00000000,
 828         (0x0e00 << 16) | (0xac08 >> 2),
 829         0x00000000,
 830         (0x0e00 << 16) | (0xac0c >> 2),
 831         0x00000000,
 832         (0x0e00 << 16) | (0xac10 >> 2),
 833         0x00000000,
 834         (0x0e00 << 16) | (0xac14 >> 2),
 835         0x00000000,
 836         (0x0e00 << 16) | (0xac58 >> 2),
 837         0x00000000,
 838         (0x0e00 << 16) | (0xac68 >> 2),
 839         0x00000000,
 840         (0x0e00 << 16) | (0xac6c >> 2),
 841         0x00000000,
 842         (0x0e00 << 16) | (0xac70 >> 2),
 843         0x00000000,
 844         (0x0e00 << 16) | (0xac74 >> 2),
 845         0x00000000,
 846         (0x0e00 << 16) | (0xac78 >> 2),
 847         0x00000000,
 848         (0x0e00 << 16) | (0xac7c >> 2),
 849         0x00000000,
 850         (0x0e00 << 16) | (0xac80 >> 2),
 851         0x00000000,
 852         (0x0e00 << 16) | (0xac84 >> 2),
 853         0x00000000,
 854         (0x0e00 << 16) | (0xac88 >> 2),
 855         0x00000000,
 856         (0x0e00 << 16) | (0xac8c >> 2),
 857         0x00000000,
 858         (0x0e00 << 16) | (0x970c >> 2),
 859         0x00000000,
 860         (0x0e00 << 16) | (0x9714 >> 2),
 861         0x00000000,
 862         (0x0e00 << 16) | (0x9718 >> 2),
 863         0x00000000,
 864         (0x0e00 << 16) | (0x971c >> 2),
 865         0x00000000,
 866         (0x0e00 << 16) | (0x31068 >> 2),
 867         0x00000000,
 868         (0x4e00 << 16) | (0x31068 >> 2),
 869         0x00000000,
 870         (0x5e00 << 16) | (0x31068 >> 2),
 871         0x00000000,
 872         (0x6e00 << 16) | (0x31068 >> 2),
 873         0x00000000,
 874         (0x7e00 << 16) | (0x31068 >> 2),
 875         0x00000000,
 876         (0x0e00 << 16) | (0xcd10 >> 2),
 877         0x00000000,
 878         (0x0e00 << 16) | (0xcd14 >> 2),
 879         0x00000000,
 880         (0x0e00 << 16) | (0x88b0 >> 2),
 881         0x00000000,
 882         (0x0e00 << 16) | (0x88b4 >> 2),
 883         0x00000000,
 884         (0x0e00 << 16) | (0x88b8 >> 2),
 885         0x00000000,
 886         (0x0e00 << 16) | (0x88bc >> 2),
 887         0x00000000,
 888         (0x0400 << 16) | (0x89c0 >> 2),
 889         0x00000000,
 890         (0x0e00 << 16) | (0x88c4 >> 2),
 891         0x00000000,
 892         (0x0e00 << 16) | (0x88c8 >> 2),
 893         0x00000000,
 894         (0x0e00 << 16) | (0x88d0 >> 2),
 895         0x00000000,
 896         (0x0e00 << 16) | (0x88d4 >> 2),
 897         0x00000000,
 898         (0x0e00 << 16) | (0x88d8 >> 2),
 899         0x00000000,
 900         (0x0e00 << 16) | (0x8980 >> 2),
 901         0x00000000,
 902         (0x0e00 << 16) | (0x30938 >> 2),
 903         0x00000000,
 904         (0x0e00 << 16) | (0x3093c >> 2),
 905         0x00000000,
 906         (0x0e00 << 16) | (0x30940 >> 2),
 907         0x00000000,
 908         (0x0e00 << 16) | (0x89a0 >> 2),
 909         0x00000000,
 910         (0x0e00 << 16) | (0x30900 >> 2),
 911         0x00000000,
 912         (0x0e00 << 16) | (0x30904 >> 2),
 913         0x00000000,
 914         (0x0e00 << 16) | (0x89b4 >> 2),
 915         0x00000000,
 916         (0x0e00 << 16) | (0x3e1fc >> 2),
 917         0x00000000,
 918         (0x0e00 << 16) | (0x3c210 >> 2),
 919         0x00000000,
 920         (0x0e00 << 16) | (0x3c214 >> 2),
 921         0x00000000,
 922         (0x0e00 << 16) | (0x3c218 >> 2),
 923         0x00000000,
 924         (0x0e00 << 16) | (0x8904 >> 2),
 925         0x00000000,
 926         0x5,
 927         (0x0e00 << 16) | (0x8c28 >> 2),
 928         (0x0e00 << 16) | (0x8c2c >> 2),
 929         (0x0e00 << 16) | (0x8c30 >> 2),
 930         (0x0e00 << 16) | (0x8c34 >> 2),
 931         (0x0e00 << 16) | (0x9600 >> 2),
 932 };
 933
 934 static const u32 bonaire_golden_spm_registers[] =
 935 {
 936         0x30800, 0xe0ffffff, 0xe0000000
 937 };
 938
 939 static const u32 bonaire_golden_common_registers[] =
 940 {
 941         0xc770, 0xffffffff, 0x00000800,
 942         0xc774, 0xffffffff, 0x00000800,
 943         0xc798, 0xffffffff, 0x00007fbf,
 944         0xc79c, 0xffffffff, 0x00007faf
 945 };
 946
 947 static const u32 bonaire_golden_registers[] =
 948 {
 949         0x3354, 0x00000333, 0x00000333,
 950         0x3350, 0x000c0fc0, 0x00040200,
 951         0x9a10, 0x00010000, 0x00058208,
 952         0x3c000, 0xffff1fff, 0x00140000,
 953         0x3c200, 0xfdfc0fff, 0x00000100,
 954         0x3c234, 0x40000000, 0x40000200,
 955         0x9830, 0xffffffff, 0x00000000,
 956         0x9834, 0xf00fffff, 0x00000400,
 957         0x9838, 0x0002021c, 0x00020200,
 958         0xc78, 0x00000080, 0x00000000,
 959         0x5bb0, 0x000000f0, 0x00000070,
 960         0x5bc0, 0xf0311fff, 0x80300000,
 961         0x98f8, 0x73773777, 0x12010001,
 962         0x350c, 0x00810000, 0x408af000,
 963         0x7030, 0x31000111, 0x00000011,
 964         0x2f48, 0x73773777, 0x12010001,
 965         0x220c, 0x00007fb6, 0x0021a1b1,
 966         0x2210, 0x00007fb6, 0x002021b1,
 967         0x2180, 0x00007fb6, 0x00002191,
 968         0x2218, 0x00007fb6, 0x002121b1,
 969         0x221c, 0x00007fb6, 0x002021b1,
 970         0x21dc, 0x00007fb6, 0x00002191,
 971         0x21e0, 0x00007fb6, 0x00002191,
 972         0x3628, 0x0000003f, 0x0000000a,
 973         0x362c, 0x0000003f, 0x0000000a,
 974         0x2ae4, 0x00073ffe, 0x000022a2,
 975         0x240c, 0x000007ff, 0x00000000,
 976         0x8a14, 0xf000003f, 0x00000007,
 977         0x8bf0, 0x00002001, 0x00000001,
 978         0x8b24, 0xffffffff, 0x00ffffff,
 979         0x30a04, 0x0000ff0f, 0x00000000,
 980         0x28a4c, 0x07ffffff, 0x06000000,
 981         0x4d8, 0x00000fff, 0x00000100,
 982         0x3e78, 0x00000001, 0x00000002,
 983         0x9100, 0x03000000, 0x0362c688,
 984         0x8c00, 0x000000ff, 0x00000001,
 985         0xe40, 0x00001fff, 0x00001fff,
 986         0x9060, 0x0000007f, 0x00000020,
 987         0x9508, 0x00010000, 0x00010000,
 988         0xac14, 0x000003ff, 0x000000f3,
 989         0xac0c, 0xffffffff, 0x00001032
 990 };
 991
 992 static const u32 bonaire_mgcg_cgcg_init[] =
 993 {
 994         0xc420, 0xffffffff, 0xfffffffc,
 995         0x30800, 0xffffffff, 0xe0000000,
 996         0x3c2a0, 0xffffffff, 0x00000100,
 997         0x3c208, 0xffffffff, 0x00000100,
 998         0x3c2c0, 0xffffffff, 0xc0000100,
 999         0x3c2c8, 0xffffffff, 0xc0000100,
1000         0x3c2c4, 0xffffffff, 0xc0000100,
1001         0x55e4, 0xffffffff, 0x00600100,
1002         0x3c280, 0xffffffff, 0x00000100,
1003         0x3c214, 0xffffffff, 0x06000100,
1004         0x3c220, 0xffffffff, 0x00000100,
1005         0x3c218, 0xffffffff, 0x06000100,
1006         0x3c204, 0xffffffff, 0x00000100,
1007         0x3c2e0, 0xffffffff, 0x00000100,
1008         0x3c224, 0xffffffff, 0x00000100,
1009         0x3c200, 0xffffffff, 0x00000100,
1010         0x3c230, 0xffffffff, 0x00000100,
1011         0x3c234, 0xffffffff, 0x00000100,
1012         0x3c250, 0xffffffff, 0x00000100,
1013         0x3c254, 0xffffffff, 0x00000100,
1014         0x3c258, 0xffffffff, 0x00000100,
1015         0x3c25c, 0xffffffff, 0x00000100,
1016         0x3c260, 0xffffffff, 0x00000100,
1017         0x3c27c, 0xffffffff, 0x00000100,
1018         0x3c278, 0xffffffff, 0x00000100,
1019         0x3c210, 0xffffffff, 0x06000100,
1020         0x3c290, 0xffffffff, 0x00000100,
1021         0x3c274, 0xffffffff, 0x00000100,
1022         0x3c2b4, 0xffffffff, 0x00000100,
1023         0x3c2b0, 0xffffffff, 0x00000100,
1024         0x3c270, 0xffffffff, 0x00000100,
1025         0x30800, 0xffffffff, 0xe0000000,
1026         0x3c020, 0xffffffff, 0x00010000,
1027         0x3c024, 0xffffffff, 0x00030002,
1028         0x3c028, 0xffffffff, 0x00040007,
1029         0x3c02c, 0xffffffff, 0x00060005,
1030         0x3c030, 0xffffffff, 0x00090008,
1031         0x3c034, 0xffffffff, 0x00010000,
1032         0x3c038, 0xffffffff, 0x00030002,
1033         0x3c03c, 0xffffffff, 0x00040007,
1034         0x3c040, 0xffffffff, 0x00060005,
1035         0x3c044, 0xffffffff, 0x00090008,
1036         0x3c048, 0xffffffff, 0x00010000,
1037         0x3c04c, 0xffffffff, 0x00030002,
1038         0x3c050, 0xffffffff, 0x00040007,
1039         0x3c054, 0xffffffff, 0x00060005,
1040         0x3c058, 0xffffffff, 0x00090008,
1041         0x3c05c, 0xffffffff, 0x00010000,
1042         0x3c060, 0xffffffff, 0x00030002,
1043         0x3c064, 0xffffffff, 0x00040007,
1044         0x3c068, 0xffffffff, 0x00060005,
1045         0x3c06c, 0xffffffff, 0x00090008,
1046         0x3c070, 0xffffffff, 0x00010000,
1047         0x3c074, 0xffffffff, 0x00030002,
1048         0x3c078, 0xffffffff, 0x00040007,
1049         0x3c07c, 0xffffffff, 0x00060005,
1050         0x3c080, 0xffffffff, 0x00090008,
1051         0x3c084, 0xffffffff, 0x00010000,
1052         0x3c088, 0xffffffff, 0x00030002,
1053         0x3c08c, 0xffffffff, 0x00040007,
1054         0x3c090, 0xffffffff, 0x00060005,
1055         0x3c094, 0xffffffff, 0x00090008,
1056         0x3c098, 0xffffffff, 0x00010000,
1057         0x3c09c, 0xffffffff, 0x00030002,
1058         0x3c0a0, 0xffffffff, 0x00040007,
1059         0x3c0a4, 0xffffffff, 0x00060005,
1060         0x3c0a8, 0xffffffff, 0x00090008,
1061         0x3c000, 0xffffffff, 0x96e00200,
1062         0x8708, 0xffffffff, 0x00900100,
1063         0xc424, 0xffffffff, 0x0020003f,
1064         0x38, 0xffffffff, 0x0140001c,
1065         0x3c, 0x000f0000, 0x000f0000,
1066         0x220, 0xffffffff, 0xC060000C,
1067         0x224, 0xc0000fff, 0x00000100,
1068         0xf90, 0xffffffff, 0x00000100,
1069         0xf98, 0x00000101, 0x00000000,
1070         0x20a8, 0xffffffff, 0x00000104,
1071         0x55e4, 0xff000fff, 0x00000100,
1072         0x30cc, 0xc0000fff, 0x00000104,
1073         0xc1e4, 0x00000001, 0x00000001,
1074         0xd00c, 0xff000ff0, 0x00000100,
1075         0xd80c, 0xff000ff0, 0x00000100
1076 };
1077
1078 static const u32 spectre_golden_spm_registers[] =
1079 {
1080         0x30800, 0xe0ffffff, 0xe0000000
1081 };
1082
1083 static const u32 spectre_golden_common_registers[] =
1084 {
1085         0xc770, 0xffffffff, 0x00000800,
1086         0xc774, 0xffffffff, 0x00000800,
1087         0xc798, 0xffffffff, 0x00007fbf,
1088         0xc79c, 0xffffffff, 0x00007faf
1089 };
1090
1091 static const u32 spectre_golden_registers[] =
1092 {
1093         0x3c000, 0xffff1fff, 0x96940200,
1094         0x3c00c, 0xffff0001, 0xff000000,
1095         0x3c200, 0xfffc0fff, 0x00000100,
1096         0x6ed8, 0x00010101, 0x00010000,
1097         0x9834, 0xf00fffff, 0x00000400,
1098         0x9838, 0xfffffffc, 0x00020200,
1099         0x5bb0, 0x000000f0, 0x00000070,
1100         0x5bc0, 0xf0311fff, 0x80300000,
1101         0x98f8, 0x73773777, 0x12010001,
1102         0x9b7c, 0x00ff0000, 0x00fc0000,
1103         0x2f48, 0x73773777, 0x12010001,
1104         0x8a14, 0xf000003f, 0x00000007,
1105         0x8b24, 0xffffffff, 0x00ffffff,
1106         0x28350, 0x3f3f3fff, 0x00000082,
1107         0x28354, 0x0000003f, 0x00000000,
1108         0x3e78, 0x00000001, 0x00000002,
1109         0x913c, 0xffff03df, 0x00000004,
1110         0xc768, 0x00000008, 0x00000008,
1111         0x8c00, 0x000008ff, 0x00000800,
1112         0x9508, 0x00010000, 0x00010000,
1113         0xac0c, 0xffffffff, 0x54763210,
1114         0x214f8, 0x01ff01ff, 0x00000002,
1115         0x21498, 0x007ff800, 0x00200000,
1116         0x2015c, 0xffffffff, 0x00000f40,
1117         0x30934, 0xffffffff, 0x00000001
1118 };
1119
1120 static const u32 spectre_mgcg_cgcg_init[] =
1121 {
1122         0xc420, 0xffffffff, 0xfffffffc,
1123         0x30800, 0xffffffff, 0xe0000000,
1124         0x3c2a0, 0xffffffff, 0x00000100,
1125         0x3c208, 0xffffffff, 0x00000100,
1126         0x3c2c0, 0xffffffff, 0x00000100,
1127         0x3c2c8, 0xffffffff, 0x00000100,
1128         0x3c2c4, 0xffffffff, 0x00000100,
1129         0x55e4, 0xffffffff, 0x00600100,
1130         0x3c280, 0xffffffff, 0x00000100,
1131         0x3c214, 0xffffffff, 0x06000100,
1132         0x3c220, 0xffffffff, 0x00000100,
1133         0x3c218, 0xffffffff, 0x06000100,
1134         0x3c204, 0xffffffff, 0x00000100,
1135         0x3c2e0, 0xffffffff, 0x00000100,
1136         0x3c224, 0xffffffff, 0x00000100,
1137         0x3c200, 0xffffffff, 0x00000100,
1138         0x3c230, 0xffffffff, 0x00000100,
1139         0x3c234, 0xffffffff, 0x00000100,
1140         0x3c250, 0xffffffff, 0x00000100,
1141         0x3c254, 0xffffffff, 0x00000100,
1142         0x3c258, 0xffffffff, 0x00000100,
1143         0x3c25c, 0xffffffff, 0x00000100,
1144         0x3c260, 0xffffffff, 0x00000100,
1145         0x3c27c, 0xffffffff, 0x00000100,
1146         0x3c278, 0xffffffff, 0x00000100,
1147         0x3c210, 0xffffffff, 0x06000100,
1148         0x3c290, 0xffffffff, 0x00000100,
1149         0x3c274, 0xffffffff, 0x00000100,
1150         0x3c2b4, 0xffffffff, 0x00000100,
1151         0x3c2b0, 0xffffffff, 0x00000100,
1152         0x3c270, 0xffffffff, 0x00000100,
1153         0x30800, 0xffffffff, 0xe0000000,
1154         0x3c020, 0xffffffff, 0x00010000,
1155         0x3c024, 0xffffffff, 0x00030002,
1156         0x3c028, 0xffffffff, 0x00040007,
1157         0x3c02c, 0xffffffff, 0x00060005,
1158         0x3c030, 0xffffffff, 0x00090008,
1159         0x3c034, 0xffffffff, 0x00010000,
1160         0x3c038, 0xffffffff, 0x00030002,
1161         0x3c03c, 0xffffffff, 0x00040007,
1162         0x3c040, 0xffffffff, 0x00060005,
1163         0x3c044, 0xffffffff, 0x00090008,
1164         0x3c048, 0xffffffff, 0x00010000,
1165         0x3c04c, 0xffffffff, 0x00030002,
1166         0x3c050, 0xffffffff, 0x00040007,
1167         0x3c054, 0xffffffff, 0x00060005,
1168         0x3c058, 0xffffffff, 0x00090008,
1169         0x3c05c, 0xffffffff, 0x00010000,
1170         0x3c060, 0xffffffff, 0x00030002,
1171         0x3c064, 0xffffffff, 0x00040007,
1172         0x3c068, 0xffffffff, 0x00060005,
1173         0x3c06c, 0xffffffff, 0x00090008,
1174         0x3c070, 0xffffffff, 0x00010000,
1175         0x3c074, 0xffffffff, 0x00030002,
1176         0x3c078, 0xffffffff, 0x00040007,
1177         0x3c07c, 0xffffffff, 0x00060005,
1178         0x3c080, 0xffffffff, 0x00090008,
1179         0x3c084, 0xffffffff, 0x00010000,
1180         0x3c088, 0xffffffff, 0x00030002,
1181         0x3c08c, 0xffffffff, 0x00040007,
1182         0x3c090, 0xffffffff, 0x00060005,
1183         0x3c094, 0xffffffff, 0x00090008,
1184         0x3c098, 0xffffffff, 0x00010000,
1185         0x3c09c, 0xffffffff, 0x00030002,
1186         0x3c0a0, 0xffffffff, 0x00040007,
1187         0x3c0a4, 0xffffffff, 0x00060005,
1188         0x3c0a8, 0xffffffff, 0x00090008,
1189         0x3c0ac, 0xffffffff, 0x00010000,
1190         0x3c0b0, 0xffffffff, 0x00030002,
1191         0x3c0b4, 0xffffffff, 0x00040007,
1192         0x3c0b8, 0xffffffff, 0x00060005,
1193         0x3c0bc, 0xffffffff, 0x00090008,
1194         0x3c000, 0xffffffff, 0x96e00200,
1195         0x8708, 0xffffffff, 0x00900100,
1196         0xc424, 0xffffffff, 0x0020003f,
1197         0x38, 0xffffffff, 0x0140001c,
1198         0x3c, 0x000f0000, 0x000f0000,
1199         0x220, 0xffffffff, 0xC060000C,
1200         0x224, 0xc0000fff, 0x00000100,
1201         0xf90, 0xffffffff, 0x00000100,
1202         0xf98, 0x00000101, 0x00000000,
1203         0x20a8, 0xffffffff, 0x00000104,
1204         0x55e4, 0xff000fff, 0x00000100,
1205         0x30cc, 0xc0000fff, 0x00000104,
1206         0xc1e4, 0x00000001, 0x00000001,
1207         0xd00c, 0xff000ff0, 0x00000100,
1208         0xd80c, 0xff000ff0, 0x00000100
1209 };
1210
1211 static const u32 kalindi_golden_spm_registers[] =
1212 {
1213         0x30800, 0xe0ffffff, 0xe0000000
1214 };
1215
1216 static const u32 kalindi_golden_common_registers[] =
1217 {
1218         0xc770, 0xffffffff, 0x00000800,
1219         0xc774, 0xffffffff, 0x00000800,
1220         0xc798, 0xffffffff, 0x00007fbf,
1221         0xc79c, 0xffffffff, 0x00007faf
1222 };
1223
1224 static const u32 kalindi_golden_registers[] =
1225 {
1226         0x3c000, 0xffffdfff, 0x6e944040,
1227         0x55e4, 0xff607fff, 0xfc000100,
1228         0x3c220, 0xff000fff, 0x00000100,
1229         0x3c224, 0xff000fff, 0x00000100,
1230         0x3c200, 0xfffc0fff, 0x00000100,
1231         0x6ed8, 0x00010101, 0x00010000,
1232         0x9830, 0xffffffff, 0x00000000,
1233         0x9834, 0xf00fffff, 0x00000400,
1234         0x5bb0, 0x000000f0, 0x00000070,
1235         0x5bc0, 0xf0311fff, 0x80300000,
1236         0x98f8, 0x73773777, 0x12010001,
1237         0x98fc, 0xffffffff, 0x00000010,
1238         0x9b7c, 0x00ff0000, 0x00fc0000,
1239         0x8030, 0x00001f0f, 0x0000100a,
1240         0x2f48, 0x73773777, 0x12010001,
1241         0x2408, 0x000fffff, 0x000c007f,
1242         0x8a14, 0xf000003f, 0x00000007,
1243         0x8b24, 0x3fff3fff, 0x00ffcfff,
1244         0x30a04, 0x0000ff0f, 0x00000000,
1245         0x28a4c, 0x07ffffff, 0x06000000,
1246         0x4d8, 0x00000fff, 0x00000100,
1247         0x3e78, 0x00000001, 0x00000002,
1248         0xc768, 0x00000008, 0x00000008,
1249         0x8c00, 0x000000ff, 0x00000003,
1250         0x214f8, 0x01ff01ff, 0x00000002,
1251         0x21498, 0x007ff800, 0x00200000,
1252         0x2015c, 0xffffffff, 0x00000f40,
1253         0x88c4, 0x001f3ae3, 0x00000082,
1254         0x88d4, 0x0000001f, 0x00000010,
1255         0x30934, 0xffffffff, 0x00000000
1256 };
1257
1258 static const u32 kalindi_mgcg_cgcg_init[] =
1259 {
1260         0xc420, 0xffffffff, 0xfffffffc,
1261         0x30800, 0xffffffff, 0xe0000000,
1262         0x3c2a0, 0xffffffff, 0x00000100,
1263         0x3c208, 0xffffffff, 0x00000100,
1264         0x3c2c0, 0xffffffff, 0x00000100,
1265         0x3c2c8, 0xffffffff, 0x00000100,
1266         0x3c2c4, 0xffffffff, 0x00000100,
1267         0x55e4, 0xffffffff, 0x00600100,
1268         0x3c280, 0xffffffff, 0x00000100,
1269         0x3c214, 0xffffffff, 0x06000100,
1270         0x3c220, 0xffffffff, 0x00000100,
1271         0x3c218, 0xffffffff, 0x06000100,
1272         0x3c204, 0xffffffff, 0x00000100,
1273         0x3c2e0, 0xffffffff, 0x00000100,
1274         0x3c224, 0xffffffff, 0x00000100,
1275         0x3c200, 0xffffffff, 0x00000100,
1276         0x3c230, 0xffffffff, 0x00000100,
1277         0x3c234, 0xffffffff, 0x00000100,
1278         0x3c250, 0xffffffff, 0x00000100,
1279         0x3c254, 0xffffffff, 0x00000100,
1280         0x3c258, 0xffffffff, 0x00000100,
1281         0x3c25c, 0xffffffff, 0x00000100,
1282         0x3c260, 0xffffffff, 0x00000100,
1283         0x3c27c, 0xffffffff, 0x00000100,
1284         0x3c278, 0xffffffff, 0x00000100,
1285         0x3c210, 0xffffffff, 0x06000100,
1286         0x3c290, 0xffffffff, 0x00000100,
1287         0x3c274, 0xffffffff, 0x00000100,
1288         0x3c2b4, 0xffffffff, 0x00000100,
1289         0x3c2b0, 0xffffffff, 0x00000100,
1290         0x3c270, 0xffffffff, 0x00000100,
1291         0x30800, 0xffffffff, 0xe0000000,
1292         0x3c020, 0xffffffff, 0x00010000,
1293         0x3c024, 0xffffffff, 0x00030002,
1294         0x3c028, 0xffffffff, 0x00040007,
1295         0x3c02c, 0xffffffff, 0x00060005,
1296         0x3c030, 0xffffffff, 0x00090008,
1297         0x3c034, 0xffffffff, 0x00010000,
1298         0x3c038, 0xffffffff, 0x00030002,
1299         0x3c03c, 0xffffffff, 0x00040007,
1300         0x3c040, 0xffffffff, 0x00060005,
1301         0x3c044, 0xffffffff, 0x00090008,
1302         0x3c000, 0xffffffff, 0x96e00200,
1303         0x8708, 0xffffffff, 0x00900100,
1304         0xc424, 0xffffffff, 0x0020003f,
1305         0x38, 0xffffffff, 0x0140001c,
1306         0x3c, 0x000f0000, 0x000f0000,
1307         0x220, 0xffffffff, 0xC060000C,
1308         0x224, 0xc0000fff, 0x00000100,
1309         0x20a8, 0xffffffff, 0x00000104,
1310         0x55e4, 0xff000fff, 0x00000100,
1311         0x30cc, 0xc0000fff, 0x00000104,
1312         0xc1e4, 0x00000001, 0x00000001,
1313         0xd00c, 0xff000ff0, 0x00000100,
1314         0xd80c, 0xff000ff0, 0x00000100
1315 };
1316
1317 static const u32 hawaii_golden_spm_registers[] =
1318 {
1319         0x30800, 0xe0ffffff, 0xe0000000
1320 };
1321
1322 static const u32 hawaii_golden_common_registers[] =
1323 {
1324         0x30800, 0xffffffff, 0xe0000000,
1325         0x28350, 0xffffffff, 0x3a00161a,
1326         0x28354, 0xffffffff, 0x0000002e,
1327         0x9a10, 0xffffffff, 0x00018208,
1328         0x98f8, 0xffffffff, 0x12011003
1329 };
1330
1331 static const u32 hawaii_golden_registers[] =
1332 {
1333         0x3354, 0x00000333, 0x00000333,
1334         0x9a10, 0x00010000, 0x00058208,
1335         0x9830, 0xffffffff, 0x00000000,
1336         0x9834, 0xf00fffff, 0x00000400,
1337         0x9838, 0x0002021c, 0x00020200,
1338         0xc78, 0x00000080, 0x00000000,
1339         0x5bb0, 0x000000f0, 0x00000070,
1340         0x5bc0, 0xf0311fff, 0x80300000,
1341         0x350c, 0x00810000, 0x408af000,
1342         0x7030, 0x31000111, 0x00000011,
1343         0x2f48, 0x73773777, 0x12010001,
1344         0x2120, 0x0000007f, 0x0000001b,
1345         0x21dc, 0x00007fb6, 0x00002191,
1346         0x3628, 0x0000003f, 0x0000000a,
1347         0x362c, 0x0000003f, 0x0000000a,
1348         0x2ae4, 0x00073ffe, 0x000022a2,
1349         0x240c, 0x000007ff, 0x00000000,
1350         0x8bf0, 0x00002001, 0x00000001,
1351         0x8b24, 0xffffffff, 0x00ffffff,
1352         0x30a04, 0x0000ff0f, 0x00000000,
1353         0x28a4c, 0x07ffffff, 0x06000000,
1354         0x3e78, 0x00000001, 0x00000002,
1355         0xc768, 0x00000008, 0x00000008,
1356         0xc770, 0x00000f00, 0x00000800,
1357         0xc774, 0x00000f00, 0x00000800,
1358         0xc798, 0x00ffffff, 0x00ff7fbf,
1359         0xc79c, 0x00ffffff, 0x00ff7faf,
1360         0x8c00, 0x000000ff, 0x00000800,
1361         0xe40, 0x00001fff, 0x00001fff,
1362         0x9060, 0x0000007f, 0x00000020,
1363         0x9508, 0x00010000, 0x00010000,
1364         0xae00, 0x00100000, 0x000ff07c,
1365         0xac14, 0x000003ff, 0x0000000f,
1366         0xac10, 0xffffffff, 0x7564fdec,
1367         0xac0c, 0xffffffff, 0x3120b9a8,
1368         0xac08, 0x20000000, 0x0f9c0000
1369 };
1370
1371 static const u32 hawaii_mgcg_cgcg_init[] =
1372 {
1373         0xc420, 0xffffffff, 0xfffffffd,
1374         0x30800, 0xffffffff, 0xe0000000,
1375         0x3c2a0, 0xffffffff, 0x00000100,
1376         0x3c208, 0xffffffff, 0x00000100,
1377         0x3c2c0, 0xffffffff, 0x00000100,
1378         0x3c2c8, 0xffffffff, 0x00000100,
1379         0x3c2c4, 0xffffffff, 0x00000100,
1380         0x55e4, 0xffffffff, 0x00200100,
1381         0x3c280, 0xffffffff, 0x00000100,
1382         0x3c214, 0xffffffff, 0x06000100,
1383         0x3c220, 0xffffffff, 0x00000100,
1384         0x3c218, 0xffffffff, 0x06000100,
1385         0x3c204, 0xffffffff, 0x00000100,
1386         0x3c2e0, 0xffffffff, 0x00000100,
1387         0x3c224, 0xffffffff, 0x00000100,
1388         0x3c200, 0xffffffff, 0x00000100,
1389         0x3c230, 0xffffffff, 0x00000100,
1390         0x3c234, 0xffffffff, 0x00000100,
1391         0x3c250, 0xffffffff, 0x00000100,
1392         0x3c254, 0xffffffff, 0x00000100,
1393         0x3c258, 0xffffffff, 0x00000100,
1394         0x3c25c, 0xffffffff, 0x00000100,
1395         0x3c260, 0xffffffff, 0x00000100,
1396         0x3c27c, 0xffffffff, 0x00000100,
1397         0x3c278, 0xffffffff, 0x00000100,
1398         0x3c210, 0xffffffff, 0x06000100,
1399         0x3c290, 0xffffffff, 0x00000100,
1400         0x3c274, 0xffffffff, 0x00000100,
1401         0x3c2b4, 0xffffffff, 0x00000100,
1402         0x3c2b0, 0xffffffff, 0x00000100,
1403         0x3c270, 0xffffffff, 0x00000100,
1404         0x30800, 0xffffffff, 0xe0000000,
1405         0x3c020, 0xffffffff, 0x00010000,
1406         0x3c024, 0xffffffff, 0x00030002,
1407         0x3c028, 0xffffffff, 0x00040007,
1408         0x3c02c, 0xffffffff, 0x00060005,
1409         0x3c030, 0xffffffff, 0x00090008,
1410         0x3c034, 0xffffffff, 0x00010000,
1411         0x3c038, 0xffffffff, 0x00030002,
1412         0x3c03c, 0xffffffff, 0x00040007,
1413         0x3c040, 0xffffffff, 0x00060005,
1414         0x3c044, 0xffffffff, 0x00090008,
1415         0x3c048, 0xffffffff, 0x00010000,
1416         0x3c04c, 0xffffffff, 0x00030002,
1417         0x3c050, 0xffffffff, 0x00040007,
1418         0x3c054, 0xffffffff, 0x00060005,
1419         0x3c058, 0xffffffff, 0x00090008,
1420         0x3c05c, 0xffffffff, 0x00010000,
1421         0x3c060, 0xffffffff, 0x00030002,
1422         0x3c064, 0xffffffff, 0x00040007,
1423         0x3c068, 0xffffffff, 0x00060005,
1424         0x3c06c, 0xffffffff, 0x00090008,
1425         0x3c070, 0xffffffff, 0x00010000,
1426         0x3c074, 0xffffffff, 0x00030002,
1427         0x3c078, 0xffffffff, 0x00040007,
1428         0x3c07c, 0xffffffff, 0x00060005,
1429         0x3c080, 0xffffffff, 0x00090008,
1430         0x3c084, 0xffffffff, 0x00010000,
1431         0x3c088, 0xffffffff, 0x00030002,
1432         0x3c08c, 0xffffffff, 0x00040007,
1433         0x3c090, 0xffffffff, 0x00060005,
1434         0x3c094, 0xffffffff, 0x00090008,
1435         0x3c098, 0xffffffff, 0x00010000,
1436         0x3c09c, 0xffffffff, 0x00030002,
1437         0x3c0a0, 0xffffffff, 0x00040007,
1438         0x3c0a4, 0xffffffff, 0x00060005,
1439         0x3c0a8, 0xffffffff, 0x00090008,
1440         0x3c0ac, 0xffffffff, 0x00010000,
1441         0x3c0b0, 0xffffffff, 0x00030002,
1442         0x3c0b4, 0xffffffff, 0x00040007,
1443         0x3c0b8, 0xffffffff, 0x00060005,
1444         0x3c0bc, 0xffffffff, 0x00090008,
1445         0x3c0c0, 0xffffffff, 0x00010000,
1446         0x3c0c4, 0xffffffff, 0x00030002,
1447         0x3c0c8, 0xffffffff, 0x00040007,
1448         0x3c0cc, 0xffffffff, 0x00060005,
1449         0x3c0d0, 0xffffffff, 0x00090008,
1450         0x3c0d4, 0xffffffff, 0x00010000,
1451         0x3c0d8, 0xffffffff, 0x00030002,
1452         0x3c0dc, 0xffffffff, 0x00040007,
1453         0x3c0e0, 0xffffffff, 0x00060005,
1454         0x3c0e4, 0xffffffff, 0x00090008,
1455         0x3c0e8, 0xffffffff, 0x00010000,
1456         0x3c0ec, 0xffffffff, 0x00030002,
1457         0x3c0f0, 0xffffffff, 0x00040007,
1458         0x3c0f4, 0xffffffff, 0x00060005,
1459         0x3c0f8, 0xffffffff, 0x00090008,
1460         0xc318, 0xffffffff, 0x00020200,
1461         0x3350, 0xffffffff, 0x00000200,
1462         0x15c0, 0xffffffff, 0x00000400,
1463         0x55e8, 0xffffffff, 0x00000000,
1464         0x2f50, 0xffffffff, 0x00000902,
1465         0x3c000, 0xffffffff, 0x96940200,
1466         0x8708, 0xffffffff, 0x00900100,
1467         0xc424, 0xffffffff, 0x0020003f,
1468         0x38, 0xffffffff, 0x0140001c,
1469         0x3c, 0x000f0000, 0x000f0000,
1470         0x220, 0xffffffff, 0xc060000c,
1471         0x224, 0xc0000fff, 0x00000100,
1472         0xf90, 0xffffffff, 0x00000100,
1473         0xf98, 0x00000101, 0x00000000,
1474         0x20a8, 0xffffffff, 0x00000104,
1475         0x55e4, 0xff000fff, 0x00000100,
1476         0x30cc, 0xc0000fff, 0x00000104,
1477         0xc1e4, 0x00000001, 0x00000001,
1478         0xd00c, 0xff000ff0, 0x00000100,
1479         0xd80c, 0xff000ff0, 0x00000100
1480 };
1481
1482 static const u32 godavari_golden_registers[] =
1483 {
1484         0x55e4, 0xff607fff, 0xfc000100,
1485         0x6ed8, 0x00010101, 0x00010000,
1486         0x9830, 0xffffffff, 0x00000000,
1487         0x98302, 0xf00fffff, 0x00000400,
1488         0x6130, 0xffffffff, 0x00010000,
1489         0x5bb0, 0x000000f0, 0x00000070,
1490         0x5bc0, 0xf0311fff, 0x80300000,
1491         0x98f8, 0x73773777, 0x12010001,
1492         0x98fc, 0xffffffff, 0x00000010,
1493         0x8030, 0x00001f0f, 0x0000100a,
1494         0x2f48, 0x73773777, 0x12010001,
1495         0x2408, 0x000fffff, 0x000c007f,
1496         0x8a14, 0xf000003f, 0x00000007,
1497         0x8b24, 0xffffffff, 0x00ff0fff,
1498         0x30a04, 0x0000ff0f, 0x00000000,
1499         0x28a4c, 0x07ffffff, 0x06000000,
1500         0x4d8, 0x00000fff, 0x00000100,
1501         0xd014, 0x00010000, 0x00810001,
1502         0xd814, 0x00010000, 0x00810001,
1503         0x3e78, 0x00000001, 0x00000002,
1504         0xc768, 0x00000008, 0x00000008,
1505         0xc770, 0x00000f00, 0x00000800,
1506         0xc774, 0x00000f00, 0x00000800,
1507         0xc798, 0x00ffffff, 0x00ff7fbf,
1508         0xc79c, 0x00ffffff, 0x00ff7faf,
1509         0x8c00, 0x000000ff, 0x00000001,
1510         0x214f8, 0x01ff01ff, 0x00000002,
1511         0x21498, 0x007ff800, 0x00200000,
1512         0x2015c, 0xffffffff, 0x00000f40,
1513         0x88c4, 0x001f3ae3, 0x00000082,
1514         0x88d4, 0x0000001f, 0x00000010,
1515         0x30934, 0xffffffff, 0x00000000
1516 };
1517
1518
1519 static void cik_init_golden_registers(struct radeon_device *rdev)
1520 {
1521         switch (rdev->family) {
1522         case CHIP_BONAIRE:
1523                 radeon_program_register_sequence(rdev,
1524                                                  bonaire_mgcg_cgcg_init,
1525                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1526                 radeon_program_register_sequence(rdev,
1527                                                  bonaire_golden_registers,
1528                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1529                 radeon_program_register_sequence(rdev,
1530                                                  bonaire_golden_common_registers,
1531                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1532                 radeon_program_register_sequence(rdev,
1533                                                  bonaire_golden_spm_registers,
1534                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1535                 break;
1536         case CHIP_KABINI:
1537                 radeon_program_register_sequence(rdev,
1538                                                  kalindi_mgcg_cgcg_init,
1539                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1540                 radeon_program_register_sequence(rdev,
1541                                                  kalindi_golden_registers,
1542                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1543                 radeon_program_register_sequence(rdev,
1544                                                  kalindi_golden_common_registers,
1545                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1546                 radeon_program_register_sequence(rdev,
1547                                                  kalindi_golden_spm_registers,
1548                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1549                 break;
1550         case CHIP_MULLINS:
1551                 radeon_program_register_sequence(rdev,
1552                                                  kalindi_mgcg_cgcg_init,
1553                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1554                 radeon_program_register_sequence(rdev,
1555                                                  godavari_golden_registers,
1556                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
1557                 radeon_program_register_sequence(rdev,
1558                                                  kalindi_golden_common_registers,
1559                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1560                 radeon_program_register_sequence(rdev,
1561                                                  kalindi_golden_spm_registers,
1562                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1563                 break;
1564         case CHIP_KAVERI:
1565                 radeon_program_register_sequence(rdev,
1566                                                  spectre_mgcg_cgcg_init,
1567                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1568                 radeon_program_register_sequence(rdev,
1569                                                  spectre_golden_registers,
1570                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1571                 radeon_program_register_sequence(rdev,
1572                                                  spectre_golden_common_registers,
1573                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1574                 radeon_program_register_sequence(rdev,
1575                                                  spectre_golden_spm_registers,
1576                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1577                 break;
1578         case CHIP_HAWAII:
1579                 radeon_program_register_sequence(rdev,
1580                                                  hawaii_mgcg_cgcg_init,
1581                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1582                 radeon_program_register_sequence(rdev,
1583                                                  hawaii_golden_registers,
1584                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1585                 radeon_program_register_sequence(rdev,
1586                                                  hawaii_golden_common_registers,
1587                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1588                 radeon_program_register_sequence(rdev,
1589                                                  hawaii_golden_spm_registers,
1590                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1591                 break;
1592         default:
1593                 break;
1594         }
1595 }
1596
1597 /**
1598  * cik_get_xclk - get the xclk
1599  *
1600  * @rdev: radeon_device pointer
1601  *
1602  * Returns the reference clock used by the gfx engine
1603  * (CIK).
1604  */
1605 u32 cik_get_xclk(struct radeon_device *rdev)
1606 {
1607         u32 reference_clock = rdev->clock.spll.reference_freq;
1608
1609         if (rdev->flags & RADEON_IS_IGP) {
1610                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1611                         return reference_clock / 2;
1612         } else {
1613                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1614                         return reference_clock / 4;
1615         }
1616         return reference_clock;
1617 }
1618
1619 /**
1620  * cik_mm_rdoorbell - read a doorbell dword
1621  *
1622  * @rdev: radeon_device pointer
1623  * @index: doorbell index
1624  *
1625  * Returns the value in the doorbell aperture at the
1626  * requested doorbell index (CIK).
1627  */
1628 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1629 {
1630         if (index < rdev->doorbell.num_doorbells) {
1631                 return readl(rdev->doorbell.ptr + index);
1632         } else {
1633                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1634                 return 0;
1635         }
1636 }
1637
1638 /**
1639  * cik_mm_wdoorbell - write a doorbell dword
1640  *
1641  * @rdev: radeon_device pointer
1642  * @index: doorbell index
1643  * @v: value to write
1644  *
1645  * Writes @v to the doorbell aperture at the
1646  * requested doorbell index (CIK).
1647  */
1648 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1649 {
1650         if (index < rdev->doorbell.num_doorbells) {
1651                 writel(v, rdev->doorbell.ptr + index);
1652         } else {
1653                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1654         }
1655 }
1656
1657 #define BONAIRE_IO_MC_REGS_SIZE 36
1658
1659 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1660 {
1661         {0x00000070, 0x04400000},
1662         {0x00000071, 0x80c01803},
1663         {0x00000072, 0x00004004},
1664         {0x00000073, 0x00000100},
1665         {0x00000074, 0x00ff0000},
1666         {0x00000075, 0x34000000},
1667         {0x00000076, 0x08000014},
1668         {0x00000077, 0x00cc08ec},
1669         {0x00000078, 0x00000400},
1670         {0x00000079, 0x00000000},
1671         {0x0000007a, 0x04090000},
1672         {0x0000007c, 0x00000000},
1673         {0x0000007e, 0x4408a8e8},
1674         {0x0000007f, 0x00000304},
1675         {0x00000080, 0x00000000},
1676         {0x00000082, 0x00000001},
1677         {0x00000083, 0x00000002},
1678         {0x00000084, 0xf3e4f400},
1679         {0x00000085, 0x052024e3},
1680         {0x00000087, 0x00000000},
1681         {0x00000088, 0x01000000},
1682         {0x0000008a, 0x1c0a0000},
1683         {0x0000008b, 0xff010000},
1684         {0x0000008d, 0xffffefff},
1685         {0x0000008e, 0xfff3efff},
1686         {0x0000008f, 0xfff3efbf},
1687         {0x00000092, 0xf7ffffff},
1688         {0x00000093, 0xffffff7f},
1689         {0x00000095, 0x00101101},
1690         {0x00000096, 0x00000fff},
1691         {0x00000097, 0x00116fff},
1692         {0x00000098, 0x60010000},
1693         {0x00000099, 0x10010000},
1694         {0x0000009a, 0x00006000},
1695         {0x0000009b, 0x00001000},
1696         {0x0000009f, 0x00b48000}
1697 };
1698
1699 #define HAWAII_IO_MC_REGS_SIZE 22
1700
1701 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1702 {
1703         {0x0000007d, 0x40000000},
1704         {0x0000007e, 0x40180304},
1705         {0x0000007f, 0x0000ff00},
1706         {0x00000081, 0x00000000},
1707         {0x00000083, 0x00000800},
1708         {0x00000086, 0x00000000},
1709         {0x00000087, 0x00000100},
1710         {0x00000088, 0x00020100},
1711         {0x00000089, 0x00000000},
1712         {0x0000008b, 0x00040000},
1713         {0x0000008c, 0x00000100},
1714         {0x0000008e, 0xff010000},
1715         {0x00000090, 0xffffefff},
1716         {0x00000091, 0xfff3efff},
1717         {0x00000092, 0xfff3efbf},
1718         {0x00000093, 0xf7ffffff},
1719         {0x00000094, 0xffffff7f},
1720         {0x00000095, 0x00000fff},
1721         {0x00000096, 0x00116fff},
1722         {0x00000097, 0x60010000},
1723         {0x00000098, 0x10010000},
1724         {0x0000009f, 0x00c79000}
1725 };
1726
1727
1728 /**
1729  * cik_srbm_select - select specific register instances
1730  *
1731  * @rdev: radeon_device pointer
1732  * @me: selected ME (micro engine)
1733  * @pipe: pipe
1734  * @queue: queue
1735  * @vmid: VMID
1736  *
1737  * Switches the currently active registers instances.  Some
1738  * registers are instanced per VMID, others are instanced per
1739  * me/pipe/queue combination.
1740  */
1741 static void cik_srbm_select(struct radeon_device *rdev,
1742                             u32 me, u32 pipe, u32 queue, u32 vmid)
1743 {
1744         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1745                              MEID(me & 0x3) |
1746                              VMID(vmid & 0xf) |
1747                              QUEUEID(queue & 0x7));
1748         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1749 }
1750
1751 /* ucode loading */
1752 /**
1753  * ci_mc_load_microcode - load MC ucode into the hw
1754  *
1755  * @rdev: radeon_device pointer
1756  *
1757  * Load the GDDR MC ucode into the hw (CIK).
1758  * Returns 0 on success, error on failure.
1759  */
1760 int ci_mc_load_microcode(struct radeon_device *rdev)
1761 {
1762         const __be32 *fw_data;
1763         u32 running, blackout = 0;
1764         u32 *io_mc_regs;
1765         int i, regs_size, ucode_size;
1766
1767         if (!rdev->mc_fw)
1768                 return -EINVAL;
1769
1770         ucode_size = rdev->mc_fw->size / 4;
1771
1772         switch (rdev->family) {
1773         case CHIP_BONAIRE:
1774                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1775                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1776                 break;
1777         case CHIP_HAWAII:
1778                 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1779                 regs_size = HAWAII_IO_MC_REGS_SIZE;
1780                 break;
1781         default:
1782                 return -EINVAL;
1783         }
1784
1785         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1786
1787         if (running == 0) {
1788                 if (running) {
1789                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1790                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1791                 }
1792
1793                 /* reset the engine and set to writable */
1794                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1795                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1796
1797                 /* load mc io regs */
1798                 for (i = 0; i < regs_size; i++) {
1799                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1800                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1801                 }
1802                 /* load the MC ucode */
1803                 fw_data = (const __be32 *)rdev->mc_fw->data;
1804                 for (i = 0; i < ucode_size; i++)
1805                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1806
1807                 /* put the engine back into the active state */
1808                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1809                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1810                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1811
1812                 /* wait for training to complete */
1813                 for (i = 0; i < rdev->usec_timeout; i++) {
1814                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1815                                 break;
1816                         udelay(1);
1817                 }
1818                 for (i = 0; i < rdev->usec_timeout; i++) {
1819                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1820                                 break;
1821                         udelay(1);
1822                 }
1823
1824                 if (running)
1825                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1826         }
1827
1828         return 0;
1829 }
1830
1831 /**
1832  * cik_init_microcode - load ucode images from disk
1833  *
1834  * @rdev: radeon_device pointer
1835  *
1836  * Use the firmware interface to load the ucode images into
1837  * the driver (not loaded into hw).
1838  * Returns 0 on success, error on failure.
1839  */
1840 static int cik_init_microcode(struct radeon_device *rdev)
1841 {
1842         const char *chip_name;
1843         size_t pfp_req_size, me_req_size, ce_req_size,
1844                 mec_req_size, rlc_req_size, mc_req_size = 0,
1845                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1846         char fw_name[30];
1847         int err;
1848
1849         DRM_DEBUG("\n");
1850
1851         switch (rdev->family) {
1852         case CHIP_BONAIRE:
1853                 chip_name = "BONAIRE";
1854                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1855                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1856                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1857                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1858                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1859                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1860                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1861                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1862                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1863                 break;
1864         case CHIP_HAWAII:
1865                 chip_name = "HAWAII";
1866                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1867                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1868                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1869                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1870                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1871                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1872                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1873                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1874                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1875                 break;
1876         case CHIP_KAVERI:
1877                 chip_name = "KAVERI";
1878                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1879                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1880                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1881                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1882                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1883                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1884                 break;
1885         case CHIP_KABINI:
1886                 chip_name = "KABINI";
1887                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1888                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1889                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1890                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1891                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1892                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1893                 break;
1894         case CHIP_MULLINS:
1895                 chip_name = "MULLINS";
1896                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1897                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1898                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1899                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1900                 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1901                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1902                 break;
1903         default: BUG();
1904         }
1905
1906         DRM_INFO("Loading %s Microcode\n", chip_name);
1907
1908         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1909         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1910         if (err)
1911                 goto out;
1912         if (rdev->pfp_fw->size != pfp_req_size) {
1913                 printk(KERN_ERR
1914                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1915                        rdev->pfp_fw->size, fw_name);
1916                 err = -EINVAL;
1917                 goto out;
1918         }
1919
1920         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1921         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1922         if (err)
1923                 goto out;
1924         if (rdev->me_fw->size != me_req_size) {
1925                 printk(KERN_ERR
1926                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1927                        rdev->me_fw->size, fw_name);
1928                 err = -EINVAL;
1929         }
1930
1931         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1932         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1933         if (err)
1934                 goto out;
1935         if (rdev->ce_fw->size != ce_req_size) {
1936                 printk(KERN_ERR
1937                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1938                        rdev->ce_fw->size, fw_name);
1939                 err = -EINVAL;
1940         }
1941
1942         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1943         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1944         if (err)
1945                 goto out;
1946         if (rdev->mec_fw->size != mec_req_size) {
1947                 printk(KERN_ERR
1948                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1949                        rdev->mec_fw->size, fw_name);
1950                 err = -EINVAL;
1951         }
1952
1953         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1954         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1955         if (err)
1956                 goto out;
1957         if (rdev->rlc_fw->size != rlc_req_size) {
1958                 printk(KERN_ERR
1959                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1960                        rdev->rlc_fw->size, fw_name);
1961                 err = -EINVAL;
1962         }
1963
1964         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1965         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1966         if (err)
1967                 goto out;
1968         if (rdev->sdma_fw->size != sdma_req_size) {
1969                 printk(KERN_ERR
1970                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1971                        rdev->sdma_fw->size, fw_name);
1972                 err = -EINVAL;
1973         }
1974
1975         /* No SMC, MC ucode on APUs */
1976         if (!(rdev->flags & RADEON_IS_IGP)) {
1977                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1978                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1979                 if (err) {
1980                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1981                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1982                         if (err)
1983                                 goto out;
1984                 }
1985                 if ((rdev->mc_fw->size != mc_req_size) &&
1986                     (rdev->mc_fw->size != mc2_req_size)){
1987                         printk(KERN_ERR
1988                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1989                                rdev->mc_fw->size, fw_name);
1990                         err = -EINVAL;
1991                 }
1992                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1993
1994                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1995                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1996                 if (err) {
1997                         printk(KERN_ERR
1998                                "smc: error loading firmware \"%s\"\n",
1999                                fw_name);
2000                         release_firmware(rdev->smc_fw);
2001                         rdev->smc_fw = NULL;
2002                         err = 0;
2003                 } else if (rdev->smc_fw->size != smc_req_size) {
2004                         printk(KERN_ERR
2005                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2006                                rdev->smc_fw->size, fw_name);
2007                         err = -EINVAL;
2008                 }
2009         }
2010
2011 out:
2012         if (err) {
2013                 if (err != -EINVAL)
2014                         printk(KERN_ERR
2015                                "cik_cp: Failed to load firmware \"%s\"\n",
2016                                fw_name);
2017                 release_firmware(rdev->pfp_fw);
2018                 rdev->pfp_fw = NULL;
2019                 release_firmware(rdev->me_fw);
2020                 rdev->me_fw = NULL;
2021                 release_firmware(rdev->ce_fw);
2022                 rdev->ce_fw = NULL;
2023                 release_firmware(rdev->rlc_fw);
2024                 rdev->rlc_fw = NULL;
2025                 release_firmware(rdev->mc_fw);
2026                 rdev->mc_fw = NULL;
2027                 release_firmware(rdev->smc_fw);
2028                 rdev->smc_fw = NULL;
2029         }
2030         return err;
2031 }
2032
2033 /*
2034  * Core functions
2035  */
2036 /**
2037  * cik_tiling_mode_table_init - init the hw tiling table
2038  *
2039  * @rdev: radeon_device pointer
2040  *
2041  * Starting with SI, the tiling setup is done globally in a
2042  * set of 32 tiling modes.  Rather than selecting each set of
2043  * parameters per surface as on older asics, we just select
2044  * which index in the tiling table we want to use, and the
2045  * surface uses those parameters (CIK).
2046  */
2047 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2048 {
2049         const u32 num_tile_mode_states = 32;
2050         const u32 num_secondary_tile_mode_states = 16;
2051         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2052         u32 num_pipe_configs;
2053         u32 num_rbs = rdev->config.cik.max_backends_per_se *
2054                 rdev->config.cik.max_shader_engines;
2055
2056         switch (rdev->config.cik.mem_row_size_in_kb) {
2057         case 1:
2058                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2059                 break;
2060         case 2:
2061         default:
2062                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2063                 break;
2064         case 4:
2065                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2066                 break;
2067         }
2068
2069         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2070         if (num_pipe_configs > 8)
2071                 num_pipe_configs = 16;
2072
2073         if (num_pipe_configs == 16) {
2074                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2075                         switch (reg_offset) {
2076                         case 0:
2077                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2078                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2079                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2080                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2081                                 break;
2082                         case 1:
2083                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2084                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2085                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2086                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2087                                 break;
2088                         case 2:
2089                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2090                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2091                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2092                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2093                                 break;
2094                         case 3:
2095                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2096                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2097                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2098                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2099                                 break;
2100                         case 4:
2101                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2102                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2103                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2104                                                  TILE_SPLIT(split_equal_to_row_size));
2105                                 break;
2106                         case 5:
2107                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2108                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2109                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2110                                 break;
2111                         case 6:
2112                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2113                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2114                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2115                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2116                                 break;
2117                         case 7:
2118                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2119                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2120                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2121                                                  TILE_SPLIT(split_equal_to_row_size));
2122                                 break;
2123                         case 8:
2124                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2125                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2126                                 break;
2127                         case 9:
2128                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2129                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2130                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2131                                 break;
2132                         case 10:
2133                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2134                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2135                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2136                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2137                                 break;
2138                         case 11:
2139                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2140                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2141                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2142                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2143                                 break;
2144                         case 12:
2145                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2146                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2147                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2148                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2149                                 break;
2150                         case 13:
2151                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2152                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2153                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2154                                 break;
2155                         case 14:
2156                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2157                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2158                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2159                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2160                                 break;
2161                         case 16:
2162                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2163                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2164                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2165                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2166                                 break;
2167                         case 17:
2168                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2169                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2170                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2171                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2172                                 break;
2173                         case 27:
2174                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2175                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2176                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2177                                 break;
2178                         case 28:
2179                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2180                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2181                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2182                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2183                                 break;
2184                         case 29:
2185                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2186                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2187                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2188                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2189                                 break;
2190                         case 30:
2191                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2192                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2193                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2194                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2195                                 break;
2196                         default:
2197                                 gb_tile_moden = 0;
2198                                 break;
2199                         }
2200                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2201                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2202                 }
2203                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2204                         switch (reg_offset) {
2205                         case 0:
2206                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2208                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2209                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2210                                 break;
2211                         case 1:
2212                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2213                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2214                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2215                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2216                                 break;
2217                         case 2:
2218                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2219                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2220                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2221                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2222                                 break;
2223                         case 3:
2224                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2225                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2226                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2227                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2228                                 break;
2229                         case 4:
2230                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2231                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2232                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2233                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2234                                 break;
2235                         case 5:
2236                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2237                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2238                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2239                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2240                                 break;
2241                         case 6:
2242                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2243                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2244                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2245                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2246                                 break;
2247                         case 8:
2248                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2249                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2250                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2251                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2252                                 break;
2253                         case 9:
2254                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2255                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2256                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2257                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2258                                 break;
2259                         case 10:
2260                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2261                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2262                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2263                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2264                                 break;
2265                         case 11:
2266                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2267                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2268                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2269                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2270                                 break;
2271                         case 12:
2272                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2273                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2274                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2275                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2276                                 break;
2277                         case 13:
2278                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2279                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2280                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2281                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2282                                 break;
2283                         case 14:
2284                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2285                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2286                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2287                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2288                                 break;
2289                         default:
2290                                 gb_tile_moden = 0;
2291                                 break;
2292                         }
2293                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2294                 }
2295         } else if (num_pipe_configs == 8) {
2296                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2297                         switch (reg_offset) {
2298                         case 0:
2299                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2300                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2301                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2302                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2303                                 break;
2304                         case 1:
2305                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2306                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2307                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2308                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2309                                 break;
2310                         case 2:
2311                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2313                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2314                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2315                                 break;
2316                         case 3:
2317                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2318                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2319                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2320                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2321                                 break;
2322                         case 4:
2323                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2324                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2325                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2326                                                  TILE_SPLIT(split_equal_to_row_size));
2327                                 break;
2328                         case 5:
2329                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2330                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2332                                 break;
2333                         case 6:
2334                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2335                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2336                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2337                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2338                                 break;
2339                         case 7:
2340                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2341                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2342                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2343                                                  TILE_SPLIT(split_equal_to_row_size));
2344                                 break;
2345                         case 8:
2346                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2347                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2348                                 break;
2349                         case 9:
2350                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2351                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2352                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2353                                 break;
2354                         case 10:
2355                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2356                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2357                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2358                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2359                                 break;
2360                         case 11:
2361                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2362                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2363                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2364                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2365                                 break;
2366                         case 12:
2367                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2368                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2369                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2370                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2371                                 break;
2372                         case 13:
2373                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2374                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2375                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2376                                 break;
2377                         case 14:
2378                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2380                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2381                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2382                                 break;
2383                         case 16:
2384                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2385                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2386                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2387                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2388                                 break;
2389                         case 17:
2390                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2391                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2392                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2393                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2394                                 break;
2395                         case 27:
2396                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2397                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2398                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2399                                 break;
2400                         case 28:
2401                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2403                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2404                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405                                 break;
2406                         case 29:
2407                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2408                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2409                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2410                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2411                                 break;
2412                         case 30:
2413                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2414                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2415                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2416                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2417                                 break;
2418                         default:
2419                                 gb_tile_moden = 0;
2420                                 break;
2421                         }
2422                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2423                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2424                 }
2425                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2426                         switch (reg_offset) {
2427                         case 0:
2428                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2430                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2431                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2432                                 break;
2433                         case 1:
2434                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2436                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2437                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2438                                 break;
2439                         case 2:
2440                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2443                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2444                                 break;
2445                         case 3:
2446                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2447                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2448                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2449                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2450                                 break;
2451                         case 4:
2452                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2454                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2455                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2456                                 break;
2457                         case 5:
2458                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2460                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2461                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2462                                 break;
2463                         case 6:
2464                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2466                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2467                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2468                                 break;
2469                         case 8:
2470                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2471                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2472                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2473                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2474                                 break;
2475                         case 9:
2476                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2478                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2479                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2480                                 break;
2481                         case 10:
2482                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2483                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2484                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2485                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2486                                 break;
2487                         case 11:
2488                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2491                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2492                                 break;
2493                         case 12:
2494                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2495                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2496                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2497                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2498                                 break;
2499                         case 13:
2500                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2502                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2503                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2504                                 break;
2505                         case 14:
2506                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2507                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2508                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2509                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2510                                 break;
2511                         default:
2512                                 gb_tile_moden = 0;
2513                                 break;
2514                         }
2515                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2516                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2517                 }
2518         } else if (num_pipe_configs == 4) {
2519                 if (num_rbs == 4) {
2520                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2521                                 switch (reg_offset) {
2522                                 case 0:
2523                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2525                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2526                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2527                                         break;
2528                                 case 1:
2529                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2530                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2531                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2532                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2533                                         break;
2534                                 case 2:
2535                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2537                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2538                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2539                                         break;
2540                                 case 3:
2541                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2543                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2544                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2545                                         break;
2546                                 case 4:
2547                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2548                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2549                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2550                                                          TILE_SPLIT(split_equal_to_row_size));
2551                                         break;
2552                                 case 5:
2553                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2554                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2555                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2556                                         break;
2557                                 case 6:
2558                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2559                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2560                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2561                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2562                                         break;
2563                                 case 7:
2564                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2565                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2567                                                          TILE_SPLIT(split_equal_to_row_size));
2568                                         break;
2569                                 case 8:
2570                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2571                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2572                                         break;
2573                                 case 9:
2574                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2575                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2576                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2577                                         break;
2578                                 case 10:
2579                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2580                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2581                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2582                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2583                                         break;
2584                                 case 11:
2585                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2586                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2587                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2589                                         break;
2590                                 case 12:
2591                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2592                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2593                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2594                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2595                                         break;
2596                                 case 13:
2597                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2598                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2599                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2600                                         break;
2601                                 case 14:
2602                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2603                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2604                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2605                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2606                                         break;
2607                                 case 16:
2608                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2609                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2610                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2611                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2612                                         break;
2613                                 case 17:
2614                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2615                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2616                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2617                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2618                                         break;
2619                                 case 27:
2620                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2621                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2622                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2623                                         break;
2624                                 case 28:
2625                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2626                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2627                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2628                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2629                                         break;
2630                                 case 29:
2631                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2632                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2633                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2634                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2635                                         break;
2636                                 case 30:
2637                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2638                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2639                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2640                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2641                                         break;
2642                                 default:
2643                                         gb_tile_moden = 0;
2644                                         break;
2645                                 }
2646                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2647                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2648                         }
2649                 } else if (num_rbs < 4) {
2650                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2651                                 switch (reg_offset) {
2652                                 case 0:
2653                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2655                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2657                                         break;
2658                                 case 1:
2659                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2661                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2662                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2663                                         break;
2664                                 case 2:
2665                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2667                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2668                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2669                                         break;
2670                                 case 3:
2671                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2673                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2674                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2675                                         break;
2676                                 case 4:
2677                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2678                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2679                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2680                                                          TILE_SPLIT(split_equal_to_row_size));
2681                                         break;
2682                                 case 5:
2683                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2684                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2685                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2686                                         break;
2687                                 case 6:
2688                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2689                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2690                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2691                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2692                                         break;
2693                                 case 7:
2694                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2695                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2696                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2697                                                          TILE_SPLIT(split_equal_to_row_size));
2698                                         break;
2699                                 case 8:
2700                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2701                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2702                                         break;
2703                                 case 9:
2704                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2705                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2706                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2707                                         break;
2708                                 case 10:
2709                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2710                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2711                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2712                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2713                                         break;
2714                                 case 11:
2715                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2716                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2717                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2718                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719                                         break;
2720                                 case 12:
2721                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2722                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2723                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2724                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2725                                         break;
2726                                 case 13:
2727                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2728                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2729                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2730                                         break;
2731                                 case 14:
2732                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2733                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2734                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2735                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2736                                         break;
2737                                 case 16:
2738                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2739                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2740                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2742                                         break;
2743                                 case 17:
2744                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2745                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2746                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2747                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2748                                         break;
2749                                 case 27:
2750                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2751                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2752                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2753                                         break;
2754                                 case 28:
2755                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2756                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2757                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2759                                         break;
2760                                 case 29:
2761                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2762                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2763                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2764                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2765                                         break;
2766                                 case 30:
2767                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2768                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2769                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2770                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2771                                         break;
2772                                 default:
2773                                         gb_tile_moden = 0;
2774                                         break;
2775                                 }
2776                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2777                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2778                         }
2779                 }
2780                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2781                         switch (reg_offset) {
2782                         case 0:
2783                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2785                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2786                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2787                                 break;
2788                         case 1:
2789                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2791                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2792                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2793                                 break;
2794                         case 2:
2795                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2798                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2799                                 break;
2800                         case 3:
2801                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2802                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2803                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2804                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2805                                 break;
2806                         case 4:
2807                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2808                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2809                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2810                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2811                                 break;
2812                         case 5:
2813                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2814                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2815                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2816                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2817                                 break;
2818                         case 6:
2819                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2820                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2821                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2822                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2823                                 break;
2824                         case 8:
2825                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2826                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2827                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2828                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2829                                 break;
2830                         case 9:
2831                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2832                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2833                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2834                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2835                                 break;
2836                         case 10:
2837                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2838                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2839                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2840                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2841                                 break;
2842                         case 11:
2843                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2845                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2846                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2847                                 break;
2848                         case 12:
2849                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2850                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2851                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2852                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2853                                 break;
2854                         case 13:
2855                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2856                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2857                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2858                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2859                                 break;
2860                         case 14:
2861                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2863                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2864                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2865                                 break;
2866                         default:
2867                                 gb_tile_moden = 0;
2868                                 break;
2869                         }
2870                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2871                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2872                 }
2873         } else if (num_pipe_configs == 2) {
2874                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2875                         switch (reg_offset) {
2876                         case 0:
2877                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2879                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2880                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2881                                 break;
2882                         case 1:
2883                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2884                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2885                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2886                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2887                                 break;
2888                         case 2:
2889                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2890                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2891                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2892                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2893                                 break;
2894                         case 3:
2895                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2896                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2897                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2898                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2899                                 break;
2900                         case 4:
2901                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2902                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2903                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2904                                                  TILE_SPLIT(split_equal_to_row_size));
2905                                 break;
2906                         case 5:
2907                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2908                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2909                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2910                                 break;
2911                         case 6:
2912                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2913                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2914                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2915                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2916                                 break;
2917                         case 7:
2918                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2919                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2920                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2921                                                  TILE_SPLIT(split_equal_to_row_size));
2922                                 break;
2923                         case 8:
2924                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2925                                                 PIPE_CONFIG(ADDR_SURF_P2);
2926                                 break;
2927                         case 9:
2928                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2929                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2930                                                  PIPE_CONFIG(ADDR_SURF_P2));
2931                                 break;
2932                         case 10:
2933                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2934                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2935                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2936                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2937                                 break;
2938                         case 11:
2939                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2940                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2941                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2942                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2943                                 break;
2944                         case 12:
2945                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2946                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2947                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2948                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2949                                 break;
2950                         case 13:
2951                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2952                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2953                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2954                                 break;
2955                         case 14:
2956                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2957                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2958                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2959                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2960                                 break;
2961                         case 16:
2962                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2963                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2964                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2965                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2966                                 break;
2967                         case 17:
2968                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2969                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2970                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2971                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2972                                 break;
2973                         case 27:
2974                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2975                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2976                                                  PIPE_CONFIG(ADDR_SURF_P2));
2977                                 break;
2978                         case 28:
2979                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2980                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2981                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2982                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2983                                 break;
2984                         case 29:
2985                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2986                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2987                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2988                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2989                                 break;
2990                         case 30:
2991                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2992                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2993                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2994                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2995                                 break;
2996                         default:
2997                                 gb_tile_moden = 0;
2998                                 break;
2999                         }
3000                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3001                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3002                 }
3003                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3004                         switch (reg_offset) {
3005                         case 0:
3006                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3007                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3008                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3010                                 break;
3011                         case 1:
3012                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3013                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3014                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3015                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3016                                 break;
3017                         case 2:
3018                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3020                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3021                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3022                                 break;
3023                         case 3:
3024                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3025                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3026                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3027                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3028                                 break;
3029                         case 4:
3030                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3031                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3032                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3033                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3034                                 break;
3035                         case 5:
3036                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3037                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3038                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3039                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3040                                 break;
3041                         case 6:
3042                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3043                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3044                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3045                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3046                                 break;
3047                         case 8:
3048                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3049                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3050                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3051                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3052                                 break;
3053                         case 9:
3054                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3055                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3056                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3057                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3058                                 break;
3059                         case 10:
3060                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3061                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3062                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3063                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3064                                 break;
3065                         case 11:
3066                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3067                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3068                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3069                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3070                                 break;
3071                         case 12:
3072                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3073                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3074                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3075                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3076                                 break;
3077                         case 13:
3078                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3079                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3080                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3081                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3082                                 break;
3083                         case 14:
3084                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3085                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3086                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3087                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3088                                 break;
3089                         default:
3090                                 gb_tile_moden = 0;
3091                                 break;
3092                         }
3093                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3094                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3095                 }
3096         } else
3097                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3098 }
3099
3100 /**
3101  * cik_select_se_sh - select which SE, SH to address
3102  *
3103  * @rdev: radeon_device pointer
3104  * @se_num: shader engine to address
3105  * @sh_num: sh block to address
3106  *
3107  * Select which SE, SH combinations to address. Certain
3108  * registers are instanced per SE or SH.  0xffffffff means
3109  * broadcast to all SEs or SHs (CIK).
3110  */
3111 static void cik_select_se_sh(struct radeon_device *rdev,
3112                              u32 se_num, u32 sh_num)
3113 {
3114         u32 data = INSTANCE_BROADCAST_WRITES;
3115
3116         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3117                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3118         else if (se_num == 0xffffffff)
3119                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3120         else if (sh_num == 0xffffffff)
3121                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3122         else
3123                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3124         WREG32(GRBM_GFX_INDEX, data);
3125 }
3126
3127 /**
3128  * cik_create_bitmask - create a bitmask
3129  *
3130  * @bit_width: length of the mask
3131  *
3132  * create a variable length bit mask (CIK).
3133  * Returns the bitmask.
3134  */
3135 static u32 cik_create_bitmask(u32 bit_width)
3136 {
3137         u32 i, mask = 0;
3138
3139         for (i = 0; i < bit_width; i++) {
3140                 mask <<= 1;
3141                 mask |= 1;
3142         }
3143         return mask;
3144 }
3145
3146 /**
3147  * cik_get_rb_disabled - computes the mask of disabled RBs
3148  *
3149  * @rdev: radeon_device pointer
3150  * @max_rb_num: max RBs (render backends) for the asic
3151  * @se_num: number of SEs (shader engines) for the asic
3152  * @sh_per_se: number of SH blocks per SE for the asic
3153  *
3154  * Calculates the bitmask of disabled RBs (CIK).
3155  * Returns the disabled RB bitmask.
3156  */
3157 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3158                               u32 max_rb_num_per_se,
3159                               u32 sh_per_se)
3160 {
3161         u32 data, mask;
3162
3163         data = RREG32(CC_RB_BACKEND_DISABLE);
3164         if (data & 1)
3165                 data &= BACKEND_DISABLE_MASK;
3166         else
3167                 data = 0;
3168         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3169
3170         data >>= BACKEND_DISABLE_SHIFT;
3171
3172         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3173
3174         return data & mask;
3175 }
3176
3177 /**
3178  * cik_setup_rb - setup the RBs on the asic
3179  *
3180  * @rdev: radeon_device pointer
3181  * @se_num: number of SEs (shader engines) for the asic
3182  * @sh_per_se: number of SH blocks per SE for the asic
3183  * @max_rb_num: max RBs (render backends) for the asic
3184  *
3185  * Configures per-SE/SH RB registers (CIK).
3186  */
3187 static void cik_setup_rb(struct radeon_device *rdev,
3188                          u32 se_num, u32 sh_per_se,
3189                          u32 max_rb_num_per_se)
3190 {
3191         int i, j;
3192         u32 data, mask;
3193         u32 disabled_rbs = 0;
3194         u32 enabled_rbs = 0;
3195
3196         for (i = 0; i < se_num; i++) {
3197                 for (j = 0; j < sh_per_se; j++) {
3198                         cik_select_se_sh(rdev, i, j);
3199                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3200                         if (rdev->family == CHIP_HAWAII)
3201                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3202                         else
3203                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3204                 }
3205         }
3206         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3207
3208         mask = 1;
3209         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3210                 if (!(disabled_rbs & mask))
3211                         enabled_rbs |= mask;
3212                 mask <<= 1;
3213         }
3214
3215         rdev->config.cik.backend_enable_mask = enabled_rbs;
3216
3217         for (i = 0; i < se_num; i++) {
3218                 cik_select_se_sh(rdev, i, 0xffffffff);
3219                 data = 0;
3220                 for (j = 0; j < sh_per_se; j++) {
3221                         switch (enabled_rbs & 3) {
3222                         case 0:
3223                                 if (j == 0)
3224                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3225                                 else
3226                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3227                                 break;
3228                         case 1:
3229                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3230                                 break;
3231                         case 2:
3232                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3233                                 break;
3234                         case 3:
3235                         default:
3236                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3237                                 break;
3238                         }
3239                         enabled_rbs >>= 2;
3240                 }
3241                 WREG32(PA_SC_RASTER_CONFIG, data);
3242         }
3243         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3244 }
3245
3246 /**
3247  * cik_gpu_init - setup the 3D engine
3248  *
3249  * @rdev: radeon_device pointer
3250  *
3251  * Configures the 3D engine and tiling configuration
3252  * registers so that the 3D engine is usable.
3253  */
3254 static void cik_gpu_init(struct radeon_device *rdev)
3255 {
3256         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3257         u32 mc_shared_chmap, mc_arb_ramcfg;
3258         u32 hdp_host_path_cntl;
3259         u32 tmp;
3260         int i, j;
3261
3262         switch (rdev->family) {
3263         case CHIP_BONAIRE:
3264                 rdev->config.cik.max_shader_engines = 2;
3265                 rdev->config.cik.max_tile_pipes = 4;
3266                 rdev->config.cik.max_cu_per_sh = 7;
3267                 rdev->config.cik.max_sh_per_se = 1;
3268                 rdev->config.cik.max_backends_per_se = 2;
3269                 rdev->config.cik.max_texture_channel_caches = 4;
3270                 rdev->config.cik.max_gprs = 256;
3271                 rdev->config.cik.max_gs_threads = 32;
3272                 rdev->config.cik.max_hw_contexts = 8;
3273
3274                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3275                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3276                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3277                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3278                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3279                 break;
3280         case CHIP_HAWAII:
3281                 rdev->config.cik.max_shader_engines = 4;
3282                 rdev->config.cik.max_tile_pipes = 16;
3283                 rdev->config.cik.max_cu_per_sh = 11;
3284                 rdev->config.cik.max_sh_per_se = 1;
3285                 rdev->config.cik.max_backends_per_se = 4;
3286                 rdev->config.cik.max_texture_channel_caches = 16;
3287                 rdev->config.cik.max_gprs = 256;
3288                 rdev->config.cik.max_gs_threads = 32;
3289                 rdev->config.cik.max_hw_contexts = 8;
3290
3291                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3292                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3293                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3294                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3295                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3296                 break;
3297         case CHIP_KAVERI:
3298                 rdev->config.cik.max_shader_engines = 1;
3299                 rdev->config.cik.max_tile_pipes = 4;
3300                 if ((rdev->pdev->device == 0x1304) ||
3301                     (rdev->pdev->device == 0x1305) ||
3302                     (rdev->pdev->device == 0x130C) ||
3303                     (rdev->pdev->device == 0x130F) ||
3304                     (rdev->pdev->device == 0x1310) ||
3305                     (rdev->pdev->device == 0x1311) ||
3306                     (rdev->pdev->device == 0x131C)) {
3307                         rdev->config.cik.max_cu_per_sh = 8;
3308                         rdev->config.cik.max_backends_per_se = 2;
3309                 } else if ((rdev->pdev->device == 0x1309) ||
3310                            (rdev->pdev->device == 0x130A) ||
3311                            (rdev->pdev->device == 0x130D) ||
3312                            (rdev->pdev->device == 0x1313) ||
3313                            (rdev->pdev->device == 0x131D)) {
3314                         rdev->config.cik.max_cu_per_sh = 6;
3315                         rdev->config.cik.max_backends_per_se = 2;
3316                 } else if ((rdev->pdev->device == 0x1306) ||
3317                            (rdev->pdev->device == 0x1307) ||
3318                            (rdev->pdev->device == 0x130B) ||
3319                            (rdev->pdev->device == 0x130E) ||
3320                            (rdev->pdev->device == 0x1315) ||
3321                            (rdev->pdev->device == 0x131B)) {
3322                         rdev->config.cik.max_cu_per_sh = 4;
3323                         rdev->config.cik.max_backends_per_se = 1;
3324                 } else {
3325                         rdev->config.cik.max_cu_per_sh = 3;
3326                         rdev->config.cik.max_backends_per_se = 1;
3327                 }
3328                 rdev->config.cik.max_sh_per_se = 1;
3329                 rdev->config.cik.max_texture_channel_caches = 4;
3330                 rdev->config.cik.max_gprs = 256;
3331                 rdev->config.cik.max_gs_threads = 16;
3332                 rdev->config.cik.max_hw_contexts = 8;
3333
3334                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3335                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3336                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3337                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3338                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3339                 break;
3340         case CHIP_KABINI:
3341         case CHIP_MULLINS:
3342         default:
3343                 rdev->config.cik.max_shader_engines = 1;
3344                 rdev->config.cik.max_tile_pipes = 2;
3345                 rdev->config.cik.max_cu_per_sh = 2;
3346                 rdev->config.cik.max_sh_per_se = 1;
3347                 rdev->config.cik.max_backends_per_se = 1;
3348                 rdev->config.cik.max_texture_channel_caches = 2;
3349                 rdev->config.cik.max_gprs = 256;
3350                 rdev->config.cik.max_gs_threads = 16;
3351                 rdev->config.cik.max_hw_contexts = 8;
3352
3353                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3354                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3355                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3356                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3357                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3358                 break;
3359         }
3360
3361         /* Initialize HDP */
3362         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3363                 WREG32((0x2c14 + j), 0x00000000);
3364                 WREG32((0x2c18 + j), 0x00000000);
3365                 WREG32((0x2c1c + j), 0x00000000);
3366                 WREG32((0x2c20 + j), 0x00000000);
3367                 WREG32((0x2c24 + j), 0x00000000);
3368         }
3369
3370         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3371
3372         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3373
3374         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3375         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3376
3377         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3378         rdev->config.cik.mem_max_burst_length_bytes = 256;
3379         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3380         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3381         if (rdev->config.cik.mem_row_size_in_kb > 4)
3382                 rdev->config.cik.mem_row_size_in_kb = 4;
3383         /* XXX use MC settings? */
3384         rdev->config.cik.shader_engine_tile_size = 32;
3385         rdev->config.cik.num_gpus = 1;
3386         rdev->config.cik.multi_gpu_tile_size = 64;
3387
3388         /* fix up row size */
3389         gb_addr_config &= ~ROW_SIZE_MASK;
3390         switch (rdev->config.cik.mem_row_size_in_kb) {
3391         case 1:
3392         default:
3393                 gb_addr_config |= ROW_SIZE(0);
3394                 break;
3395         case 2:
3396                 gb_addr_config |= ROW_SIZE(1);
3397                 break;
3398         case 4:
3399                 gb_addr_config |= ROW_SIZE(2);
3400                 break;
3401         }
3402
3403         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3404          * not have bank info, so create a custom tiling dword.
3405          * bits 3:0   num_pipes
3406          * bits 7:4   num_banks
3407          * bits 11:8  group_size
3408          * bits 15:12 row_size
3409          */
3410         rdev->config.cik.tile_config = 0;
3411         switch (rdev->config.cik.num_tile_pipes) {
3412         case 1:
3413                 rdev->config.cik.tile_config |= (0 << 0);
3414                 break;
3415         case 2:
3416                 rdev->config.cik.tile_config |= (1 << 0);
3417                 break;
3418         case 4:
3419                 rdev->config.cik.tile_config |= (2 << 0);
3420                 break;
3421         case 8:
3422         default:
3423                 /* XXX what about 12? */
3424                 rdev->config.cik.tile_config |= (3 << 0);
3425                 break;
3426         }
3427         rdev->config.cik.tile_config |=
3428                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3429         rdev->config.cik.tile_config |=
3430                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3431         rdev->config.cik.tile_config |=
3432                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3433
3434         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3435         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3436         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3437         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3438         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3439         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3440         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3441         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3442
3443         cik_tiling_mode_table_init(rdev);
3444
3445         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3446                      rdev->config.cik.max_sh_per_se,
3447                      rdev->config.cik.max_backends_per_se);
3448
3449         /* set HW defaults for 3D engine */
3450         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3451
3452         WREG32(SX_DEBUG_1, 0x20);
3453
3454         WREG32(TA_CNTL_AUX, 0x00010000);
3455
3456         tmp = RREG32(SPI_CONFIG_CNTL);
3457         tmp |= 0x03000000;
3458         WREG32(SPI_CONFIG_CNTL, tmp);
3459
3460         WREG32(SQ_CONFIG, 1);
3461
3462         WREG32(DB_DEBUG, 0);
3463
3464         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3465         tmp |= 0x00000400;
3466         WREG32(DB_DEBUG2, tmp);
3467
3468         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3469         tmp |= 0x00020200;
3470         WREG32(DB_DEBUG3, tmp);
3471
3472         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3473         tmp |= 0x00018208;
3474         WREG32(CB_HW_CONTROL, tmp);
3475
3476         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3477
3478         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3479                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3480                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3481                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3482
3483         WREG32(VGT_NUM_INSTANCES, 1);
3484
3485         WREG32(CP_PERFMON_CNTL, 0);
3486
3487         WREG32(SQ_CONFIG, 0);
3488
3489         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3490                                           FORCE_EOV_MAX_REZ_CNT(255)));
3491
3492         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3493                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3494
3495         WREG32(VGT_GS_VERTEX_REUSE, 16);
3496         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3497
3498         tmp = RREG32(HDP_MISC_CNTL);
3499         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3500         WREG32(HDP_MISC_CNTL, tmp);
3501
3502         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3503         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3504
3505         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3506         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3507
3508         udelay(50);
3509 }
3510
3511 /*
3512  * GPU scratch registers helpers function.
3513  */
3514 /**
3515  * cik_scratch_init - setup driver info for CP scratch regs
3516  *
3517  * @rdev: radeon_device pointer
3518  *
3519  * Set up the number and offset of the CP scratch registers.
3520  * NOTE: use of CP scratch registers is a legacy inferface and
3521  * is not used by default on newer asics (r6xx+).  On newer asics,
3522  * memory buffers are used for fences rather than scratch regs.
3523  */
3524 static void cik_scratch_init(struct radeon_device *rdev)
3525 {
3526         int i;
3527
3528         rdev->scratch.num_reg = 7;
3529         rdev->scratch.reg_base = SCRATCH_REG0;
3530         for (i = 0; i < rdev->scratch.num_reg; i++) {
3531                 rdev->scratch.free[i] = true;
3532                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3533         }
3534 }
3535
3536 /**
3537  * cik_ring_test - basic gfx ring test
3538  *
3539  * @rdev: radeon_device pointer
3540  * @ring: radeon_ring structure holding ring information
3541  *
3542  * Allocate a scratch register and write to it using the gfx ring (CIK).
3543  * Provides a basic gfx ring test to verify that the ring is working.
3544  * Used by cik_cp_gfx_resume();
3545  * Returns 0 on success, error on failure.
3546  */
3547 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3548 {
3549         uint32_t scratch;
3550         uint32_t tmp = 0;
3551         unsigned i;
3552         int r;
3553
3554         r = radeon_scratch_get(rdev, &scratch);
3555         if (r) {
3556                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3557                 return r;
3558         }
3559         WREG32(scratch, 0xCAFEDEAD);
3560         r = radeon_ring_lock(rdev, ring, 3);
3561         if (r) {
3562                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3563                 radeon_scratch_free(rdev, scratch);
3564                 return r;
3565         }
3566         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3567         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3568         radeon_ring_write(ring, 0xDEADBEEF);
3569         radeon_ring_unlock_commit(rdev, ring);
3570
3571         for (i = 0; i < rdev->usec_timeout; i++) {
3572                 tmp = RREG32(scratch);
3573                 if (tmp == 0xDEADBEEF)
3574                         break;
3575                 DRM_UDELAY(1);
3576         }
3577         if (i < rdev->usec_timeout) {
3578                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3579         } else {
3580                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3581                           ring->idx, scratch, tmp);
3582                 r = -EINVAL;
3583         }
3584         radeon_scratch_free(rdev, scratch);
3585         return r;
3586 }
3587
3588 /**
3589  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3590  *
3591  * @rdev: radeon_device pointer
3592  * @ridx: radeon ring index
3593  *
3594  * Emits an hdp flush on the cp.
3595  */
3596 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3597                                        int ridx)
3598 {
3599         struct radeon_ring *ring = &rdev->ring[ridx];
3600         u32 ref_and_mask;
3601
3602         switch (ring->idx) {
3603         case CAYMAN_RING_TYPE_CP1_INDEX:
3604         case CAYMAN_RING_TYPE_CP2_INDEX:
3605         default:
3606                 switch (ring->me) {
3607                 case 0:
3608                         ref_and_mask = CP2 << ring->pipe;
3609                         break;
3610                 case 1:
3611                         ref_and_mask = CP6 << ring->pipe;
3612                         break;
3613                 default:
3614                         return;
3615                 }
3616                 break;
3617         case RADEON_RING_TYPE_GFX_INDEX:
3618                 ref_and_mask = CP0;
3619                 break;
3620         }
3621
3622         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3623         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3624                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3625                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3626         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3627         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3628         radeon_ring_write(ring, ref_and_mask);
3629         radeon_ring_write(ring, ref_and_mask);
3630         radeon_ring_write(ring, 0x20); /* poll interval */
3631 }
3632
3633 /**
3634  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3635  *
3636  * @rdev: radeon_device pointer
3637  * @fence: radeon fence object
3638  *
3639  * Emits a fence sequnce number on the gfx ring and flushes
3640  * GPU caches.
3641  */
3642 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3643                              struct radeon_fence *fence)
3644 {
3645         struct radeon_ring *ring = &rdev->ring[fence->ring];
3646         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3647
3648         /* EVENT_WRITE_EOP - flush caches, send int */
3649         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3650         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3651                                  EOP_TC_ACTION_EN |
3652                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3653                                  EVENT_INDEX(5)));
3654         radeon_ring_write(ring, addr & 0xfffffffc);
3655         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3656         radeon_ring_write(ring, fence->seq);
3657         radeon_ring_write(ring, 0);
3658         /* HDP flush */
3659         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3660 }
3661
3662 /**
3663  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3664  *
3665  * @rdev: radeon_device pointer
3666  * @fence: radeon fence object
3667  *
3668  * Emits a fence sequnce number on the compute ring and flushes
3669  * GPU caches.
3670  */
3671 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3672                                  struct radeon_fence *fence)
3673 {
3674         struct radeon_ring *ring = &rdev->ring[fence->ring];
3675         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3676
3677         /* RELEASE_MEM - flush caches, send int */
3678         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3679         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3680                                  EOP_TC_ACTION_EN |
3681                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3682                                  EVENT_INDEX(5)));
3683         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3684         radeon_ring_write(ring, addr & 0xfffffffc);
3685         radeon_ring_write(ring, upper_32_bits(addr));
3686         radeon_ring_write(ring, fence->seq);
3687         radeon_ring_write(ring, 0);
3688         /* HDP flush */
3689         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3690 }
3691
3692 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3693                              struct radeon_ring *ring,
3694                              struct radeon_semaphore *semaphore,
3695                              bool emit_wait)
3696 {
3697         uint64_t addr = semaphore->gpu_addr;
3698         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3699
3700         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3701         radeon_ring_write(ring, addr & 0xffffffff);
3702         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3703
3704         return true;
3705 }
3706
3707 /**
3708  * cik_copy_cpdma - copy pages using the CP DMA engine
3709  *
3710  * @rdev: radeon_device pointer
3711  * @src_offset: src GPU address
3712  * @dst_offset: dst GPU address
3713  * @num_gpu_pages: number of GPU pages to xfer
3714  * @fence: radeon fence object
3715  *
3716  * Copy GPU paging using the CP DMA engine (CIK+).
3717  * Used by the radeon ttm implementation to move pages if
3718  * registered as the asic copy callback.
3719  */
3720 int cik_copy_cpdma(struct radeon_device *rdev,
3721                    uint64_t src_offset, uint64_t dst_offset,
3722                    unsigned num_gpu_pages,
3723                    struct radeon_fence **fence)
3724 {
3725         struct radeon_semaphore *sem = NULL;
3726         int ring_index = rdev->asic->copy.blit_ring_index;
3727         struct radeon_ring *ring = &rdev->ring[ring_index];
3728         u32 size_in_bytes, cur_size_in_bytes, control;
3729         int i, num_loops;
3730         int r = 0;
3731
3732         r = radeon_semaphore_create(rdev, &sem);
3733         if (r) {
3734                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3735                 return r;
3736         }
3737
3738         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3739         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3740         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3741         if (r) {
3742                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3743                 radeon_semaphore_free(rdev, &sem, NULL);
3744                 return r;
3745         }
3746
3747         radeon_semaphore_sync_to(sem, *fence);
3748         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3749
3750         for (i = 0; i < num_loops; i++) {
3751                 cur_size_in_bytes = size_in_bytes;
3752                 if (cur_size_in_bytes > 0x1fffff)
3753                         cur_size_in_bytes = 0x1fffff;
3754                 size_in_bytes -= cur_size_in_bytes;
3755                 control = 0;
3756                 if (size_in_bytes == 0)
3757                         control |= PACKET3_DMA_DATA_CP_SYNC;
3758                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3759                 radeon_ring_write(ring, control);
3760                 radeon_ring_write(ring, lower_32_bits(src_offset));
3761                 radeon_ring_write(ring, upper_32_bits(src_offset));
3762                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3763                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3764                 radeon_ring_write(ring, cur_size_in_bytes);
3765                 src_offset += cur_size_in_bytes;
3766                 dst_offset += cur_size_in_bytes;
3767         }
3768
3769         r = radeon_fence_emit(rdev, fence, ring->idx);
3770         if (r) {
3771                 radeon_ring_unlock_undo(rdev, ring);
3772                 radeon_semaphore_free(rdev, &sem, NULL);
3773                 return r;
3774         }
3775
3776         radeon_ring_unlock_commit(rdev, ring);
3777         radeon_semaphore_free(rdev, &sem, *fence);
3778
3779         return r;
3780 }
3781
3782 /*
3783  * IB stuff
3784  */
3785 /**
3786  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3787  *
3788  * @rdev: radeon_device pointer
3789  * @ib: radeon indirect buffer object
3790  *
3791  * Emits an DE (drawing engine) or CE (constant engine) IB
3792  * on the gfx ring.  IBs are usually generated by userspace
3793  * acceleration drivers and submitted to the kernel for
3794  * sheduling on the ring.  This function schedules the IB
3795  * on the gfx ring for execution by the GPU.
3796  */
3797 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3798 {
3799         struct radeon_ring *ring = &rdev->ring[ib->ring];
3800         u32 header, control = INDIRECT_BUFFER_VALID;
3801
3802         if (ib->is_const_ib) {
3803                 /* set switch buffer packet before const IB */
3804                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3805                 radeon_ring_write(ring, 0);
3806
3807                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3808         } else {
3809                 u32 next_rptr;
3810                 if (ring->rptr_save_reg) {
3811                         next_rptr = ring->wptr + 3 + 4;
3812                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3813                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3814                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3815                         radeon_ring_write(ring, next_rptr);
3816                 } else if (rdev->wb.enabled) {
3817                         next_rptr = ring->wptr + 5 + 4;
3818                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3819                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3820                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3821                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3822                         radeon_ring_write(ring, next_rptr);
3823                 }
3824
3825                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3826         }
3827
3828         control |= ib->length_dw |
3829                 (ib->vm ? (ib->vm->id << 24) : 0);
3830
3831         radeon_ring_write(ring, header);
3832         radeon_ring_write(ring,
3833 #ifdef __BIG_ENDIAN
3834                           (2 << 0) |
3835 #endif
3836                           (ib->gpu_addr & 0xFFFFFFFC));
3837         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3838         radeon_ring_write(ring, control);
3839 }
3840
3841 /**
3842  * cik_ib_test - basic gfx ring IB test
3843  *
3844  * @rdev: radeon_device pointer
3845  * @ring: radeon_ring structure holding ring information
3846  *
3847  * Allocate an IB and execute it on the gfx ring (CIK).
3848  * Provides a basic gfx ring test to verify that IBs are working.
3849  * Returns 0 on success, error on failure.
3850  */
3851 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3852 {
3853         struct radeon_ib ib;
3854         uint32_t scratch;
3855         uint32_t tmp = 0;
3856         unsigned i;
3857         int r;
3858
3859         r = radeon_scratch_get(rdev, &scratch);
3860         if (r) {
3861                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3862                 return r;
3863         }
3864         WREG32(scratch, 0xCAFEDEAD);
3865         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3866         if (r) {
3867                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3868                 radeon_scratch_free(rdev, scratch);
3869                 return r;
3870         }
3871         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3872         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3873         ib.ptr[2] = 0xDEADBEEF;
3874         ib.length_dw = 3;
3875         r = radeon_ib_schedule(rdev, &ib, NULL);
3876         if (r) {
3877                 radeon_scratch_free(rdev, scratch);
3878                 radeon_ib_free(rdev, &ib);
3879                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3880                 return r;
3881         }
3882         r = radeon_fence_wait(ib.fence, false);
3883         if (r) {
3884                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3885                 radeon_scratch_free(rdev, scratch);
3886                 radeon_ib_free(rdev, &ib);
3887                 return r;
3888         }
3889         for (i = 0; i < rdev->usec_timeout; i++) {
3890                 tmp = RREG32(scratch);
3891                 if (tmp == 0xDEADBEEF)
3892                         break;
3893                 DRM_UDELAY(1);
3894         }
3895         if (i < rdev->usec_timeout) {
3896                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3897         } else {
3898                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3899                           scratch, tmp);
3900                 r = -EINVAL;
3901         }
3902         radeon_scratch_free(rdev, scratch);
3903         radeon_ib_free(rdev, &ib);
3904         return r;
3905 }
3906
3907 /*
3908  * CP.
3909  * On CIK, gfx and compute now have independant command processors.
3910  *
3911  * GFX
3912  * Gfx consists of a single ring and can process both gfx jobs and
3913  * compute jobs.  The gfx CP consists of three microengines (ME):
3914  * PFP - Pre-Fetch Parser
3915  * ME - Micro Engine
3916  * CE - Constant Engine
3917  * The PFP and ME make up what is considered the Drawing Engine (DE).
3918  * The CE is an asynchronous engine used for updating buffer desciptors
3919  * used by the DE so that they can be loaded into cache in parallel
3920  * while the DE is processing state update packets.
3921  *
3922  * Compute
3923  * The compute CP consists of two microengines (ME):
3924  * MEC1 - Compute MicroEngine 1
3925  * MEC2 - Compute MicroEngine 2
3926  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3927  * The queues are exposed to userspace and are programmed directly
3928  * by the compute runtime.
3929  */
3930 /**
3931  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3932  *
3933  * @rdev: radeon_device pointer
3934  * @enable: enable or disable the MEs
3935  *
3936  * Halts or unhalts the gfx MEs.
3937  */
3938 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3939 {
3940         if (enable)
3941                 WREG32(CP_ME_CNTL, 0);
3942         else {
3943                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3944                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3945                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3946                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3947         }
3948         udelay(50);
3949 }
3950
3951 /**
3952  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3953  *
3954  * @rdev: radeon_device pointer
3955  *
3956  * Loads the gfx PFP, ME, and CE ucode.
3957  * Returns 0 for success, -EINVAL if the ucode is not available.
3958  */
3959 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3960 {
3961         const __be32 *fw_data;
3962         int i;
3963
3964         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3965                 return -EINVAL;
3966
3967         cik_cp_gfx_enable(rdev, false);
3968
3969         /* PFP */
3970         fw_data = (const __be32 *)rdev->pfp_fw->data;
3971         WREG32(CP_PFP_UCODE_ADDR, 0);
3972         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3973                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3974         WREG32(CP_PFP_UCODE_ADDR, 0);
3975
3976         /* CE */
3977         fw_data = (const __be32 *)rdev->ce_fw->data;
3978         WREG32(CP_CE_UCODE_ADDR, 0);
3979         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3980                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3981         WREG32(CP_CE_UCODE_ADDR, 0);
3982
3983         /* ME */
3984         fw_data = (const __be32 *)rdev->me_fw->data;
3985         WREG32(CP_ME_RAM_WADDR, 0);
3986         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3987                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3988         WREG32(CP_ME_RAM_WADDR, 0);
3989
3990         WREG32(CP_PFP_UCODE_ADDR, 0);
3991         WREG32(CP_CE_UCODE_ADDR, 0);
3992         WREG32(CP_ME_RAM_WADDR, 0);
3993         WREG32(CP_ME_RAM_RADDR, 0);
3994         return 0;
3995 }
3996
3997 /**
3998  * cik_cp_gfx_start - start the gfx ring
3999  *
4000  * @rdev: radeon_device pointer
4001  *
4002  * Enables the ring and loads the clear state context and other
4003  * packets required to init the ring.
4004  * Returns 0 for success, error for failure.
4005  */
4006 static int cik_cp_gfx_start(struct radeon_device *rdev)
4007 {
4008         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4009         int r, i;
4010
4011         /* init the CP */
4012         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4013         WREG32(CP_ENDIAN_SWAP, 0);
4014         WREG32(CP_DEVICE_ID, 1);
4015
4016         cik_cp_gfx_enable(rdev, true);
4017
4018         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4019         if (r) {
4020                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4021                 return r;
4022         }
4023
4024         /* init the CE partitions.  CE only used for gfx on CIK */
4025         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4026         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4027         radeon_ring_write(ring, 0xc000);
4028         radeon_ring_write(ring, 0xc000);
4029
4030         /* setup clear context state */
4031         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4032         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4033
4034         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4035         radeon_ring_write(ring, 0x80000000);
4036         radeon_ring_write(ring, 0x80000000);
4037
4038         for (i = 0; i < cik_default_size; i++)
4039                 radeon_ring_write(ring, cik_default_state[i]);
4040
4041         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4042         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4043
4044         /* set clear context state */
4045         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4046         radeon_ring_write(ring, 0);
4047
4048         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4049         radeon_ring_write(ring, 0x00000316);
4050         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4051         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4052
4053         radeon_ring_unlock_commit(rdev, ring);
4054
4055         return 0;
4056 }
4057
4058 /**
4059  * cik_cp_gfx_fini - stop the gfx ring
4060  *
4061  * @rdev: radeon_device pointer
4062  *
4063  * Stop the gfx ring and tear down the driver ring
4064  * info.
4065  */
4066 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4067 {
4068         cik_cp_gfx_enable(rdev, false);
4069         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4070 }
4071
4072 /**
4073  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4074  *
4075  * @rdev: radeon_device pointer
4076  *
4077  * Program the location and size of the gfx ring buffer
4078  * and test it to make sure it's working.
4079  * Returns 0 for success, error for failure.
4080  */
4081 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4082 {
4083         struct radeon_ring *ring;
4084         u32 tmp;
4085         u32 rb_bufsz;
4086         u64 rb_addr;
4087         int r;
4088
4089         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4090         if (rdev->family != CHIP_HAWAII)
4091                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4092
4093         /* Set the write pointer delay */
4094         WREG32(CP_RB_WPTR_DELAY, 0);
4095
4096         /* set the RB to use vmid 0 */
4097         WREG32(CP_RB_VMID, 0);
4098
4099         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4100
4101         /* ring 0 - compute and gfx */
4102         /* Set ring buffer size */
4103         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4104         rb_bufsz = order_base_2(ring->ring_size / 8);
4105         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4106 #ifdef __BIG_ENDIAN
4107         tmp |= BUF_SWAP_32BIT;
4108 #endif
4109         WREG32(CP_RB0_CNTL, tmp);
4110
4111         /* Initialize the ring buffer's read and write pointers */
4112         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4113         ring->wptr = 0;
4114         WREG32(CP_RB0_WPTR, ring->wptr);
4115
4116         /* set the wb address wether it's enabled or not */
4117         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4118         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4119
4120         /* scratch register shadowing is no longer supported */
4121         WREG32(SCRATCH_UMSK, 0);
4122
4123         if (!rdev->wb.enabled)
4124                 tmp |= RB_NO_UPDATE;
4125
4126         mdelay(1);
4127         WREG32(CP_RB0_CNTL, tmp);
4128
4129         rb_addr = ring->gpu_addr >> 8;
4130         WREG32(CP_RB0_BASE, rb_addr);
4131         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4132
4133         /* start the ring */
4134         cik_cp_gfx_start(rdev);
4135         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4136         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4137         if (r) {
4138                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4139                 return r;
4140         }
4141
4142         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4143                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4144
4145         return 0;
4146 }
4147
4148 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4149                      struct radeon_ring *ring)
4150 {
4151         u32 rptr;
4152
4153         if (rdev->wb.enabled)
4154                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4155         else
4156                 rptr = RREG32(CP_RB0_RPTR);
4157
4158         return rptr;
4159 }
4160
4161 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4162                      struct radeon_ring *ring)
4163 {
4164         u32 wptr;
4165
4166         wptr = RREG32(CP_RB0_WPTR);
4167
4168         return wptr;
4169 }
4170
4171 void cik_gfx_set_wptr(struct radeon_device *rdev,
4172                       struct radeon_ring *ring)
4173 {
4174         WREG32(CP_RB0_WPTR, ring->wptr);
4175         (void)RREG32(CP_RB0_WPTR);
4176 }
4177
4178 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4179                          struct radeon_ring *ring)
4180 {
4181         u32 rptr;
4182
4183         if (rdev->wb.enabled) {
4184                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4185         } else {
4186                 mutex_lock(&rdev->srbm_mutex);
4187                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4188                 rptr = RREG32(CP_HQD_PQ_RPTR);
4189                 cik_srbm_select(rdev, 0, 0, 0, 0);
4190                 mutex_unlock(&rdev->srbm_mutex);
4191         }
4192
4193         return rptr;
4194 }
4195
4196 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4197                          struct radeon_ring *ring)
4198 {
4199         u32 wptr;
4200
4201         if (rdev->wb.enabled) {
4202                 /* XXX check if swapping is necessary on BE */
4203                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4204         } else {
4205                 mutex_lock(&rdev->srbm_mutex);
4206                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4207                 wptr = RREG32(CP_HQD_PQ_WPTR);
4208                 cik_srbm_select(rdev, 0, 0, 0, 0);
4209                 mutex_unlock(&rdev->srbm_mutex);
4210         }
4211
4212         return wptr;
4213 }
4214
4215 void cik_compute_set_wptr(struct radeon_device *rdev,
4216                           struct radeon_ring *ring)
4217 {
4218         /* XXX check if swapping is necessary on BE */
4219         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4220         WDOORBELL32(ring->doorbell_index, ring->wptr);
4221 }
4222
4223 /**
4224  * cik_cp_compute_enable - enable/disable the compute CP MEs
4225  *
4226  * @rdev: radeon_device pointer
4227  * @enable: enable or disable the MEs
4228  *
4229  * Halts or unhalts the compute MEs.
4230  */
4231 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4232 {
4233         if (enable)
4234                 WREG32(CP_MEC_CNTL, 0);
4235         else {
4236                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4237                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4238                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4239         }
4240         udelay(50);
4241 }
4242
4243 /**
4244  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4245  *
4246  * @rdev: radeon_device pointer
4247  *
4248  * Loads the compute MEC1&2 ucode.
4249  * Returns 0 for success, -EINVAL if the ucode is not available.
4250  */
4251 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4252 {
4253         const __be32 *fw_data;
4254         int i;
4255
4256         if (!rdev->mec_fw)
4257                 return -EINVAL;
4258
4259         cik_cp_compute_enable(rdev, false);
4260
4261         /* MEC1 */
4262         fw_data = (const __be32 *)rdev->mec_fw->data;
4263         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4264         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4265                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4266         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4267
4268         if (rdev->family == CHIP_KAVERI) {
4269                 /* MEC2 */
4270                 fw_data = (const __be32 *)rdev->mec_fw->data;
4271                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4272                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4273                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4274                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4275         }
4276
4277         return 0;
4278 }
4279
4280 /**
4281  * cik_cp_compute_start - start the compute queues
4282  *
4283  * @rdev: radeon_device pointer
4284  *
4285  * Enable the compute queues.
4286  * Returns 0 for success, error for failure.
4287  */
4288 static int cik_cp_compute_start(struct radeon_device *rdev)
4289 {
4290         cik_cp_compute_enable(rdev, true);
4291
4292         return 0;
4293 }
4294
4295 /**
4296  * cik_cp_compute_fini - stop the compute queues
4297  *
4298  * @rdev: radeon_device pointer
4299  *
4300  * Stop the compute queues and tear down the driver queue
4301  * info.
4302  */
4303 static void cik_cp_compute_fini(struct radeon_device *rdev)
4304 {
4305         int i, idx, r;
4306
4307         cik_cp_compute_enable(rdev, false);
4308
4309         for (i = 0; i < 2; i++) {
4310                 if (i == 0)
4311                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4312                 else
4313                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4314
4315                 if (rdev->ring[idx].mqd_obj) {
4316                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4317                         if (unlikely(r != 0))
4318                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4319
4320                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4321                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4322
4323                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4324                         rdev->ring[idx].mqd_obj = NULL;
4325                 }
4326         }
4327 }
4328
4329 static void cik_mec_fini(struct radeon_device *rdev)
4330 {
4331         int r;
4332
4333         if (rdev->mec.hpd_eop_obj) {
4334                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4335                 if (unlikely(r != 0))
4336                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4337                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4338                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4339
4340                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4341                 rdev->mec.hpd_eop_obj = NULL;
4342         }
4343 }
4344
4345 #define MEC_HPD_SIZE 2048
4346
4347 static int cik_mec_init(struct radeon_device *rdev)
4348 {
4349         int r;
4350         u32 *hpd;
4351
4352         /*
4353          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4354          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4355          */
4356         if (rdev->family == CHIP_KAVERI)
4357                 rdev->mec.num_mec = 2;
4358         else
4359                 rdev->mec.num_mec = 1;
4360         rdev->mec.num_pipe = 4;
4361         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4362
4363         if (rdev->mec.hpd_eop_obj == NULL) {
4364                 r = radeon_bo_create(rdev,
4365                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4366                                      PAGE_SIZE, true,
4367                                      RADEON_GEM_DOMAIN_GTT, NULL,
4368                                      &rdev->mec.hpd_eop_obj);
4369                 if (r) {
4370                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4371                         return r;
4372                 }
4373         }
4374
4375         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4376         if (unlikely(r != 0)) {
4377                 cik_mec_fini(rdev);
4378                 return r;
4379         }
4380         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4381                           &rdev->mec.hpd_eop_gpu_addr);
4382         if (r) {
4383                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4384                 cik_mec_fini(rdev);
4385                 return r;
4386         }
4387         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4388         if (r) {
4389                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4390                 cik_mec_fini(rdev);
4391                 return r;
4392         }
4393
4394         /* clear memory.  Not sure if this is required or not */
4395         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4396
4397         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4398         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4399
4400         return 0;
4401 }
4402
4403 struct hqd_registers
4404 {
4405         u32 cp_mqd_base_addr;
4406         u32 cp_mqd_base_addr_hi;
4407         u32 cp_hqd_active;
4408         u32 cp_hqd_vmid;
4409         u32 cp_hqd_persistent_state;
4410         u32 cp_hqd_pipe_priority;
4411         u32 cp_hqd_queue_priority;
4412         u32 cp_hqd_quantum;
4413         u32 cp_hqd_pq_base;
4414         u32 cp_hqd_pq_base_hi;
4415         u32 cp_hqd_pq_rptr;
4416         u32 cp_hqd_pq_rptr_report_addr;
4417         u32 cp_hqd_pq_rptr_report_addr_hi;
4418         u32 cp_hqd_pq_wptr_poll_addr;
4419         u32 cp_hqd_pq_wptr_poll_addr_hi;
4420         u32 cp_hqd_pq_doorbell_control;
4421         u32 cp_hqd_pq_wptr;
4422         u32 cp_hqd_pq_control;
4423         u32 cp_hqd_ib_base_addr;
4424         u32 cp_hqd_ib_base_addr_hi;
4425         u32 cp_hqd_ib_rptr;
4426         u32 cp_hqd_ib_control;
4427         u32 cp_hqd_iq_timer;
4428         u32 cp_hqd_iq_rptr;
4429         u32 cp_hqd_dequeue_request;
4430         u32 cp_hqd_dma_offload;
4431         u32 cp_hqd_sema_cmd;
4432         u32 cp_hqd_msg_type;
4433         u32 cp_hqd_atomic0_preop_lo;
4434         u32 cp_hqd_atomic0_preop_hi;
4435         u32 cp_hqd_atomic1_preop_lo;
4436         u32 cp_hqd_atomic1_preop_hi;
4437         u32 cp_hqd_hq_scheduler0;
4438         u32 cp_hqd_hq_scheduler1;
4439         u32 cp_mqd_control;
4440 };
4441
4442 struct bonaire_mqd
4443 {
4444         u32 header;
4445         u32 dispatch_initiator;
4446         u32 dimensions[3];
4447         u32 start_idx[3];
4448         u32 num_threads[3];
4449         u32 pipeline_stat_enable;
4450         u32 perf_counter_enable;
4451         u32 pgm[2];
4452         u32 tba[2];
4453         u32 tma[2];
4454         u32 pgm_rsrc[2];
4455         u32 vmid;
4456         u32 resource_limits;
4457         u32 static_thread_mgmt01[2];
4458         u32 tmp_ring_size;
4459         u32 static_thread_mgmt23[2];
4460         u32 restart[3];
4461         u32 thread_trace_enable;
4462         u32 reserved1;
4463         u32 user_data[16];
4464         u32 vgtcs_invoke_count[2];
4465         struct hqd_registers queue_state;
4466         u32 dequeue_cntr;
4467         u32 interrupt_queue[64];
4468 };
4469
4470 /**
4471  * cik_cp_compute_resume - setup the compute queue registers
4472  *
4473  * @rdev: radeon_device pointer
4474  *
4475  * Program the compute queues and test them to make sure they
4476  * are working.
4477  * Returns 0 for success, error for failure.
4478  */
4479 static int cik_cp_compute_resume(struct radeon_device *rdev)
4480 {
4481         int r, i, idx;
4482         u32 tmp;
4483         bool use_doorbell = true;
4484         u64 hqd_gpu_addr;
4485         u64 mqd_gpu_addr;
4486         u64 eop_gpu_addr;
4487         u64 wb_gpu_addr;
4488         u32 *buf;
4489         struct bonaire_mqd *mqd;
4490
4491         r = cik_cp_compute_start(rdev);
4492         if (r)
4493                 return r;
4494
4495         /* fix up chicken bits */
4496         tmp = RREG32(CP_CPF_DEBUG);
4497         tmp |= (1 << 23);
4498         WREG32(CP_CPF_DEBUG, tmp);
4499
4500         /* init the pipes */
4501         mutex_lock(&rdev->srbm_mutex);
4502         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4503                 int me = (i < 4) ? 1 : 2;
4504                 int pipe = (i < 4) ? i : (i - 4);
4505
4506                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4507
4508                 cik_srbm_select(rdev, me, pipe, 0, 0);
4509
4510                 /* write the EOP addr */
4511                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4512                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4513
4514                 /* set the VMID assigned */
4515                 WREG32(CP_HPD_EOP_VMID, 0);
4516
4517                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4518                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4519                 tmp &= ~EOP_SIZE_MASK;
4520                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4521                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4522         }
4523         cik_srbm_select(rdev, 0, 0, 0, 0);
4524         mutex_unlock(&rdev->srbm_mutex);
4525
4526         /* init the queues.  Just two for now. */
4527         for (i = 0; i < 2; i++) {
4528                 if (i == 0)
4529                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4530                 else
4531                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4532
4533                 if (rdev->ring[idx].mqd_obj == NULL) {
4534                         r = radeon_bo_create(rdev,
4535                                              sizeof(struct bonaire_mqd),
4536                                              PAGE_SIZE, true,
4537                                              RADEON_GEM_DOMAIN_GTT, NULL,
4538                                              &rdev->ring[idx].mqd_obj);
4539                         if (r) {
4540                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4541                                 return r;
4542                         }
4543                 }
4544
4545                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4546                 if (unlikely(r != 0)) {
4547                         cik_cp_compute_fini(rdev);
4548                         return r;
4549                 }
4550                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4551                                   &mqd_gpu_addr);
4552                 if (r) {
4553                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4554                         cik_cp_compute_fini(rdev);
4555                         return r;
4556                 }
4557                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4558                 if (r) {
4559                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4560                         cik_cp_compute_fini(rdev);
4561                         return r;
4562                 }
4563
4564                 /* init the mqd struct */
4565                 memset(buf, 0, sizeof(struct bonaire_mqd));
4566
4567                 mqd = (struct bonaire_mqd *)buf;
4568                 mqd->header = 0xC0310800;
4569                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4570                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4571                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4572                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4573
4574                 mutex_lock(&rdev->srbm_mutex);
4575                 cik_srbm_select(rdev, rdev->ring[idx].me,
4576                                 rdev->ring[idx].pipe,
4577                                 rdev->ring[idx].queue, 0);
4578
4579                 /* disable wptr polling */
4580                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4581                 tmp &= ~WPTR_POLL_EN;
4582                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4583
4584                 /* enable doorbell? */
4585                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4586                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4587                 if (use_doorbell)
4588                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4589                 else
4590                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4591                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4592                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4593
4594                 /* disable the queue if it's active */
4595                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4596                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4597                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4598                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4599                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4600                         for (i = 0; i < rdev->usec_timeout; i++) {
4601                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4602                                         break;
4603                                 udelay(1);
4604                         }
4605                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4606                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4607                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4608                 }
4609
4610                 /* set the pointer to the MQD */
4611                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4612                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4613                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4614                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4615                 /* set MQD vmid to 0 */
4616                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4617                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4618                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4619
4620                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4621                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4622                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4623                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4624                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4625                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4626
4627                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4628                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4629                 mqd->queue_state.cp_hqd_pq_control &=
4630                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4631
4632                 mqd->queue_state.cp_hqd_pq_control |=
4633                         order_base_2(rdev->ring[idx].ring_size / 8);
4634                 mqd->queue_state.cp_hqd_pq_control |=
4635                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4636 #ifdef __BIG_ENDIAN
4637                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4638 #endif
4639                 mqd->queue_state.cp_hqd_pq_control &=
4640                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4641                 mqd->queue_state.cp_hqd_pq_control |=
4642                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4643                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4644
4645                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4646                 if (i == 0)
4647                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4648                 else
4649                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4650                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4651                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4652                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4653                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4654                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4655
4656                 /* set the wb address wether it's enabled or not */
4657                 if (i == 0)
4658                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4659                 else
4660                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4661                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4662                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4663                         upper_32_bits(wb_gpu_addr) & 0xffff;
4664                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4665                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4666                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4667                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4668
4669                 /* enable the doorbell if requested */
4670                 if (use_doorbell) {
4671                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4672                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4673                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4674                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4675                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4676                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4677                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4678                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4679
4680                 } else {
4681                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4682                 }
4683                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4684                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4685
4686                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4687                 rdev->ring[idx].wptr = 0;
4688                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4689                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4690                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4691
4692                 /* set the vmid for the queue */
4693                 mqd->queue_state.cp_hqd_vmid = 0;
4694                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4695
4696                 /* activate the queue */
4697                 mqd->queue_state.cp_hqd_active = 1;
4698                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4699
4700                 cik_srbm_select(rdev, 0, 0, 0, 0);
4701                 mutex_unlock(&rdev->srbm_mutex);
4702
4703                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4704                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4705
4706                 rdev->ring[idx].ready = true;
4707                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4708                 if (r)
4709                         rdev->ring[idx].ready = false;
4710         }
4711
4712         return 0;
4713 }
4714
4715 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4716 {
4717         cik_cp_gfx_enable(rdev, enable);
4718         cik_cp_compute_enable(rdev, enable);
4719 }
4720
4721 static int cik_cp_load_microcode(struct radeon_device *rdev)
4722 {
4723         int r;
4724
4725         r = cik_cp_gfx_load_microcode(rdev);
4726         if (r)
4727                 return r;
4728         r = cik_cp_compute_load_microcode(rdev);
4729         if (r)
4730                 return r;
4731
4732         return 0;
4733 }
4734
4735 static void cik_cp_fini(struct radeon_device *rdev)
4736 {
4737         cik_cp_gfx_fini(rdev);
4738         cik_cp_compute_fini(rdev);
4739 }
4740
4741 static int cik_cp_resume(struct radeon_device *rdev)
4742 {
4743         int r;
4744
4745         cik_enable_gui_idle_interrupt(rdev, false);
4746
4747         r = cik_cp_load_microcode(rdev);
4748         if (r)
4749                 return r;
4750
4751         r = cik_cp_gfx_resume(rdev);
4752         if (r)
4753                 return r;
4754         r = cik_cp_compute_resume(rdev);
4755         if (r)
4756                 return r;
4757
4758         cik_enable_gui_idle_interrupt(rdev, true);
4759
4760         return 0;
4761 }
4762
4763 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4764 {
4765         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4766                 RREG32(GRBM_STATUS));
4767         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4768                 RREG32(GRBM_STATUS2));
4769         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4770                 RREG32(GRBM_STATUS_SE0));
4771         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4772                 RREG32(GRBM_STATUS_SE1));
4773         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4774                 RREG32(GRBM_STATUS_SE2));
4775         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4776                 RREG32(GRBM_STATUS_SE3));
4777         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4778                 RREG32(SRBM_STATUS));
4779         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4780                 RREG32(SRBM_STATUS2));
4781         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4782                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4783         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4784                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4785         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4786         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4787                  RREG32(CP_STALLED_STAT1));
4788         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4789                  RREG32(CP_STALLED_STAT2));
4790         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4791                  RREG32(CP_STALLED_STAT3));
4792         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4793                  RREG32(CP_CPF_BUSY_STAT));
4794         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4795                  RREG32(CP_CPF_STALLED_STAT1));
4796         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4797         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4798         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4799                  RREG32(CP_CPC_STALLED_STAT1));
4800         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4801 }
4802
4803 /**
4804  * cik_gpu_check_soft_reset - check which blocks are busy
4805  *
4806  * @rdev: radeon_device pointer
4807  *
4808  * Check which blocks are busy and return the relevant reset
4809  * mask to be used by cik_gpu_soft_reset().
4810  * Returns a mask of the blocks to be reset.
4811  */
4812 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4813 {
4814         u32 reset_mask = 0;
4815         u32 tmp;
4816
4817         /* GRBM_STATUS */
4818         tmp = RREG32(GRBM_STATUS);
4819         if (tmp & (PA_BUSY | SC_BUSY |
4820                    BCI_BUSY | SX_BUSY |
4821                    TA_BUSY | VGT_BUSY |
4822                    DB_BUSY | CB_BUSY |
4823                    GDS_BUSY | SPI_BUSY |
4824                    IA_BUSY | IA_BUSY_NO_DMA))
4825                 reset_mask |= RADEON_RESET_GFX;
4826
4827         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4828                 reset_mask |= RADEON_RESET_CP;
4829
4830         /* GRBM_STATUS2 */
4831         tmp = RREG32(GRBM_STATUS2);
4832         if (tmp & RLC_BUSY)
4833                 reset_mask |= RADEON_RESET_RLC;
4834
4835         /* SDMA0_STATUS_REG */
4836         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4837         if (!(tmp & SDMA_IDLE))
4838                 reset_mask |= RADEON_RESET_DMA;
4839
4840         /* SDMA1_STATUS_REG */
4841         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4842         if (!(tmp & SDMA_IDLE))
4843                 reset_mask |= RADEON_RESET_DMA1;
4844
4845         /* SRBM_STATUS2 */
4846         tmp = RREG32(SRBM_STATUS2);
4847         if (tmp & SDMA_BUSY)
4848                 reset_mask |= RADEON_RESET_DMA;
4849
4850         if (tmp & SDMA1_BUSY)
4851                 reset_mask |= RADEON_RESET_DMA1;
4852
4853         /* SRBM_STATUS */
4854         tmp = RREG32(SRBM_STATUS);
4855
4856         if (tmp & IH_BUSY)
4857                 reset_mask |= RADEON_RESET_IH;
4858
4859         if (tmp & SEM_BUSY)
4860                 reset_mask |= RADEON_RESET_SEM;
4861
4862         if (tmp & GRBM_RQ_PENDING)
4863                 reset_mask |= RADEON_RESET_GRBM;
4864
4865         if (tmp & VMC_BUSY)
4866                 reset_mask |= RADEON_RESET_VMC;
4867
4868         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4869                    MCC_BUSY | MCD_BUSY))
4870                 reset_mask |= RADEON_RESET_MC;
4871
4872         if (evergreen_is_display_hung(rdev))
4873                 reset_mask |= RADEON_RESET_DISPLAY;
4874
4875         /* Skip MC reset as it's mostly likely not hung, just busy */
4876         if (reset_mask & RADEON_RESET_MC) {
4877                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4878                 reset_mask &= ~RADEON_RESET_MC;
4879         }
4880
4881         return reset_mask;
4882 }
4883
4884 /**
4885  * cik_gpu_soft_reset - soft reset GPU
4886  *
4887  * @rdev: radeon_device pointer
4888  * @reset_mask: mask of which blocks to reset
4889  *
4890  * Soft reset the blocks specified in @reset_mask.
4891  */
4892 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4893 {
4894         struct evergreen_mc_save save;
4895         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4896         u32 tmp;
4897
4898         if (reset_mask == 0)
4899                 return;
4900
4901         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4902
4903         cik_print_gpu_status_regs(rdev);
4904         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4905                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4906         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4907                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4908
4909         /* disable CG/PG */
4910         cik_fini_pg(rdev);
4911         cik_fini_cg(rdev);
4912
4913         /* stop the rlc */
4914         cik_rlc_stop(rdev);
4915
4916         /* Disable GFX parsing/prefetching */
4917         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4918
4919         /* Disable MEC parsing/prefetching */
4920         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4921
4922         if (reset_mask & RADEON_RESET_DMA) {
4923                 /* sdma0 */
4924                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4925                 tmp |= SDMA_HALT;
4926                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4927         }
4928         if (reset_mask & RADEON_RESET_DMA1) {
4929                 /* sdma1 */
4930                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4931                 tmp |= SDMA_HALT;
4932                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4933         }
4934
4935         evergreen_mc_stop(rdev, &save);
4936         if (evergreen_mc_wait_for_idle(rdev)) {
4937                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4938         }
4939
4940         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4941                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4942
4943         if (reset_mask & RADEON_RESET_CP) {
4944                 grbm_soft_reset |= SOFT_RESET_CP;
4945
4946                 srbm_soft_reset |= SOFT_RESET_GRBM;
4947         }
4948
4949         if (reset_mask & RADEON_RESET_DMA)
4950                 srbm_soft_reset |= SOFT_RESET_SDMA;
4951
4952         if (reset_mask & RADEON_RESET_DMA1)
4953                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4954
4955         if (reset_mask & RADEON_RESET_DISPLAY)
4956                 srbm_soft_reset |= SOFT_RESET_DC;
4957
4958         if (reset_mask & RADEON_RESET_RLC)
4959                 grbm_soft_reset |= SOFT_RESET_RLC;
4960
4961         if (reset_mask & RADEON_RESET_SEM)
4962                 srbm_soft_reset |= SOFT_RESET_SEM;
4963
4964         if (reset_mask & RADEON_RESET_IH)
4965                 srbm_soft_reset |= SOFT_RESET_IH;
4966
4967         if (reset_mask & RADEON_RESET_GRBM)
4968                 srbm_soft_reset |= SOFT_RESET_GRBM;
4969
4970         if (reset_mask & RADEON_RESET_VMC)
4971                 srbm_soft_reset |= SOFT_RESET_VMC;
4972
4973         if (!(rdev->flags & RADEON_IS_IGP)) {
4974                 if (reset_mask & RADEON_RESET_MC)
4975                         srbm_soft_reset |= SOFT_RESET_MC;
4976         }
4977
4978         if (grbm_soft_reset) {
4979                 tmp = RREG32(GRBM_SOFT_RESET);
4980                 tmp |= grbm_soft_reset;
4981                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4982                 WREG32(GRBM_SOFT_RESET, tmp);
4983                 tmp = RREG32(GRBM_SOFT_RESET);
4984
4985                 udelay(50);
4986
4987                 tmp &= ~grbm_soft_reset;
4988                 WREG32(GRBM_SOFT_RESET, tmp);
4989                 tmp = RREG32(GRBM_SOFT_RESET);
4990         }
4991
4992         if (srbm_soft_reset) {
4993                 tmp = RREG32(SRBM_SOFT_RESET);
4994                 tmp |= srbm_soft_reset;
4995                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4996                 WREG32(SRBM_SOFT_RESET, tmp);
4997                 tmp = RREG32(SRBM_SOFT_RESET);
4998
4999                 udelay(50);
5000
5001                 tmp &= ~srbm_soft_reset;
5002                 WREG32(SRBM_SOFT_RESET, tmp);
5003                 tmp = RREG32(SRBM_SOFT_RESET);
5004         }
5005
5006         /* Wait a little for things to settle down */
5007         udelay(50);
5008
5009         evergreen_mc_resume(rdev, &save);
5010         udelay(50);
5011
5012         cik_print_gpu_status_regs(rdev);
5013 }
5014
5015 struct kv_reset_save_regs {
5016         u32 gmcon_reng_execute;
5017         u32 gmcon_misc;
5018         u32 gmcon_misc3;
5019 };
5020
5021 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5022                                    struct kv_reset_save_regs *save)
5023 {
5024         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5025         save->gmcon_misc = RREG32(GMCON_MISC);
5026         save->gmcon_misc3 = RREG32(GMCON_MISC3);
5027
5028         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5029         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5030                                                 STCTRL_STUTTER_EN));
5031 }
5032
5033 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5034                                       struct kv_reset_save_regs *save)
5035 {
5036         int i;
5037
5038         WREG32(GMCON_PGFSM_WRITE, 0);
5039         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5040
5041         for (i = 0; i < 5; i++)
5042                 WREG32(GMCON_PGFSM_WRITE, 0);
5043
5044         WREG32(GMCON_PGFSM_WRITE, 0);
5045         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5046
5047         for (i = 0; i < 5; i++)
5048                 WREG32(GMCON_PGFSM_WRITE, 0);
5049
5050         WREG32(GMCON_PGFSM_WRITE, 0x210000);
5051         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5052
5053         for (i = 0; i < 5; i++)
5054                 WREG32(GMCON_PGFSM_WRITE, 0);
5055
5056         WREG32(GMCON_PGFSM_WRITE, 0x21003);
5057         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5058
5059         for (i = 0; i < 5; i++)
5060                 WREG32(GMCON_PGFSM_WRITE, 0);
5061
5062         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5063         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5064
5065         for (i = 0; i < 5; i++)
5066                 WREG32(GMCON_PGFSM_WRITE, 0);
5067
5068         WREG32(GMCON_PGFSM_WRITE, 0);
5069         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5070
5071         for (i = 0; i < 5; i++)
5072                 WREG32(GMCON_PGFSM_WRITE, 0);
5073
5074         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5075         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5076
5077         for (i = 0; i < 5; i++)
5078                 WREG32(GMCON_PGFSM_WRITE, 0);
5079
5080         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5081         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5082
5083         for (i = 0; i < 5; i++)
5084                 WREG32(GMCON_PGFSM_WRITE, 0);
5085
5086         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5087         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5088
5089         for (i = 0; i < 5; i++)
5090                 WREG32(GMCON_PGFSM_WRITE, 0);
5091
5092         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5093         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5094
5095         for (i = 0; i < 5; i++)
5096                 WREG32(GMCON_PGFSM_WRITE, 0);
5097
5098         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5099         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5100
5101         WREG32(GMCON_MISC3, save->gmcon_misc3);
5102         WREG32(GMCON_MISC, save->gmcon_misc);
5103         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5104 }
5105
5106 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5107 {
5108         struct evergreen_mc_save save;
5109         struct kv_reset_save_regs kv_save = { 0 };
5110         u32 tmp, i;
5111
5112         dev_info(rdev->dev, "GPU pci config reset\n");
5113
5114         /* disable dpm? */
5115
5116         /* disable cg/pg */
5117         cik_fini_pg(rdev);
5118         cik_fini_cg(rdev);
5119
5120         /* Disable GFX parsing/prefetching */
5121         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5122
5123         /* Disable MEC parsing/prefetching */
5124         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5125
5126         /* sdma0 */
5127         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5128         tmp |= SDMA_HALT;
5129         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5130         /* sdma1 */
5131         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5132         tmp |= SDMA_HALT;
5133         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5134         /* XXX other engines? */
5135
5136         /* halt the rlc, disable cp internal ints */
5137         cik_rlc_stop(rdev);
5138
5139         udelay(50);
5140
5141         /* disable mem access */
5142         evergreen_mc_stop(rdev, &save);
5143         if (evergreen_mc_wait_for_idle(rdev)) {
5144                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5145         }
5146
5147         if (rdev->flags & RADEON_IS_IGP)
5148                 kv_save_regs_for_reset(rdev, &kv_save);
5149
5150         /* disable BM */
5151         pci_clear_master(rdev->pdev);
5152         /* reset */
5153         radeon_pci_config_reset(rdev);
5154
5155         udelay(100);
5156
5157         /* wait for asic to come out of reset */
5158         for (i = 0; i < rdev->usec_timeout; i++) {
5159                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5160                         break;
5161                 udelay(1);
5162         }
5163
5164         /* does asic init need to be run first??? */
5165         if (rdev->flags & RADEON_IS_IGP)
5166                 kv_restore_regs_for_reset(rdev, &kv_save);
5167 }
5168
5169 /**
5170  * cik_asic_reset - soft reset GPU
5171  *
5172  * @rdev: radeon_device pointer
5173  *
5174  * Look up which blocks are hung and attempt
5175  * to reset them.
5176  * Returns 0 for success.
5177  */
5178 int cik_asic_reset(struct radeon_device *rdev)
5179 {
5180         u32 reset_mask;
5181
5182         reset_mask = cik_gpu_check_soft_reset(rdev);
5183
5184         if (reset_mask)
5185                 r600_set_bios_scratch_engine_hung(rdev, true);
5186
5187         /* try soft reset */
5188         cik_gpu_soft_reset(rdev, reset_mask);
5189
5190         reset_mask = cik_gpu_check_soft_reset(rdev);
5191
5192         /* try pci config reset */
5193         if (reset_mask && radeon_hard_reset)
5194                 cik_gpu_pci_config_reset(rdev);
5195
5196         reset_mask = cik_gpu_check_soft_reset(rdev);
5197
5198         if (!reset_mask)
5199                 r600_set_bios_scratch_engine_hung(rdev, false);
5200
5201         return 0;
5202 }
5203
5204 /**
5205  * cik_gfx_is_lockup - check if the 3D engine is locked up
5206  *
5207  * @rdev: radeon_device pointer
5208  * @ring: radeon_ring structure holding ring information
5209  *
5210  * Check if the 3D engine is locked up (CIK).
5211  * Returns true if the engine is locked, false if not.
5212  */
5213 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5214 {
5215         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5216
5217         if (!(reset_mask & (RADEON_RESET_GFX |
5218                             RADEON_RESET_COMPUTE |
5219                             RADEON_RESET_CP))) {
5220                 radeon_ring_lockup_update(rdev, ring);
5221                 return false;
5222         }
5223         return radeon_ring_test_lockup(rdev, ring);
5224 }
5225
5226 /* MC */
5227 /**
5228  * cik_mc_program - program the GPU memory controller
5229  *
5230  * @rdev: radeon_device pointer
5231  *
5232  * Set the location of vram, gart, and AGP in the GPU's
5233  * physical address space (CIK).
5234  */
5235 static void cik_mc_program(struct radeon_device *rdev)
5236 {
5237         struct evergreen_mc_save save;
5238         u32 tmp;
5239         int i, j;
5240
5241         /* Initialize HDP */
5242         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5243                 WREG32((0x2c14 + j), 0x00000000);
5244                 WREG32((0x2c18 + j), 0x00000000);
5245                 WREG32((0x2c1c + j), 0x00000000);
5246                 WREG32((0x2c20 + j), 0x00000000);
5247                 WREG32((0x2c24 + j), 0x00000000);
5248         }
5249         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5250
5251         evergreen_mc_stop(rdev, &save);
5252         if (radeon_mc_wait_for_idle(rdev)) {
5253                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5254         }
5255         /* Lockout access through VGA aperture*/
5256         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5257         /* Update configuration */
5258         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5259                rdev->mc.vram_start >> 12);
5260         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5261                rdev->mc.vram_end >> 12);
5262         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5263                rdev->vram_scratch.gpu_addr >> 12);
5264         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5265         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5266         WREG32(MC_VM_FB_LOCATION, tmp);
5267         /* XXX double check these! */
5268         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5269         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5270         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5271         WREG32(MC_VM_AGP_BASE, 0);
5272         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5273         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5274         if (radeon_mc_wait_for_idle(rdev)) {
5275                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5276         }
5277         evergreen_mc_resume(rdev, &save);
5278         /* we need to own VRAM, so turn off the VGA renderer here
5279          * to stop it overwriting our objects */
5280         rv515_vga_render_disable(rdev);
5281 }
5282
5283 /**
5284  * cik_mc_init - initialize the memory controller driver params
5285  *
5286  * @rdev: radeon_device pointer
5287  *
5288  * Look up the amount of vram, vram width, and decide how to place
5289  * vram and gart within the GPU's physical address space (CIK).
5290  * Returns 0 for success.
5291  */
5292 static int cik_mc_init(struct radeon_device *rdev)
5293 {
5294         u32 tmp;
5295         int chansize, numchan;
5296
5297         /* Get VRAM informations */
5298         rdev->mc.vram_is_ddr = true;
5299         tmp = RREG32(MC_ARB_RAMCFG);
5300         if (tmp & CHANSIZE_MASK) {
5301                 chansize = 64;
5302         } else {
5303                 chansize = 32;
5304         }
5305         tmp = RREG32(MC_SHARED_CHMAP);
5306         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5307         case 0:
5308         default:
5309                 numchan = 1;
5310                 break;
5311         case 1:
5312                 numchan = 2;
5313                 break;
5314         case 2:
5315                 numchan = 4;
5316                 break;
5317         case 3:
5318                 numchan = 8;
5319                 break;
5320         case 4:
5321                 numchan = 3;
5322                 break;
5323         case 5:
5324                 numchan = 6;
5325                 break;
5326         case 6:
5327                 numchan = 10;
5328                 break;
5329         case 7:
5330                 numchan = 12;
5331                 break;
5332         case 8:
5333                 numchan = 16;
5334                 break;
5335         }
5336         rdev->mc.vram_width = numchan * chansize;
5337         /* Could aper size report 0 ? */
5338         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5339         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5340         /* size in MB on si */
5341         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5342         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5343         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5344         si_vram_gtt_location(rdev, &rdev->mc);
5345         radeon_update_bandwidth_info(rdev);
5346
5347         return 0;
5348 }
5349
5350 /*
5351  * GART
5352  * VMID 0 is the physical GPU addresses as used by the kernel.
5353  * VMIDs 1-15 are used for userspace clients and are handled
5354  * by the radeon vm/hsa code.
5355  */
5356 /**
5357  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5358  *
5359  * @rdev: radeon_device pointer
5360  *
5361  * Flush the TLB for the VMID 0 page table (CIK).
5362  */
5363 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5364 {
5365         /* flush hdp cache */
5366         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5367
5368         /* bits 0-15 are the VM contexts0-15 */
5369         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5370 }
5371
5372 /**
5373  * cik_pcie_gart_enable - gart enable
5374  *
5375  * @rdev: radeon_device pointer
5376  *
5377  * This sets up the TLBs, programs the page tables for VMID0,
5378  * sets up the hw for VMIDs 1-15 which are allocated on
5379  * demand, and sets up the global locations for the LDS, GDS,
5380  * and GPUVM for FSA64 clients (CIK).
5381  * Returns 0 for success, errors for failure.
5382  */
5383 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5384 {
5385         int r, i;
5386
5387         if (rdev->gart.robj == NULL) {
5388                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5389                 return -EINVAL;
5390         }
5391         r = radeon_gart_table_vram_pin(rdev);
5392         if (r)
5393                 return r;
5394         radeon_gart_restore(rdev);
5395         /* Setup TLB control */
5396         WREG32(MC_VM_MX_L1_TLB_CNTL,
5397                (0xA << 7) |
5398                ENABLE_L1_TLB |
5399                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5400                ENABLE_ADVANCED_DRIVER_MODEL |
5401                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5402         /* Setup L2 cache */
5403         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5404                ENABLE_L2_FRAGMENT_PROCESSING |
5405                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5406                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5407                EFFECTIVE_L2_QUEUE_SIZE(7) |
5408                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5409         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5410         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5411                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5412         /* setup context0 */
5413         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5414         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5415         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5416         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5417                         (u32)(rdev->dummy_page.addr >> 12));
5418         WREG32(VM_CONTEXT0_CNTL2, 0);
5419         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5420                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5421
5422         WREG32(0x15D4, 0);
5423         WREG32(0x15D8, 0);
5424         WREG32(0x15DC, 0);
5425
5426         /* empty context1-15 */
5427         /* FIXME start with 4G, once using 2 level pt switch to full
5428          * vm size space
5429          */
5430         /* set vm size, must be a multiple of 4 */
5431         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5432         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5433         for (i = 1; i < 16; i++) {
5434                 if (i < 8)
5435                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5436                                rdev->gart.table_addr >> 12);
5437                 else
5438                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5439                                rdev->gart.table_addr >> 12);
5440         }
5441
5442         /* enable context1-15 */
5443         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5444                (u32)(rdev->dummy_page.addr >> 12));
5445         WREG32(VM_CONTEXT1_CNTL2, 4);
5446         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5447                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5448                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5449                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5450                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5451                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5452                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5453                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5454                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5455                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5456                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5457                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5458                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5459
5460         if (rdev->family == CHIP_KAVERI) {
5461                 u32 tmp = RREG32(CHUB_CONTROL);
5462                 tmp &= ~BYPASS_VM;
5463                 WREG32(CHUB_CONTROL, tmp);
5464         }
5465
5466         /* XXX SH_MEM regs */
5467         /* where to put LDS, scratch, GPUVM in FSA64 space */
5468         mutex_lock(&rdev->srbm_mutex);
5469         for (i = 0; i < 16; i++) {
5470                 cik_srbm_select(rdev, 0, 0, 0, i);
5471                 /* CP and shaders */
5472                 WREG32(SH_MEM_CONFIG, 0);
5473                 WREG32(SH_MEM_APE1_BASE, 1);
5474                 WREG32(SH_MEM_APE1_LIMIT, 0);
5475                 WREG32(SH_MEM_BASES, 0);
5476                 /* SDMA GFX */
5477                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5478                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5479                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5480                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5481                 /* XXX SDMA RLC - todo */
5482         }
5483         cik_srbm_select(rdev, 0, 0, 0, 0);
5484         mutex_unlock(&rdev->srbm_mutex);
5485
5486         cik_pcie_gart_tlb_flush(rdev);
5487         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5488                  (unsigned)(rdev->mc.gtt_size >> 20),
5489                  (unsigned long long)rdev->gart.table_addr);
5490         rdev->gart.ready = true;
5491         return 0;
5492 }
5493
5494 /**
5495  * cik_pcie_gart_disable - gart disable
5496  *
5497  * @rdev: radeon_device pointer
5498  *
5499  * This disables all VM page table (CIK).
5500  */
5501 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5502 {
5503         /* Disable all tables */
5504         WREG32(VM_CONTEXT0_CNTL, 0);
5505         WREG32(VM_CONTEXT1_CNTL, 0);
5506         /* Setup TLB control */
5507         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5508                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5509         /* Setup L2 cache */
5510         WREG32(VM_L2_CNTL,
5511                ENABLE_L2_FRAGMENT_PROCESSING |
5512                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5513                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5514                EFFECTIVE_L2_QUEUE_SIZE(7) |
5515                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5516         WREG32(VM_L2_CNTL2, 0);
5517         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5518                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5519         radeon_gart_table_vram_unpin(rdev);
5520 }
5521
5522 /**
5523  * cik_pcie_gart_fini - vm fini callback
5524  *
5525  * @rdev: radeon_device pointer
5526  *
5527  * Tears down the driver GART/VM setup (CIK).
5528  */
5529 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5530 {
5531         cik_pcie_gart_disable(rdev);
5532         radeon_gart_table_vram_free(rdev);
5533         radeon_gart_fini(rdev);
5534 }
5535
5536 /* vm parser */
5537 /**
5538  * cik_ib_parse - vm ib_parse callback
5539  *
5540  * @rdev: radeon_device pointer
5541  * @ib: indirect buffer pointer
5542  *
5543  * CIK uses hw IB checking so this is a nop (CIK).
5544  */
5545 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5546 {
5547         return 0;
5548 }
5549
5550 /*
5551  * vm
5552  * VMID 0 is the physical GPU addresses as used by the kernel.
5553  * VMIDs 1-15 are used for userspace clients and are handled
5554  * by the radeon vm/hsa code.
5555  */
5556 /**
5557  * cik_vm_init - cik vm init callback
5558  *
5559  * @rdev: radeon_device pointer
5560  *
5561  * Inits cik specific vm parameters (number of VMs, base of vram for
5562  * VMIDs 1-15) (CIK).
5563  * Returns 0 for success.
5564  */
5565 int cik_vm_init(struct radeon_device *rdev)
5566 {
5567         /* number of VMs */
5568         rdev->vm_manager.nvm = 16;
5569         /* base offset of vram pages */
5570         if (rdev->flags & RADEON_IS_IGP) {
5571                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5572                 tmp <<= 22;
5573                 rdev->vm_manager.vram_base_offset = tmp;
5574         } else
5575                 rdev->vm_manager.vram_base_offset = 0;
5576
5577         return 0;
5578 }
5579
5580 /**
5581  * cik_vm_fini - cik vm fini callback
5582  *
5583  * @rdev: radeon_device pointer
5584  *
5585  * Tear down any asic specific VM setup (CIK).
5586  */
5587 void cik_vm_fini(struct radeon_device *rdev)
5588 {
5589 }
5590
5591 /**
5592  * cik_vm_decode_fault - print human readable fault info
5593  *
5594  * @rdev: radeon_device pointer
5595  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5596  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5597  *
5598  * Print human readable fault information (CIK).
5599  */
5600 static void cik_vm_decode_fault(struct radeon_device *rdev,
5601                                 u32 status, u32 addr, u32 mc_client)
5602 {
5603         u32 mc_id;
5604         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5605         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5606         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5607                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5608
5609         if (rdev->family == CHIP_HAWAII)
5610                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5611         else
5612                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5613
5614         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5615                protections, vmid, addr,
5616                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5617                block, mc_client, mc_id);
5618 }
5619
5620 /**
5621  * cik_vm_flush - cik vm flush using the CP
5622  *
5623  * @rdev: radeon_device pointer
5624  *
5625  * Update the page table base and flush the VM TLB
5626  * using the CP (CIK).
5627  */
5628 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5629 {
5630         struct radeon_ring *ring = &rdev->ring[ridx];
5631
5632         if (vm == NULL)
5633                 return;
5634
5635         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5636         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5637                                  WRITE_DATA_DST_SEL(0)));
5638         if (vm->id < 8) {
5639                 radeon_ring_write(ring,
5640                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5641         } else {
5642                 radeon_ring_write(ring,
5643                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5644         }
5645         radeon_ring_write(ring, 0);
5646         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5647
5648         /* update SH_MEM_* regs */
5649         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5650         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5651                                  WRITE_DATA_DST_SEL(0)));
5652         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5653         radeon_ring_write(ring, 0);
5654         radeon_ring_write(ring, VMID(vm->id));
5655
5656         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5657         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5658                                  WRITE_DATA_DST_SEL(0)));
5659         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5660         radeon_ring_write(ring, 0);
5661
5662         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5663         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5664         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5665         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5666
5667         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5668         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5669                                  WRITE_DATA_DST_SEL(0)));
5670         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5671         radeon_ring_write(ring, 0);
5672         radeon_ring_write(ring, VMID(0));
5673
5674         /* HDP flush */
5675         cik_hdp_flush_cp_ring_emit(rdev, ridx);
5676
5677         /* bits 0-15 are the VM contexts0-15 */
5678         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5679         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5680                                  WRITE_DATA_DST_SEL(0)));
5681         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5682         radeon_ring_write(ring, 0);
5683         radeon_ring_write(ring, 1 << vm->id);
5684
5685         /* compute doesn't have PFP */
5686         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5687                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5688                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5689                 radeon_ring_write(ring, 0x0);
5690         }
5691 }
5692
5693 /*
5694  * RLC
5695  * The RLC is a multi-purpose microengine that handles a
5696  * variety of functions, the most important of which is
5697  * the interrupt controller.
5698  */
5699 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5700                                           bool enable)
5701 {
5702         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5703
5704         if (enable)
5705                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5706         else
5707                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5708         WREG32(CP_INT_CNTL_RING0, tmp);
5709 }
5710
5711 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5712 {
5713         u32 tmp;
5714
5715         tmp = RREG32(RLC_LB_CNTL);
5716         if (enable)
5717                 tmp |= LOAD_BALANCE_ENABLE;
5718         else
5719                 tmp &= ~LOAD_BALANCE_ENABLE;
5720         WREG32(RLC_LB_CNTL, tmp);
5721 }
5722
5723 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5724 {
5725         u32 i, j, k;
5726         u32 mask;
5727
5728         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5729                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5730                         cik_select_se_sh(rdev, i, j);
5731                         for (k = 0; k < rdev->usec_timeout; k++) {
5732                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5733                                         break;
5734                                 udelay(1);
5735                         }
5736                 }
5737         }
5738         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5739
5740         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5741         for (k = 0; k < rdev->usec_timeout; k++) {
5742                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5743                         break;
5744                 udelay(1);
5745         }
5746 }
5747
5748 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5749 {
5750         u32 tmp;
5751
5752         tmp = RREG32(RLC_CNTL);
5753         if (tmp != rlc)
5754                 WREG32(RLC_CNTL, rlc);
5755 }
5756
5757 static u32 cik_halt_rlc(struct radeon_device *rdev)
5758 {
5759         u32 data, orig;
5760
5761         orig = data = RREG32(RLC_CNTL);
5762
5763         if (data & RLC_ENABLE) {
5764                 u32 i;
5765
5766                 data &= ~RLC_ENABLE;
5767                 WREG32(RLC_CNTL, data);
5768
5769                 for (i = 0; i < rdev->usec_timeout; i++) {
5770                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5771                                 break;
5772                         udelay(1);
5773                 }
5774
5775                 cik_wait_for_rlc_serdes(rdev);
5776         }
5777
5778         return orig;
5779 }
5780
5781 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5782 {
5783         u32 tmp, i, mask;
5784
5785         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5786         WREG32(RLC_GPR_REG2, tmp);
5787
5788         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5789         for (i = 0; i < rdev->usec_timeout; i++) {
5790                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5791                         break;
5792                 udelay(1);
5793         }
5794
5795         for (i = 0; i < rdev->usec_timeout; i++) {
5796                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5797                         break;
5798                 udelay(1);
5799         }
5800 }
5801
5802 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5803 {
5804         u32 tmp;
5805
5806         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5807         WREG32(RLC_GPR_REG2, tmp);
5808 }
5809
5810 /**
5811  * cik_rlc_stop - stop the RLC ME
5812  *
5813  * @rdev: radeon_device pointer
5814  *
5815  * Halt the RLC ME (MicroEngine) (CIK).
5816  */
5817 static void cik_rlc_stop(struct radeon_device *rdev)
5818 {
5819         WREG32(RLC_CNTL, 0);
5820
5821         cik_enable_gui_idle_interrupt(rdev, false);
5822
5823         cik_wait_for_rlc_serdes(rdev);
5824 }
5825
5826 /**
5827  * cik_rlc_start - start the RLC ME
5828  *
5829  * @rdev: radeon_device pointer
5830  *
5831  * Unhalt the RLC ME (MicroEngine) (CIK).
5832  */
5833 static void cik_rlc_start(struct radeon_device *rdev)
5834 {
5835         WREG32(RLC_CNTL, RLC_ENABLE);
5836
5837         cik_enable_gui_idle_interrupt(rdev, true);
5838
5839         udelay(50);
5840 }
5841
5842 /**
5843  * cik_rlc_resume - setup the RLC hw
5844  *
5845  * @rdev: radeon_device pointer
5846  *
5847  * Initialize the RLC registers, load the ucode,
5848  * and start the RLC (CIK).
5849  * Returns 0 for success, -EINVAL if the ucode is not available.
5850  */
5851 static int cik_rlc_resume(struct radeon_device *rdev)
5852 {
5853         u32 i, size, tmp;
5854         const __be32 *fw_data;
5855
5856         if (!rdev->rlc_fw)
5857                 return -EINVAL;
5858
5859         switch (rdev->family) {
5860         case CHIP_BONAIRE:
5861         case CHIP_HAWAII:
5862         default:
5863                 size = BONAIRE_RLC_UCODE_SIZE;
5864                 break;
5865         case CHIP_KAVERI:
5866                 size = KV_RLC_UCODE_SIZE;
5867                 break;
5868         case CHIP_KABINI:
5869                 size = KB_RLC_UCODE_SIZE;
5870                 break;
5871         case CHIP_MULLINS:
5872                 size = ML_RLC_UCODE_SIZE;
5873                 break;
5874         }
5875
5876         cik_rlc_stop(rdev);
5877
5878         /* disable CG */
5879         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5880         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5881
5882         si_rlc_reset(rdev);
5883
5884         cik_init_pg(rdev);
5885
5886         cik_init_cg(rdev);
5887
5888         WREG32(RLC_LB_CNTR_INIT, 0);
5889         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5890
5891         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5892         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5893         WREG32(RLC_LB_PARAMS, 0x00600408);
5894         WREG32(RLC_LB_CNTL, 0x80000004);
5895
5896         WREG32(RLC_MC_CNTL, 0);
5897         WREG32(RLC_UCODE_CNTL, 0);
5898
5899         fw_data = (const __be32 *)rdev->rlc_fw->data;
5900                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5901         for (i = 0; i < size; i++)
5902                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5903         WREG32(RLC_GPM_UCODE_ADDR, 0);
5904
5905         /* XXX - find out what chips support lbpw */
5906         cik_enable_lbpw(rdev, false);
5907
5908         if (rdev->family == CHIP_BONAIRE)
5909                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5910
5911         cik_rlc_start(rdev);
5912
5913         return 0;
5914 }
5915
5916 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5917 {
5918         u32 data, orig, tmp, tmp2;
5919
5920         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5921
5922         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5923                 cik_enable_gui_idle_interrupt(rdev, true);
5924
5925                 tmp = cik_halt_rlc(rdev);
5926
5927                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5928                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5929                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5930                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5931                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5932
5933                 cik_update_rlc(rdev, tmp);
5934
5935                 data |= CGCG_EN | CGLS_EN;
5936         } else {
5937                 cik_enable_gui_idle_interrupt(rdev, false);
5938
5939                 RREG32(CB_CGTT_SCLK_CTRL);
5940                 RREG32(CB_CGTT_SCLK_CTRL);
5941                 RREG32(CB_CGTT_SCLK_CTRL);
5942                 RREG32(CB_CGTT_SCLK_CTRL);
5943
5944                 data &= ~(CGCG_EN | CGLS_EN);
5945         }
5946
5947         if (orig != data)
5948                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5949
5950 }
5951
5952 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5953 {
5954         u32 data, orig, tmp = 0;
5955
5956         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5957                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5958                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5959                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5960                                 data |= CP_MEM_LS_EN;
5961                                 if (orig != data)
5962                                         WREG32(CP_MEM_SLP_CNTL, data);
5963                         }
5964                 }
5965
5966                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5967                 data &= 0xfffffffd;
5968                 if (orig != data)
5969                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5970
5971                 tmp = cik_halt_rlc(rdev);
5972
5973                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5974                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5975                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5976                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5977                 WREG32(RLC_SERDES_WR_CTRL, data);
5978
5979                 cik_update_rlc(rdev, tmp);
5980
5981                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5982                         orig = data = RREG32(CGTS_SM_CTRL_REG);
5983                         data &= ~SM_MODE_MASK;
5984                         data |= SM_MODE(0x2);
5985                         data |= SM_MODE_ENABLE;
5986                         data &= ~CGTS_OVERRIDE;
5987                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5988                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5989                                 data &= ~CGTS_LS_OVERRIDE;
5990                         data &= ~ON_MONITOR_ADD_MASK;
5991                         data |= ON_MONITOR_ADD_EN;
5992                         data |= ON_MONITOR_ADD(0x96);
5993                         if (orig != data)
5994                                 WREG32(CGTS_SM_CTRL_REG, data);
5995                 }
5996         } else {
5997                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5998                 data |= 0x00000002;
5999                 if (orig != data)
6000                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6001
6002                 data = RREG32(RLC_MEM_SLP_CNTL);
6003                 if (data & RLC_MEM_LS_EN) {
6004                         data &= ~RLC_MEM_LS_EN;
6005                         WREG32(RLC_MEM_SLP_CNTL, data);
6006                 }
6007
6008                 data = RREG32(CP_MEM_SLP_CNTL);
6009                 if (data & CP_MEM_LS_EN) {
6010                         data &= ~CP_MEM_LS_EN;
6011                         WREG32(CP_MEM_SLP_CNTL, data);
6012                 }
6013
6014                 orig = data = RREG32(CGTS_SM_CTRL_REG);
6015                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6016                 if (orig != data)
6017                         WREG32(CGTS_SM_CTRL_REG, data);
6018
6019                 tmp = cik_halt_rlc(rdev);
6020
6021                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6022                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6023                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6024                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6025                 WREG32(RLC_SERDES_WR_CTRL, data);
6026
6027                 cik_update_rlc(rdev, tmp);
6028         }
6029 }
6030
6031 static const u32 mc_cg_registers[] =
6032 {
6033         MC_HUB_MISC_HUB_CG,
6034         MC_HUB_MISC_SIP_CG,
6035         MC_HUB_MISC_VM_CG,
6036         MC_XPB_CLK_GAT,
6037         ATC_MISC_CG,
6038         MC_CITF_MISC_WR_CG,
6039         MC_CITF_MISC_RD_CG,
6040         MC_CITF_MISC_VM_CG,
6041         VM_L2_CG,
6042 };
6043
6044 static void cik_enable_mc_ls(struct radeon_device *rdev,
6045                              bool enable)
6046 {
6047         int i;
6048         u32 orig, data;
6049
6050         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6051                 orig = data = RREG32(mc_cg_registers[i]);
6052                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6053                         data |= MC_LS_ENABLE;
6054                 else
6055                         data &= ~MC_LS_ENABLE;
6056                 if (data != orig)
6057                         WREG32(mc_cg_registers[i], data);
6058         }
6059 }
6060
6061 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6062                                bool enable)
6063 {
6064         int i;
6065         u32 orig, data;
6066
6067         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6068                 orig = data = RREG32(mc_cg_registers[i]);
6069                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6070                         data |= MC_CG_ENABLE;
6071                 else
6072                         data &= ~MC_CG_ENABLE;
6073                 if (data != orig)
6074                         WREG32(mc_cg_registers[i], data);
6075         }
6076 }
6077
6078 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6079                                  bool enable)
6080 {
6081         u32 orig, data;
6082
6083         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6084                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6085                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6086         } else {
6087                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6088                 data |= 0xff000000;
6089                 if (data != orig)
6090                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6091
6092                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6093                 data |= 0xff000000;
6094                 if (data != orig)
6095                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6096         }
6097 }
6098
6099 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6100                                  bool enable)
6101 {
6102         u32 orig, data;
6103
6104         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6105                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6106                 data |= 0x100;
6107                 if (orig != data)
6108                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6109
6110                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6111                 data |= 0x100;
6112                 if (orig != data)
6113                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6114         } else {
6115                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6116                 data &= ~0x100;
6117                 if (orig != data)
6118                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6119
6120                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6121                 data &= ~0x100;
6122                 if (orig != data)
6123                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6124         }
6125 }
6126
6127 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6128                                 bool enable)
6129 {
6130         u32 orig, data;
6131
6132         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6133                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6134                 data = 0xfff;
6135                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6136
6137                 orig = data = RREG32(UVD_CGC_CTRL);
6138                 data |= DCM;
6139                 if (orig != data)
6140                         WREG32(UVD_CGC_CTRL, data);
6141         } else {
6142                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6143                 data &= ~0xfff;
6144                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6145
6146                 orig = data = RREG32(UVD_CGC_CTRL);
6147                 data &= ~DCM;
6148                 if (orig != data)
6149                         WREG32(UVD_CGC_CTRL, data);
6150         }
6151 }
6152
6153 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6154                                bool enable)
6155 {
6156         u32 orig, data;
6157
6158         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6159
6160         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6161                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6162                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6163         else
6164                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6165                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6166
6167         if (orig != data)
6168                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6169 }
6170
6171 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6172                                 bool enable)
6173 {
6174         u32 orig, data;
6175
6176         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6177
6178         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6179                 data &= ~CLOCK_GATING_DIS;
6180         else
6181                 data |= CLOCK_GATING_DIS;
6182
6183         if (orig != data)
6184                 WREG32(HDP_HOST_PATH_CNTL, data);
6185 }
6186
6187 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6188                               bool enable)
6189 {
6190         u32 orig, data;
6191
6192         orig = data = RREG32(HDP_MEM_POWER_LS);
6193
6194         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6195                 data |= HDP_LS_ENABLE;
6196         else
6197                 data &= ~HDP_LS_ENABLE;
6198
6199         if (orig != data)
6200                 WREG32(HDP_MEM_POWER_LS, data);
6201 }
6202
6203 void cik_update_cg(struct radeon_device *rdev,
6204                    u32 block, bool enable)
6205 {
6206
6207         if (block & RADEON_CG_BLOCK_GFX) {
6208                 cik_enable_gui_idle_interrupt(rdev, false);
6209                 /* order matters! */
6210                 if (enable) {
6211                         cik_enable_mgcg(rdev, true);
6212                         cik_enable_cgcg(rdev, true);
6213                 } else {
6214                         cik_enable_cgcg(rdev, false);
6215                         cik_enable_mgcg(rdev, false);
6216                 }
6217                 cik_enable_gui_idle_interrupt(rdev, true);
6218         }
6219
6220         if (block & RADEON_CG_BLOCK_MC) {
6221                 if (!(rdev->flags & RADEON_IS_IGP)) {
6222                         cik_enable_mc_mgcg(rdev, enable);
6223                         cik_enable_mc_ls(rdev, enable);
6224                 }
6225         }
6226
6227         if (block & RADEON_CG_BLOCK_SDMA) {
6228                 cik_enable_sdma_mgcg(rdev, enable);
6229                 cik_enable_sdma_mgls(rdev, enable);
6230         }
6231
6232         if (block & RADEON_CG_BLOCK_BIF) {
6233                 cik_enable_bif_mgls(rdev, enable);
6234         }
6235
6236         if (block & RADEON_CG_BLOCK_UVD) {
6237                 if (rdev->has_uvd)
6238                         cik_enable_uvd_mgcg(rdev, enable);
6239         }
6240
6241         if (block & RADEON_CG_BLOCK_HDP) {
6242                 cik_enable_hdp_mgcg(rdev, enable);
6243                 cik_enable_hdp_ls(rdev, enable);
6244         }
6245
6246         if (block & RADEON_CG_BLOCK_VCE) {
6247                 vce_v2_0_enable_mgcg(rdev, enable);
6248         }
6249 }
6250
6251 static void cik_init_cg(struct radeon_device *rdev)
6252 {
6253
6254         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6255
6256         if (rdev->has_uvd)
6257                 si_init_uvd_internal_cg(rdev);
6258
6259         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6260                              RADEON_CG_BLOCK_SDMA |
6261                              RADEON_CG_BLOCK_BIF |
6262                              RADEON_CG_BLOCK_UVD |
6263                              RADEON_CG_BLOCK_HDP), true);
6264 }
6265
6266 static void cik_fini_cg(struct radeon_device *rdev)
6267 {
6268         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6269                              RADEON_CG_BLOCK_SDMA |
6270                              RADEON_CG_BLOCK_BIF |
6271                              RADEON_CG_BLOCK_UVD |
6272                              RADEON_CG_BLOCK_HDP), false);
6273
6274         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6275 }
6276
6277 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6278                                           bool enable)
6279 {
6280         u32 data, orig;
6281
6282         orig = data = RREG32(RLC_PG_CNTL);
6283         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6284                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6285         else
6286                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6287         if (orig != data)
6288                 WREG32(RLC_PG_CNTL, data);
6289 }
6290
6291 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6292                                           bool enable)
6293 {
6294         u32 data, orig;
6295
6296         orig = data = RREG32(RLC_PG_CNTL);
6297         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6298                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6299         else
6300                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6301         if (orig != data)
6302                 WREG32(RLC_PG_CNTL, data);
6303 }
6304
6305 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6306 {
6307         u32 data, orig;
6308
6309         orig = data = RREG32(RLC_PG_CNTL);
6310         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6311                 data &= ~DISABLE_CP_PG;
6312         else
6313                 data |= DISABLE_CP_PG;
6314         if (orig != data)
6315                 WREG32(RLC_PG_CNTL, data);
6316 }
6317
6318 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6319 {
6320         u32 data, orig;
6321
6322         orig = data = RREG32(RLC_PG_CNTL);
6323         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6324                 data &= ~DISABLE_GDS_PG;
6325         else
6326                 data |= DISABLE_GDS_PG;
6327         if (orig != data)
6328                 WREG32(RLC_PG_CNTL, data);
6329 }
6330
6331 #define CP_ME_TABLE_SIZE    96
6332 #define CP_ME_TABLE_OFFSET  2048
6333 #define CP_MEC_TABLE_OFFSET 4096
6334
6335 void cik_init_cp_pg_table(struct radeon_device *rdev)
6336 {
6337         const __be32 *fw_data;
6338         volatile u32 *dst_ptr;
6339         int me, i, max_me = 4;
6340         u32 bo_offset = 0;
6341         u32 table_offset;
6342
6343         if (rdev->family == CHIP_KAVERI)
6344                 max_me = 5;
6345
6346         if (rdev->rlc.cp_table_ptr == NULL)
6347                 return;
6348
6349         /* write the cp table buffer */
6350         dst_ptr = rdev->rlc.cp_table_ptr;
6351         for (me = 0; me < max_me; me++) {
6352                 if (me == 0) {
6353                         fw_data = (const __be32 *)rdev->ce_fw->data;
6354                         table_offset = CP_ME_TABLE_OFFSET;
6355                 } else if (me == 1) {
6356                         fw_data = (const __be32 *)rdev->pfp_fw->data;
6357                         table_offset = CP_ME_TABLE_OFFSET;
6358                 } else if (me == 2) {
6359                         fw_data = (const __be32 *)rdev->me_fw->data;
6360                         table_offset = CP_ME_TABLE_OFFSET;
6361                 } else {
6362                         fw_data = (const __be32 *)rdev->mec_fw->data;
6363                         table_offset = CP_MEC_TABLE_OFFSET;
6364                 }
6365
6366                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6367                         dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6368                 }
6369                 bo_offset += CP_ME_TABLE_SIZE;
6370         }
6371 }
6372
6373 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6374                                 bool enable)
6375 {
6376         u32 data, orig;
6377
6378         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6379                 orig = data = RREG32(RLC_PG_CNTL);
6380                 data |= GFX_PG_ENABLE;
6381                 if (orig != data)
6382                         WREG32(RLC_PG_CNTL, data);
6383
6384                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6385                 data |= AUTO_PG_EN;
6386                 if (orig != data)
6387                         WREG32(RLC_AUTO_PG_CTRL, data);
6388         } else {
6389                 orig = data = RREG32(RLC_PG_CNTL);
6390                 data &= ~GFX_PG_ENABLE;
6391                 if (orig != data)
6392                         WREG32(RLC_PG_CNTL, data);
6393
6394                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6395                 data &= ~AUTO_PG_EN;
6396                 if (orig != data)
6397                         WREG32(RLC_AUTO_PG_CTRL, data);
6398
6399                 data = RREG32(DB_RENDER_CONTROL);
6400         }
6401 }
6402
6403 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6404 {
6405         u32 mask = 0, tmp, tmp1;
6406         int i;
6407
6408         cik_select_se_sh(rdev, se, sh);
6409         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6410         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6411         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6412
6413         tmp &= 0xffff0000;
6414
6415         tmp |= tmp1;
6416         tmp >>= 16;
6417
6418         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6419                 mask <<= 1;
6420                 mask |= 1;
6421         }
6422
6423         return (~tmp) & mask;
6424 }
6425
6426 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6427 {
6428         u32 i, j, k, active_cu_number = 0;
6429         u32 mask, counter, cu_bitmap;
6430         u32 tmp = 0;
6431
6432         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6433                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6434                         mask = 1;
6435                         cu_bitmap = 0;
6436                         counter = 0;
6437                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6438                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6439                                         if (counter < 2)
6440                                                 cu_bitmap |= mask;
6441                                         counter ++;
6442                                 }
6443                                 mask <<= 1;
6444                         }
6445
6446                         active_cu_number += counter;
6447                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6448                 }
6449         }
6450
6451         WREG32(RLC_PG_AO_CU_MASK, tmp);
6452
6453         tmp = RREG32(RLC_MAX_PG_CU);
6454         tmp &= ~MAX_PU_CU_MASK;
6455         tmp |= MAX_PU_CU(active_cu_number);
6456         WREG32(RLC_MAX_PG_CU, tmp);
6457 }
6458
6459 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6460                                        bool enable)
6461 {
6462         u32 data, orig;
6463
6464         orig = data = RREG32(RLC_PG_CNTL);
6465         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6466                 data |= STATIC_PER_CU_PG_ENABLE;
6467         else
6468                 data &= ~STATIC_PER_CU_PG_ENABLE;
6469         if (orig != data)
6470                 WREG32(RLC_PG_CNTL, data);
6471 }
6472
6473 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6474                                         bool enable)
6475 {
6476         u32 data, orig;
6477
6478         orig = data = RREG32(RLC_PG_CNTL);
6479         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6480                 data |= DYN_PER_CU_PG_ENABLE;
6481         else
6482                 data &= ~DYN_PER_CU_PG_ENABLE;
6483         if (orig != data)
6484                 WREG32(RLC_PG_CNTL, data);
6485 }
6486
6487 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6488 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6489
6490 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6491 {
6492         u32 data, orig;
6493         u32 i;
6494
6495         if (rdev->rlc.cs_data) {
6496                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6497                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6498                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6499                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6500         } else {
6501                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6502                 for (i = 0; i < 3; i++)
6503                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6504         }
6505         if (rdev->rlc.reg_list) {
6506                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6507                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6508                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6509         }
6510
6511         orig = data = RREG32(RLC_PG_CNTL);
6512         data |= GFX_PG_SRC;
6513         if (orig != data)
6514                 WREG32(RLC_PG_CNTL, data);
6515
6516         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6517         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6518
6519         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6520         data &= ~IDLE_POLL_COUNT_MASK;
6521         data |= IDLE_POLL_COUNT(0x60);
6522         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6523
6524         data = 0x10101010;
6525         WREG32(RLC_PG_DELAY, data);
6526
6527         data = RREG32(RLC_PG_DELAY_2);
6528         data &= ~0xff;
6529         data |= 0x3;
6530         WREG32(RLC_PG_DELAY_2, data);
6531
6532         data = RREG32(RLC_AUTO_PG_CTRL);
6533         data &= ~GRBM_REG_SGIT_MASK;
6534         data |= GRBM_REG_SGIT(0x700);
6535         WREG32(RLC_AUTO_PG_CTRL, data);
6536
6537 }
6538
6539 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6540 {
6541         cik_enable_gfx_cgpg(rdev, enable);
6542         cik_enable_gfx_static_mgpg(rdev, enable);
6543         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6544 }
6545
6546 u32 cik_get_csb_size(struct radeon_device *rdev)
6547 {
6548         u32 count = 0;
6549         const struct cs_section_def *sect = NULL;
6550         const struct cs_extent_def *ext = NULL;
6551
6552         if (rdev->rlc.cs_data == NULL)
6553                 return 0;
6554
6555         /* begin clear state */
6556         count += 2;
6557         /* context control state */
6558         count += 3;
6559
6560         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6561                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6562                         if (sect->id == SECT_CONTEXT)
6563                                 count += 2 + ext->reg_count;
6564                         else
6565                                 return 0;
6566                 }
6567         }
6568         /* pa_sc_raster_config/pa_sc_raster_config1 */
6569         count += 4;
6570         /* end clear state */
6571         count += 2;
6572         /* clear state */
6573         count += 2;
6574
6575         return count;
6576 }
6577
6578 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6579 {
6580         u32 count = 0, i;
6581         const struct cs_section_def *sect = NULL;
6582         const struct cs_extent_def *ext = NULL;
6583
6584         if (rdev->rlc.cs_data == NULL)
6585                 return;
6586         if (buffer == NULL)
6587                 return;
6588
6589         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6590         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6591
6592         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6593         buffer[count++] = cpu_to_le32(0x80000000);
6594         buffer[count++] = cpu_to_le32(0x80000000);
6595
6596         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6597                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6598                         if (sect->id == SECT_CONTEXT) {
6599                                 buffer[count++] =
6600                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6601                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6602                                 for (i = 0; i < ext->reg_count; i++)
6603                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6604                         } else {
6605                                 return;
6606                         }
6607                 }
6608         }
6609
6610         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6611         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6612         switch (rdev->family) {
6613         case CHIP_BONAIRE:
6614                 buffer[count++] = cpu_to_le32(0x16000012);
6615                 buffer[count++] = cpu_to_le32(0x00000000);
6616                 break;
6617         case CHIP_KAVERI:
6618                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6619                 buffer[count++] = cpu_to_le32(0x00000000);
6620                 break;
6621         case CHIP_KABINI:
6622         case CHIP_MULLINS:
6623                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6624                 buffer[count++] = cpu_to_le32(0x00000000);
6625                 break;
6626         case CHIP_HAWAII:
6627                 buffer[count++] = cpu_to_le32(0x3a00161a);
6628                 buffer[count++] = cpu_to_le32(0x0000002e);
6629                 break;
6630         default:
6631                 buffer[count++] = cpu_to_le32(0x00000000);
6632                 buffer[count++] = cpu_to_le32(0x00000000);
6633                 break;
6634         }
6635
6636         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6637         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6638
6639         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6640         buffer[count++] = cpu_to_le32(0);
6641 }
6642
6643 static void cik_init_pg(struct radeon_device *rdev)
6644 {
6645         if (rdev->pg_flags) {
6646                 cik_enable_sck_slowdown_on_pu(rdev, true);
6647                 cik_enable_sck_slowdown_on_pd(rdev, true);
6648                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6649                         cik_init_gfx_cgpg(rdev);
6650                         cik_enable_cp_pg(rdev, true);
6651                         cik_enable_gds_pg(rdev, true);
6652                 }
6653                 cik_init_ao_cu_mask(rdev);
6654                 cik_update_gfx_pg(rdev, true);
6655         }
6656 }
6657
6658 static void cik_fini_pg(struct radeon_device *rdev)
6659 {
6660         if (rdev->pg_flags) {
6661                 cik_update_gfx_pg(rdev, false);
6662                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6663                         cik_enable_cp_pg(rdev, false);
6664                         cik_enable_gds_pg(rdev, false);
6665                 }
6666         }
6667 }
6668
6669 /*
6670  * Interrupts
6671  * Starting with r6xx, interrupts are handled via a ring buffer.
6672  * Ring buffers are areas of GPU accessible memory that the GPU
6673  * writes interrupt vectors into and the host reads vectors out of.
6674  * There is a rptr (read pointer) that determines where the
6675  * host is currently reading, and a wptr (write pointer)
6676  * which determines where the GPU has written.  When the
6677  * pointers are equal, the ring is idle.  When the GPU
6678  * writes vectors to the ring buffer, it increments the
6679  * wptr.  When there is an interrupt, the host then starts
6680  * fetching commands and processing them until the pointers are
6681  * equal again at which point it updates the rptr.
6682  */
6683
6684 /**
6685  * cik_enable_interrupts - Enable the interrupt ring buffer
6686  *
6687  * @rdev: radeon_device pointer
6688  *
6689  * Enable the interrupt ring buffer (CIK).
6690  */
6691 static void cik_enable_interrupts(struct radeon_device *rdev)
6692 {
6693         u32 ih_cntl = RREG32(IH_CNTL);
6694         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6695
6696         ih_cntl |= ENABLE_INTR;
6697         ih_rb_cntl |= IH_RB_ENABLE;
6698         WREG32(IH_CNTL, ih_cntl);
6699         WREG32(IH_RB_CNTL, ih_rb_cntl);
6700         rdev->ih.enabled = true;
6701 }
6702
6703 /**
6704  * cik_disable_interrupts - Disable the interrupt ring buffer
6705  *
6706  * @rdev: radeon_device pointer
6707  *
6708  * Disable the interrupt ring buffer (CIK).
6709  */
6710 static void cik_disable_interrupts(struct radeon_device *rdev)
6711 {
6712         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6713         u32 ih_cntl = RREG32(IH_CNTL);
6714
6715         ih_rb_cntl &= ~IH_RB_ENABLE;
6716         ih_cntl &= ~ENABLE_INTR;
6717         WREG32(IH_RB_CNTL, ih_rb_cntl);
6718         WREG32(IH_CNTL, ih_cntl);
6719         /* set rptr, wptr to 0 */
6720         WREG32(IH_RB_RPTR, 0);
6721         WREG32(IH_RB_WPTR, 0);
6722         rdev->ih.enabled = false;
6723         rdev->ih.rptr = 0;
6724 }
6725
6726 /**
6727  * cik_disable_interrupt_state - Disable all interrupt sources
6728  *
6729  * @rdev: radeon_device pointer
6730  *
6731  * Clear all interrupt enable bits used by the driver (CIK).
6732  */
6733 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6734 {
6735         u32 tmp;
6736
6737         /* gfx ring */
6738         tmp = RREG32(CP_INT_CNTL_RING0) &
6739                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6740         WREG32(CP_INT_CNTL_RING0, tmp);
6741         /* sdma */
6742         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6743         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6744         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6745         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6746         /* compute queues */
6747         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6748         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6749         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6750         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6751         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6752         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6753         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6754         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6755         /* grbm */
6756         WREG32(GRBM_INT_CNTL, 0);
6757         /* vline/vblank, etc. */
6758         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6759         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6760         if (rdev->num_crtc >= 4) {
6761                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6762                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6763         }
6764         if (rdev->num_crtc >= 6) {
6765                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6766                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6767         }
6768         /* pflip */
6769         if (rdev->num_crtc >= 2) {
6770                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6771                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6772         }
6773         if (rdev->num_crtc >= 4) {
6774                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6775                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6776         }
6777         if (rdev->num_crtc >= 6) {
6778                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6779                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6780         }
6781
6782         /* dac hotplug */
6783         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6784
6785         /* digital hotplug */
6786         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6787         WREG32(DC_HPD1_INT_CONTROL, tmp);
6788         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6789         WREG32(DC_HPD2_INT_CONTROL, tmp);
6790         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6791         WREG32(DC_HPD3_INT_CONTROL, tmp);
6792         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6793         WREG32(DC_HPD4_INT_CONTROL, tmp);
6794         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6795         WREG32(DC_HPD5_INT_CONTROL, tmp);
6796         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6797         WREG32(DC_HPD6_INT_CONTROL, tmp);
6798
6799 }
6800
6801 /**
6802  * cik_irq_init - init and enable the interrupt ring
6803  *
6804  * @rdev: radeon_device pointer
6805  *
6806  * Allocate a ring buffer for the interrupt controller,
6807  * enable the RLC, disable interrupts, enable the IH
6808  * ring buffer and enable it (CIK).
6809  * Called at device load and reume.
6810  * Returns 0 for success, errors for failure.
6811  */
6812 static int cik_irq_init(struct radeon_device *rdev)
6813 {
6814         int ret = 0;
6815         int rb_bufsz;
6816         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6817
6818         /* allocate ring */
6819         ret = r600_ih_ring_alloc(rdev);
6820         if (ret)
6821                 return ret;
6822
6823         /* disable irqs */
6824         cik_disable_interrupts(rdev);
6825
6826         /* init rlc */
6827         ret = cik_rlc_resume(rdev);
6828         if (ret) {
6829                 r600_ih_ring_fini(rdev);
6830                 return ret;
6831         }
6832
6833         /* setup interrupt control */
6834         /* XXX this should actually be a bus address, not an MC address. same on older asics */
6835         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6836         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6837         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6838          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6839          */
6840         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6841         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6842         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6843         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6844
6845         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6846         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6847
6848         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6849                       IH_WPTR_OVERFLOW_CLEAR |
6850                       (rb_bufsz << 1));
6851
6852         if (rdev->wb.enabled)
6853                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6854
6855         /* set the writeback address whether it's enabled or not */
6856         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6857         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6858
6859         WREG32(IH_RB_CNTL, ih_rb_cntl);
6860
6861         /* set rptr, wptr to 0 */
6862         WREG32(IH_RB_RPTR, 0);
6863         WREG32(IH_RB_WPTR, 0);
6864
6865         /* Default settings for IH_CNTL (disabled at first) */
6866         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6867         /* RPTR_REARM only works if msi's are enabled */
6868         if (rdev->msi_enabled)
6869                 ih_cntl |= RPTR_REARM;
6870         WREG32(IH_CNTL, ih_cntl);
6871
6872         /* force the active interrupt state to all disabled */
6873         cik_disable_interrupt_state(rdev);
6874
6875         pci_set_master(rdev->pdev);
6876
6877         /* enable irqs */
6878         cik_enable_interrupts(rdev);
6879
6880         return ret;
6881 }
6882
6883 /**
6884  * cik_irq_set - enable/disable interrupt sources
6885  *
6886  * @rdev: radeon_device pointer
6887  *
6888  * Enable interrupt sources on the GPU (vblanks, hpd,
6889  * etc.) (CIK).
6890  * Returns 0 for success, errors for failure.
6891  */
6892 int cik_irq_set(struct radeon_device *rdev)
6893 {
6894         u32 cp_int_cntl;
6895         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6896         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6897         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6898         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6899         u32 grbm_int_cntl = 0;
6900         u32 dma_cntl, dma_cntl1;
6901         u32 thermal_int;
6902
6903         if (!rdev->irq.installed) {
6904                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6905                 return -EINVAL;
6906         }
6907         /* don't enable anything if the ih is disabled */
6908         if (!rdev->ih.enabled) {
6909                 cik_disable_interrupts(rdev);
6910                 /* force the active interrupt state to all disabled */
6911                 cik_disable_interrupt_state(rdev);
6912                 return 0;
6913         }
6914
6915         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6916                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6917         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6918
6919         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6920         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6921         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6922         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6923         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6924         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6925
6926         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6927         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6928
6929         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6930         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6931         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6932         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6933         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6934         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6935         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6936         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6937
6938         if (rdev->flags & RADEON_IS_IGP)
6939                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6940                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6941         else
6942                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6943                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6944
6945         /* enable CP interrupts on all rings */
6946         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6947                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6948                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6949         }
6950         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6951                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6952                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6953                 if (ring->me == 1) {
6954                         switch (ring->pipe) {
6955                         case 0:
6956                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6957                                 break;
6958                         case 1:
6959                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6960                                 break;
6961                         case 2:
6962                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6963                                 break;
6964                         case 3:
6965                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6966                                 break;
6967                         default:
6968                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6969                                 break;
6970                         }
6971                 } else if (ring->me == 2) {
6972                         switch (ring->pipe) {
6973                         case 0:
6974                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6975                                 break;
6976                         case 1:
6977                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6978                                 break;
6979                         case 2:
6980                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6981                                 break;
6982                         case 3:
6983                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6984                                 break;
6985                         default:
6986                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6987                                 break;
6988                         }
6989                 } else {
6990                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6991                 }
6992         }
6993         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6994                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6995                 DRM_DEBUG("si_irq_set: sw int cp2\n");
6996                 if (ring->me == 1) {
6997                         switch (ring->pipe) {
6998                         case 0:
6999                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7000                                 break;
7001                         case 1:
7002                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7003                                 break;
7004                         case 2:
7005                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7006                                 break;
7007                         case 3:
7008                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7009                                 break;
7010                         default:
7011                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7012                                 break;
7013                         }
7014                 } else if (ring->me == 2) {
7015                         switch (ring->pipe) {
7016                         case 0:
7017                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7018                                 break;
7019                         case 1:
7020                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7021                                 break;
7022                         case 2:
7023                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7024                                 break;
7025                         case 3:
7026                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7027                                 break;
7028                         default:
7029                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7030                                 break;
7031                         }
7032                 } else {
7033                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7034                 }
7035         }
7036
7037         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7038                 DRM_DEBUG("cik_irq_set: sw int dma\n");
7039                 dma_cntl |= TRAP_ENABLE;
7040         }
7041
7042         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7043                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7044                 dma_cntl1 |= TRAP_ENABLE;
7045         }
7046
7047         if (rdev->irq.crtc_vblank_int[0] ||
7048             atomic_read(&rdev->irq.pflip[0])) {
7049                 DRM_DEBUG("cik_irq_set: vblank 0\n");
7050                 crtc1 |= VBLANK_INTERRUPT_MASK;
7051         }
7052         if (rdev->irq.crtc_vblank_int[1] ||
7053             atomic_read(&rdev->irq.pflip[1])) {
7054                 DRM_DEBUG("cik_irq_set: vblank 1\n");
7055                 crtc2 |= VBLANK_INTERRUPT_MASK;
7056         }
7057         if (rdev->irq.crtc_vblank_int[2] ||
7058             atomic_read(&rdev->irq.pflip[2])) {
7059                 DRM_DEBUG("cik_irq_set: vblank 2\n");
7060                 crtc3 |= VBLANK_INTERRUPT_MASK;
7061         }
7062         if (rdev->irq.crtc_vblank_int[3] ||
7063             atomic_read(&rdev->irq.pflip[3])) {
7064                 DRM_DEBUG("cik_irq_set: vblank 3\n");
7065                 crtc4 |= VBLANK_INTERRUPT_MASK;
7066         }
7067         if (rdev->irq.crtc_vblank_int[4] ||
7068             atomic_read(&rdev->irq.pflip[4])) {
7069                 DRM_DEBUG("cik_irq_set: vblank 4\n");
7070                 crtc5 |= VBLANK_INTERRUPT_MASK;
7071         }
7072         if (rdev->irq.crtc_vblank_int[5] ||
7073             atomic_read(&rdev->irq.pflip[5])) {
7074                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7075                 crtc6 |= VBLANK_INTERRUPT_MASK;
7076         }
7077         if (rdev->irq.hpd[0]) {
7078                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7079                 hpd1 |= DC_HPDx_INT_EN;
7080         }
7081         if (rdev->irq.hpd[1]) {
7082                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7083                 hpd2 |= DC_HPDx_INT_EN;
7084         }
7085         if (rdev->irq.hpd[2]) {
7086                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7087                 hpd3 |= DC_HPDx_INT_EN;
7088         }
7089         if (rdev->irq.hpd[3]) {
7090                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7091                 hpd4 |= DC_HPDx_INT_EN;
7092         }
7093         if (rdev->irq.hpd[4]) {
7094                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7095                 hpd5 |= DC_HPDx_INT_EN;
7096         }
7097         if (rdev->irq.hpd[5]) {
7098                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7099                 hpd6 |= DC_HPDx_INT_EN;
7100         }
7101
7102         if (rdev->irq.dpm_thermal) {
7103                 DRM_DEBUG("dpm thermal\n");
7104                 if (rdev->flags & RADEON_IS_IGP)
7105                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7106                 else
7107                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7108         }
7109
7110         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7111
7112         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7113         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7114
7115         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7116         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7117         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7118         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7119         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7120         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7121         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7122         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7123
7124         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7125
7126         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7127         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7128         if (rdev->num_crtc >= 4) {
7129                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7130                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7131         }
7132         if (rdev->num_crtc >= 6) {
7133                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7134                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7135         }
7136
7137         if (rdev->num_crtc >= 2) {
7138                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7139                        GRPH_PFLIP_INT_MASK);
7140                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7141                        GRPH_PFLIP_INT_MASK);
7142         }
7143         if (rdev->num_crtc >= 4) {
7144                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7145                        GRPH_PFLIP_INT_MASK);
7146                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7147                        GRPH_PFLIP_INT_MASK);
7148         }
7149         if (rdev->num_crtc >= 6) {
7150                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7151                        GRPH_PFLIP_INT_MASK);
7152                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7153                        GRPH_PFLIP_INT_MASK);
7154         }
7155
7156         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7157         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7158         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7159         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7160         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7161         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7162
7163         if (rdev->flags & RADEON_IS_IGP)
7164                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7165         else
7166                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7167
7168         return 0;
7169 }
7170
7171 /**
7172  * cik_irq_ack - ack interrupt sources
7173  *
7174  * @rdev: radeon_device pointer
7175  *
7176  * Ack interrupt sources on the GPU (vblanks, hpd,
7177  * etc.) (CIK).  Certain interrupts sources are sw
7178  * generated and do not require an explicit ack.
7179  */
7180 static inline void cik_irq_ack(struct radeon_device *rdev)
7181 {
7182         u32 tmp;
7183
7184         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7185         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7186         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7187         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7188         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7189         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7190         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7191
7192         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7193                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7194         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7195                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7196         if (rdev->num_crtc >= 4) {
7197                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7198                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7199                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7200                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7201         }
7202         if (rdev->num_crtc >= 6) {
7203                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7204                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7205                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7206                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7207         }
7208
7209         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7210                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7211                        GRPH_PFLIP_INT_CLEAR);
7212         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7213                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7214                        GRPH_PFLIP_INT_CLEAR);
7215         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7216                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7217         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7218                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7219         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7220                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7221         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7222                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7223
7224         if (rdev->num_crtc >= 4) {
7225                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7226                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7227                                GRPH_PFLIP_INT_CLEAR);
7228                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7229                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7230                                GRPH_PFLIP_INT_CLEAR);
7231                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7232                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7233                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7234                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7235                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7236                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7237                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7238                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7239         }
7240
7241         if (rdev->num_crtc >= 6) {
7242                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7243                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7244                                GRPH_PFLIP_INT_CLEAR);
7245                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7246                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7247                                GRPH_PFLIP_INT_CLEAR);
7248                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7249                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7250                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7251                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7252                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7253                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7254                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7255                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7256         }
7257
7258         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7259                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7260                 tmp |= DC_HPDx_INT_ACK;
7261                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7262         }
7263         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7264                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7265                 tmp |= DC_HPDx_INT_ACK;
7266                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7267         }
7268         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7269                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7270                 tmp |= DC_HPDx_INT_ACK;
7271                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7272         }
7273         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7274                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7275                 tmp |= DC_HPDx_INT_ACK;
7276                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7277         }
7278         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7279                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7280                 tmp |= DC_HPDx_INT_ACK;
7281                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7282         }
7283         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7284                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7285                 tmp |= DC_HPDx_INT_ACK;
7286                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7287         }
7288 }
7289
7290 /**
7291  * cik_irq_disable - disable interrupts
7292  *
7293  * @rdev: radeon_device pointer
7294  *
7295  * Disable interrupts on the hw (CIK).
7296  */
7297 static void cik_irq_disable(struct radeon_device *rdev)
7298 {
7299         cik_disable_interrupts(rdev);
7300         /* Wait and acknowledge irq */
7301         mdelay(1);
7302         cik_irq_ack(rdev);
7303         cik_disable_interrupt_state(rdev);
7304 }
7305
7306 /**
7307  * cik_irq_disable - disable interrupts for suspend
7308  *
7309  * @rdev: radeon_device pointer
7310  *
7311  * Disable interrupts and stop the RLC (CIK).
7312  * Used for suspend.
7313  */
7314 static void cik_irq_suspend(struct radeon_device *rdev)
7315 {
7316         cik_irq_disable(rdev);
7317         cik_rlc_stop(rdev);
7318 }
7319
7320 /**
7321  * cik_irq_fini - tear down interrupt support
7322  *
7323  * @rdev: radeon_device pointer
7324  *
7325  * Disable interrupts on the hw and free the IH ring
7326  * buffer (CIK).
7327  * Used for driver unload.
7328  */
7329 static void cik_irq_fini(struct radeon_device *rdev)
7330 {
7331         cik_irq_suspend(rdev);
7332         r600_ih_ring_fini(rdev);
7333 }
7334
7335 /**
7336  * cik_get_ih_wptr - get the IH ring buffer wptr
7337  *
7338  * @rdev: radeon_device pointer
7339  *
7340  * Get the IH ring buffer wptr from either the register
7341  * or the writeback memory buffer (CIK).  Also check for
7342  * ring buffer overflow and deal with it.
7343  * Used by cik_irq_process().
7344  * Returns the value of the wptr.
7345  */
7346 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7347 {
7348         u32 wptr, tmp;
7349
7350         if (rdev->wb.enabled)
7351                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7352         else
7353                 wptr = RREG32(IH_RB_WPTR);
7354
7355         if (wptr & RB_OVERFLOW) {
7356                 /* When a ring buffer overflow happen start parsing interrupt
7357                  * from the last not overwritten vector (wptr + 16). Hopefully
7358                  * this should allow us to catchup.
7359                  */
7360                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7361                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7362                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7363                 tmp = RREG32(IH_RB_CNTL);
7364                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7365                 WREG32(IH_RB_CNTL, tmp);
7366         }
7367         return (wptr & rdev->ih.ptr_mask);
7368 }
7369
7370 /*        CIK IV Ring
7371  * Each IV ring entry is 128 bits:
7372  * [7:0]    - interrupt source id
7373  * [31:8]   - reserved
7374  * [59:32]  - interrupt source data
7375  * [63:60]  - reserved
7376  * [71:64]  - RINGID
7377  *            CP:
7378  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7379  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7380  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7381  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7382  *            PIPE_ID - ME0 0=3D
7383  *                    - ME1&2 compute dispatcher (4 pipes each)
7384  *            SDMA:
7385  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7386  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7387  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7388  * [79:72]  - VMID
7389  * [95:80]  - PASID
7390  * [127:96] - reserved
7391  */
7392 /**
7393  * cik_irq_process - interrupt handler
7394  *
7395  * @rdev: radeon_device pointer
7396  *
7397  * Interrupt hander (CIK).  Walk the IH ring,
7398  * ack interrupts and schedule work to handle
7399  * interrupt events.
7400  * Returns irq process return code.
7401  */
7402 int cik_irq_process(struct radeon_device *rdev)
7403 {
7404         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7405         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7406         u32 wptr;
7407         u32 rptr;
7408         u32 src_id, src_data, ring_id;
7409         u8 me_id, pipe_id, queue_id;
7410         u32 ring_index;
7411         bool queue_hotplug = false;
7412         bool queue_reset = false;
7413         u32 addr, status, mc_client;
7414         bool queue_thermal = false;
7415
7416         if (!rdev->ih.enabled || rdev->shutdown)
7417                 return IRQ_NONE;
7418
7419         wptr = cik_get_ih_wptr(rdev);
7420
7421 restart_ih:
7422         /* is somebody else already processing irqs? */
7423         if (atomic_xchg(&rdev->ih.lock, 1))
7424                 return IRQ_NONE;
7425
7426         rptr = rdev->ih.rptr;
7427         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7428
7429         /* Order reading of wptr vs. reading of IH ring data */
7430         rmb();
7431
7432         /* display interrupts */
7433         cik_irq_ack(rdev);
7434
7435         while (rptr != wptr) {
7436                 /* wptr/rptr are in bytes! */
7437                 ring_index = rptr / 4;
7438                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7439                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7440                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7441
7442                 switch (src_id) {
7443                 case 1: /* D1 vblank/vline */
7444                         switch (src_data) {
7445                         case 0: /* D1 vblank */
7446                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7447                                         if (rdev->irq.crtc_vblank_int[0]) {
7448                                                 drm_handle_vblank(rdev->ddev, 0);
7449                                                 rdev->pm.vblank_sync = true;
7450                                                 wake_up(&rdev->irq.vblank_queue);
7451                                         }
7452                                         if (atomic_read(&rdev->irq.pflip[0]))
7453                                                 radeon_crtc_handle_flip(rdev, 0);
7454                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7455                                         DRM_DEBUG("IH: D1 vblank\n");
7456                                 }
7457                                 break;
7458                         case 1: /* D1 vline */
7459                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7460                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7461                                         DRM_DEBUG("IH: D1 vline\n");
7462                                 }
7463                                 break;
7464                         default:
7465                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7466                                 break;
7467                         }
7468                         break;
7469                 case 2: /* D2 vblank/vline */
7470                         switch (src_data) {
7471                         case 0: /* D2 vblank */
7472                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7473                                         if (rdev->irq.crtc_vblank_int[1]) {
7474                                                 drm_handle_vblank(rdev->ddev, 1);
7475                                                 rdev->pm.vblank_sync = true;
7476                                                 wake_up(&rdev->irq.vblank_queue);
7477                                         }
7478                                         if (atomic_read(&rdev->irq.pflip[1]))
7479                                                 radeon_crtc_handle_flip(rdev, 1);
7480                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7481                                         DRM_DEBUG("IH: D2 vblank\n");
7482                                 }
7483                                 break;
7484                         case 1: /* D2 vline */
7485                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7486                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7487                                         DRM_DEBUG("IH: D2 vline\n");
7488                                 }
7489                                 break;
7490                         default:
7491                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7492                                 break;
7493                         }
7494                         break;
7495                 case 3: /* D3 vblank/vline */
7496                         switch (src_data) {
7497                         case 0: /* D3 vblank */
7498                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7499                                         if (rdev->irq.crtc_vblank_int[2]) {
7500                                                 drm_handle_vblank(rdev->ddev, 2);
7501                                                 rdev->pm.vblank_sync = true;
7502                                                 wake_up(&rdev->irq.vblank_queue);
7503                                         }
7504                                         if (atomic_read(&rdev->irq.pflip[2]))
7505                                                 radeon_crtc_handle_flip(rdev, 2);
7506                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7507                                         DRM_DEBUG("IH: D3 vblank\n");
7508                                 }
7509                                 break;
7510                         case 1: /* D3 vline */
7511                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7512                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7513                                         DRM_DEBUG("IH: D3 vline\n");
7514                                 }
7515                                 break;
7516                         default:
7517                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7518                                 break;
7519                         }
7520                         break;
7521                 case 4: /* D4 vblank/vline */
7522                         switch (src_data) {
7523                         case 0: /* D4 vblank */
7524                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7525                                         if (rdev->irq.crtc_vblank_int[3]) {
7526                                                 drm_handle_vblank(rdev->ddev, 3);
7527                                                 rdev->pm.vblank_sync = true;
7528                                                 wake_up(&rdev->irq.vblank_queue);
7529                                         }
7530                                         if (atomic_read(&rdev->irq.pflip[3]))
7531                                                 radeon_crtc_handle_flip(rdev, 3);
7532                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7533                                         DRM_DEBUG("IH: D4 vblank\n");
7534                                 }
7535                                 break;
7536                         case 1: /* D4 vline */
7537                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7538                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7539                                         DRM_DEBUG("IH: D4 vline\n");
7540                                 }
7541                                 break;
7542                         default:
7543                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7544                                 break;
7545                         }
7546                         break;
7547                 case 5: /* D5 vblank/vline */
7548                         switch (src_data) {
7549                         case 0: /* D5 vblank */
7550                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7551                                         if (rdev->irq.crtc_vblank_int[4]) {
7552                                                 drm_handle_vblank(rdev->ddev, 4);
7553                                                 rdev->pm.vblank_sync = true;
7554                                                 wake_up(&rdev->irq.vblank_queue);
7555                                         }
7556                                         if (atomic_read(&rdev->irq.pflip[4]))
7557                                                 radeon_crtc_handle_flip(rdev, 4);
7558                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7559                                         DRM_DEBUG("IH: D5 vblank\n");
7560                                 }
7561                                 break;
7562                         case 1: /* D5 vline */
7563                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7564                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7565                                         DRM_DEBUG("IH: D5 vline\n");
7566                                 }
7567                                 break;
7568                         default:
7569                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7570                                 break;
7571                         }
7572                         break;
7573                 case 6: /* D6 vblank/vline */
7574                         switch (src_data) {
7575                         case 0: /* D6 vblank */
7576                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7577                                         if (rdev->irq.crtc_vblank_int[5]) {
7578                                                 drm_handle_vblank(rdev->ddev, 5);
7579                                                 rdev->pm.vblank_sync = true;
7580                                                 wake_up(&rdev->irq.vblank_queue);
7581                                         }
7582                                         if (atomic_read(&rdev->irq.pflip[5]))
7583                                                 radeon_crtc_handle_flip(rdev, 5);
7584                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7585                                         DRM_DEBUG("IH: D6 vblank\n");
7586                                 }
7587                                 break;
7588                         case 1: /* D6 vline */
7589                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7590                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7591                                         DRM_DEBUG("IH: D6 vline\n");
7592                                 }
7593                                 break;
7594                         default:
7595                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7596                                 break;
7597                         }
7598                         break;
7599                 case 8: /* D1 page flip */
7600                 case 10: /* D2 page flip */
7601                 case 12: /* D3 page flip */
7602                 case 14: /* D4 page flip */
7603                 case 16: /* D5 page flip */
7604                 case 18: /* D6 page flip */
7605                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7606                         radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7607                         break;
7608                 case 42: /* HPD hotplug */
7609                         switch (src_data) {
7610                         case 0:
7611                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7612                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7613                                         queue_hotplug = true;
7614                                         DRM_DEBUG("IH: HPD1\n");
7615                                 }
7616                                 break;
7617                         case 1:
7618                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7619                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7620                                         queue_hotplug = true;
7621                                         DRM_DEBUG("IH: HPD2\n");
7622                                 }
7623                                 break;
7624                         case 2:
7625                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7626                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7627                                         queue_hotplug = true;
7628                                         DRM_DEBUG("IH: HPD3\n");
7629                                 }
7630                                 break;
7631                         case 3:
7632                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7633                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7634                                         queue_hotplug = true;
7635                                         DRM_DEBUG("IH: HPD4\n");
7636                                 }
7637                                 break;
7638                         case 4:
7639                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7640                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7641                                         queue_hotplug = true;
7642                                         DRM_DEBUG("IH: HPD5\n");
7643                                 }
7644                                 break;
7645                         case 5:
7646                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7647                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7648                                         queue_hotplug = true;
7649                                         DRM_DEBUG("IH: HPD6\n");
7650                                 }
7651                                 break;
7652                         default:
7653                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7654                                 break;
7655                         }
7656                         break;
7657                 case 124: /* UVD */
7658                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7659                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7660                         break;
7661                 case 146:
7662                 case 147:
7663                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7664                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7665                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7666                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7667                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7668                                 addr);
7669                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7670                                 status);
7671                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7672                         /* reset addr and status */
7673                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7674                         break;
7675                 case 167: /* VCE */
7676                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7677                         switch (src_data) {
7678                         case 0:
7679                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7680                                 break;
7681                         case 1:
7682                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7683                                 break;
7684                         default:
7685                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7686                                 break;
7687                         }
7688                         break;
7689                 case 176: /* GFX RB CP_INT */
7690                 case 177: /* GFX IB CP_INT */
7691                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7692                         break;
7693                 case 181: /* CP EOP event */
7694                         DRM_DEBUG("IH: CP EOP\n");
7695                         /* XXX check the bitfield order! */
7696                         me_id = (ring_id & 0x60) >> 5;
7697                         pipe_id = (ring_id & 0x18) >> 3;
7698                         queue_id = (ring_id & 0x7) >> 0;
7699                         switch (me_id) {
7700                         case 0:
7701                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7702                                 break;
7703                         case 1:
7704                         case 2:
7705                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7706                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7707                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7708                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7709                                 break;
7710                         }
7711                         break;
7712                 case 184: /* CP Privileged reg access */
7713                         DRM_ERROR("Illegal register access in command stream\n");
7714                         /* XXX check the bitfield order! */
7715                         me_id = (ring_id & 0x60) >> 5;
7716                         pipe_id = (ring_id & 0x18) >> 3;
7717                         queue_id = (ring_id & 0x7) >> 0;
7718                         switch (me_id) {
7719                         case 0:
7720                                 /* This results in a full GPU reset, but all we need to do is soft
7721                                  * reset the CP for gfx
7722                                  */
7723                                 queue_reset = true;
7724                                 break;
7725                         case 1:
7726                                 /* XXX compute */
7727                                 queue_reset = true;
7728                                 break;
7729                         case 2:
7730                                 /* XXX compute */
7731                                 queue_reset = true;
7732                                 break;
7733                         }
7734                         break;
7735                 case 185: /* CP Privileged inst */
7736                         DRM_ERROR("Illegal instruction in command stream\n");
7737                         /* XXX check the bitfield order! */
7738                         me_id = (ring_id & 0x60) >> 5;
7739                         pipe_id = (ring_id & 0x18) >> 3;
7740                         queue_id = (ring_id & 0x7) >> 0;
7741                         switch (me_id) {
7742                         case 0:
7743                                 /* This results in a full GPU reset, but all we need to do is soft
7744                                  * reset the CP for gfx
7745                                  */
7746                                 queue_reset = true;
7747                                 break;
7748                         case 1:
7749                                 /* XXX compute */
7750                                 queue_reset = true;
7751                                 break;
7752                         case 2:
7753                                 /* XXX compute */
7754                                 queue_reset = true;
7755                                 break;
7756                         }
7757                         break;
7758                 case 224: /* SDMA trap event */
7759                         /* XXX check the bitfield order! */
7760                         me_id = (ring_id & 0x3) >> 0;
7761                         queue_id = (ring_id & 0xc) >> 2;
7762                         DRM_DEBUG("IH: SDMA trap\n");
7763                         switch (me_id) {
7764                         case 0:
7765                                 switch (queue_id) {
7766                                 case 0:
7767                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7768                                         break;
7769                                 case 1:
7770                                         /* XXX compute */
7771                                         break;
7772                                 case 2:
7773                                         /* XXX compute */
7774                                         break;
7775                                 }
7776                                 break;
7777                         case 1:
7778                                 switch (queue_id) {
7779                                 case 0:
7780                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7781                                         break;
7782                                 case 1:
7783                                         /* XXX compute */
7784                                         break;
7785                                 case 2:
7786                                         /* XXX compute */
7787                                         break;
7788                                 }
7789                                 break;
7790                         }
7791                         break;
7792                 case 230: /* thermal low to high */
7793                         DRM_DEBUG("IH: thermal low to high\n");
7794                         rdev->pm.dpm.thermal.high_to_low = false;
7795                         queue_thermal = true;
7796                         break;
7797                 case 231: /* thermal high to low */
7798                         DRM_DEBUG("IH: thermal high to low\n");
7799                         rdev->pm.dpm.thermal.high_to_low = true;
7800                         queue_thermal = true;
7801                         break;
7802                 case 233: /* GUI IDLE */
7803                         DRM_DEBUG("IH: GUI idle\n");
7804                         break;
7805                 case 241: /* SDMA Privileged inst */
7806                 case 247: /* SDMA Privileged inst */
7807                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
7808                         /* XXX check the bitfield order! */
7809                         me_id = (ring_id & 0x3) >> 0;
7810                         queue_id = (ring_id & 0xc) >> 2;
7811                         switch (me_id) {
7812                         case 0:
7813                                 switch (queue_id) {
7814                                 case 0:
7815                                         queue_reset = true;
7816                                         break;
7817                                 case 1:
7818                                         /* XXX compute */
7819                                         queue_reset = true;
7820                                         break;
7821                                 case 2:
7822                                         /* XXX compute */
7823                                         queue_reset = true;
7824                                         break;
7825                                 }
7826                                 break;
7827                         case 1:
7828                                 switch (queue_id) {
7829                                 case 0:
7830                                         queue_reset = true;
7831                                         break;
7832                                 case 1:
7833                                         /* XXX compute */
7834                                         queue_reset = true;
7835                                         break;
7836                                 case 2:
7837                                         /* XXX compute */
7838                                         queue_reset = true;
7839                                         break;
7840                                 }
7841                                 break;
7842                         }
7843                         break;
7844                 default:
7845                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7846                         break;
7847                 }
7848
7849                 /* wptr/rptr are in bytes! */
7850                 rptr += 16;
7851                 rptr &= rdev->ih.ptr_mask;
7852         }
7853         if (queue_hotplug)
7854                 schedule_work(&rdev->hotplug_work);
7855         if (queue_reset)
7856                 schedule_work(&rdev->reset_work);
7857         if (queue_thermal)
7858                 schedule_work(&rdev->pm.dpm.thermal.work);
7859         rdev->ih.rptr = rptr;
7860         WREG32(IH_RB_RPTR, rdev->ih.rptr);
7861         atomic_set(&rdev->ih.lock, 0);
7862
7863         /* make sure wptr hasn't changed while processing */
7864         wptr = cik_get_ih_wptr(rdev);
7865         if (wptr != rptr)
7866                 goto restart_ih;
7867
7868         return IRQ_HANDLED;
7869 }
7870
7871 /*
7872  * startup/shutdown callbacks
7873  */
7874 /**
7875  * cik_startup - program the asic to a functional state
7876  *
7877  * @rdev: radeon_device pointer
7878  *
7879  * Programs the asic to a functional state (CIK).
7880  * Called by cik_init() and cik_resume().
7881  * Returns 0 for success, error for failure.
7882  */
7883 static int cik_startup(struct radeon_device *rdev)
7884 {
7885         struct radeon_ring *ring;
7886         int r;
7887
7888         /* enable pcie gen2/3 link */
7889         cik_pcie_gen3_enable(rdev);
7890         /* enable aspm */
7891         cik_program_aspm(rdev);
7892
7893         /* scratch needs to be initialized before MC */
7894         r = r600_vram_scratch_init(rdev);
7895         if (r)
7896                 return r;
7897
7898         cik_mc_program(rdev);
7899
7900         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7901                 r = ci_mc_load_microcode(rdev);
7902                 if (r) {
7903                         DRM_ERROR("Failed to load MC firmware!\n");
7904                         return r;
7905                 }
7906         }
7907
7908         r = cik_pcie_gart_enable(rdev);
7909         if (r)
7910                 return r;
7911         cik_gpu_init(rdev);
7912
7913         /* allocate rlc buffers */
7914         if (rdev->flags & RADEON_IS_IGP) {
7915                 if (rdev->family == CHIP_KAVERI) {
7916                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7917                         rdev->rlc.reg_list_size =
7918                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7919                 } else {
7920                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7921                         rdev->rlc.reg_list_size =
7922                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7923                 }
7924         }
7925         rdev->rlc.cs_data = ci_cs_data;
7926         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7927         r = sumo_rlc_init(rdev);
7928         if (r) {
7929                 DRM_ERROR("Failed to init rlc BOs!\n");
7930                 return r;
7931         }
7932
7933         /* allocate wb buffer */
7934         r = radeon_wb_init(rdev);
7935         if (r)
7936                 return r;
7937
7938         /* allocate mec buffers */
7939         r = cik_mec_init(rdev);
7940         if (r) {
7941                 DRM_ERROR("Failed to init MEC BOs!\n");
7942                 return r;
7943         }
7944
7945         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7946         if (r) {
7947                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7948                 return r;
7949         }
7950
7951         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7952         if (r) {
7953                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7954                 return r;
7955         }
7956
7957         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7958         if (r) {
7959                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7960                 return r;
7961         }
7962
7963         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7964         if (r) {
7965                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7966                 return r;
7967         }
7968
7969         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7970         if (r) {
7971                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7972                 return r;
7973         }
7974
7975         r = radeon_uvd_resume(rdev);
7976         if (!r) {
7977                 r = uvd_v4_2_resume(rdev);
7978                 if (!r) {
7979                         r = radeon_fence_driver_start_ring(rdev,
7980                                                            R600_RING_TYPE_UVD_INDEX);
7981                         if (r)
7982                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7983                 }
7984         }
7985         if (r)
7986                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7987
7988         r = radeon_vce_resume(rdev);
7989         if (!r) {
7990                 r = vce_v2_0_resume(rdev);
7991                 if (!r)
7992                         r = radeon_fence_driver_start_ring(rdev,
7993                                                            TN_RING_TYPE_VCE1_INDEX);
7994                 if (!r)
7995                         r = radeon_fence_driver_start_ring(rdev,
7996                                                            TN_RING_TYPE_VCE2_INDEX);
7997         }
7998         if (r) {
7999                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8000                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8001                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8002         }
8003
8004         /* Enable IRQ */
8005         if (!rdev->irq.installed) {
8006                 r = radeon_irq_kms_init(rdev);
8007                 if (r)
8008                         return r;
8009         }
8010
8011         r = cik_irq_init(rdev);
8012         if (r) {
8013                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8014                 radeon_irq_kms_fini(rdev);
8015                 return r;
8016         }
8017         cik_irq_set(rdev);
8018
8019         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8020         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8021                              PACKET3(PACKET3_NOP, 0x3FFF));
8022         if (r)
8023                 return r;
8024
8025         /* set up the compute queues */
8026         /* type-2 packets are deprecated on MEC, use type-3 instead */
8027         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8028         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8029                              PACKET3(PACKET3_NOP, 0x3FFF));
8030         if (r)
8031                 return r;
8032         ring->me = 1; /* first MEC */
8033         ring->pipe = 0; /* first pipe */
8034         ring->queue = 0; /* first queue */
8035         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8036
8037         /* type-2 packets are deprecated on MEC, use type-3 instead */
8038         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8039         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8040                              PACKET3(PACKET3_NOP, 0x3FFF));
8041         if (r)
8042                 return r;
8043         /* dGPU only have 1 MEC */
8044         ring->me = 1; /* first MEC */
8045         ring->pipe = 0; /* first pipe */
8046         ring->queue = 1; /* second queue */
8047         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8048
8049         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8050         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8051                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8052         if (r)
8053                 return r;
8054
8055         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8056         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8057                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8058         if (r)
8059                 return r;
8060
8061         r = cik_cp_resume(rdev);
8062         if (r)
8063                 return r;
8064
8065         r = cik_sdma_resume(rdev);
8066         if (r)
8067                 return r;
8068
8069         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8070         if (ring->ring_size) {
8071                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8072                                      RADEON_CP_PACKET2);
8073                 if (!r)
8074                         r = uvd_v1_0_init(rdev);
8075                 if (r)
8076                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8077         }
8078
8079         r = -ENOENT;
8080
8081         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8082         if (ring->ring_size)
8083                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8084                                      VCE_CMD_NO_OP);
8085
8086         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8087         if (ring->ring_size)
8088                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8089                                      VCE_CMD_NO_OP);
8090
8091         if (!r)
8092                 r = vce_v1_0_init(rdev);
8093         else if (r != -ENOENT)
8094                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8095
8096         r = radeon_ib_pool_init(rdev);
8097         if (r) {
8098                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8099                 return r;
8100         }
8101
8102         r = radeon_vm_manager_init(rdev);
8103         if (r) {
8104                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8105                 return r;
8106         }
8107
8108         r = dce6_audio_init(rdev);
8109         if (r)
8110                 return r;
8111
8112         return 0;
8113 }
8114
8115 /**
8116  * cik_resume - resume the asic to a functional state
8117  *
8118  * @rdev: radeon_device pointer
8119  *
8120  * Programs the asic to a functional state (CIK).
8121  * Called at resume.
8122  * Returns 0 for success, error for failure.
8123  */
8124 int cik_resume(struct radeon_device *rdev)
8125 {
8126         int r;
8127
8128         /* post card */
8129         atom_asic_init(rdev->mode_info.atom_context);
8130
8131         /* init golden registers */
8132         cik_init_golden_registers(rdev);
8133
8134         if (rdev->pm.pm_method == PM_METHOD_DPM)
8135                 radeon_pm_resume(rdev);
8136
8137         rdev->accel_working = true;
8138         r = cik_startup(rdev);
8139         if (r) {
8140                 DRM_ERROR("cik startup failed on resume\n");
8141                 rdev->accel_working = false;
8142                 return r;
8143         }
8144
8145         return r;
8146
8147 }
8148
8149 /**
8150  * cik_suspend - suspend the asic
8151  *
8152  * @rdev: radeon_device pointer
8153  *
8154  * Bring the chip into a state suitable for suspend (CIK).
8155  * Called at suspend.
8156  * Returns 0 for success.
8157  */
8158 int cik_suspend(struct radeon_device *rdev)
8159 {
8160         radeon_pm_suspend(rdev);
8161         dce6_audio_fini(rdev);
8162         radeon_vm_manager_fini(rdev);
8163         cik_cp_enable(rdev, false);
8164         cik_sdma_enable(rdev, false);
8165         uvd_v1_0_fini(rdev);
8166         radeon_uvd_suspend(rdev);
8167         radeon_vce_suspend(rdev);
8168         cik_fini_pg(rdev);
8169         cik_fini_cg(rdev);
8170         cik_irq_suspend(rdev);
8171         radeon_wb_disable(rdev);
8172         cik_pcie_gart_disable(rdev);
8173         return 0;
8174 }
8175
8176 /* Plan is to move initialization in that function and use
8177  * helper function so that radeon_device_init pretty much
8178  * do nothing more than calling asic specific function. This
8179  * should also allow to remove a bunch of callback function
8180  * like vram_info.
8181  */
8182 /**
8183  * cik_init - asic specific driver and hw init
8184  *
8185  * @rdev: radeon_device pointer
8186  *
8187  * Setup asic specific driver variables and program the hw
8188  * to a functional state (CIK).
8189  * Called at driver startup.
8190  * Returns 0 for success, errors for failure.
8191  */
8192 int cik_init(struct radeon_device *rdev)
8193 {
8194         struct radeon_ring *ring;
8195         int r;
8196
8197         /* Read BIOS */
8198         if (!radeon_get_bios(rdev)) {
8199                 if (ASIC_IS_AVIVO(rdev))
8200                         return -EINVAL;
8201         }
8202         /* Must be an ATOMBIOS */
8203         if (!rdev->is_atom_bios) {
8204                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8205                 return -EINVAL;
8206         }
8207         r = radeon_atombios_init(rdev);
8208         if (r)
8209                 return r;
8210
8211         /* Post card if necessary */
8212         if (!radeon_card_posted(rdev)) {
8213                 if (!rdev->bios) {
8214                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8215                         return -EINVAL;
8216                 }
8217                 DRM_INFO("GPU not posted. posting now...\n");
8218                 atom_asic_init(rdev->mode_info.atom_context);
8219         }
8220         /* init golden registers */
8221         cik_init_golden_registers(rdev);
8222         /* Initialize scratch registers */
8223         cik_scratch_init(rdev);
8224         /* Initialize surface registers */
8225         radeon_surface_init(rdev);
8226         /* Initialize clocks */
8227         radeon_get_clock_info(rdev->ddev);
8228
8229         /* Fence driver */
8230         r = radeon_fence_driver_init(rdev);
8231         if (r)
8232                 return r;
8233
8234         /* initialize memory controller */
8235         r = cik_mc_init(rdev);
8236         if (r)
8237                 return r;
8238         /* Memory manager */
8239         r = radeon_bo_init(rdev);
8240         if (r)
8241                 return r;
8242
8243         if (rdev->flags & RADEON_IS_IGP) {
8244                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8245                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8246                         r = cik_init_microcode(rdev);
8247                         if (r) {
8248                                 DRM_ERROR("Failed to load firmware!\n");
8249                                 return r;
8250                         }
8251                 }
8252         } else {
8253                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8254                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8255                     !rdev->mc_fw) {
8256                         r = cik_init_microcode(rdev);
8257                         if (r) {
8258                                 DRM_ERROR("Failed to load firmware!\n");
8259                                 return r;
8260                         }
8261                 }
8262         }
8263
8264         /* Initialize power management */
8265         radeon_pm_init(rdev);
8266
8267         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8268         ring->ring_obj = NULL;
8269         r600_ring_init(rdev, ring, 1024 * 1024);
8270
8271         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8272         ring->ring_obj = NULL;
8273         r600_ring_init(rdev, ring, 1024 * 1024);
8274         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8275         if (r)
8276                 return r;
8277
8278         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8279         ring->ring_obj = NULL;
8280         r600_ring_init(rdev, ring, 1024 * 1024);
8281         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8282         if (r)
8283                 return r;
8284
8285         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8286         ring->ring_obj = NULL;
8287         r600_ring_init(rdev, ring, 256 * 1024);
8288
8289         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8290         ring->ring_obj = NULL;
8291         r600_ring_init(rdev, ring, 256 * 1024);
8292
8293         r = radeon_uvd_init(rdev);
8294         if (!r) {
8295                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8296                 ring->ring_obj = NULL;
8297                 r600_ring_init(rdev, ring, 4096);
8298         }
8299
8300         r = radeon_vce_init(rdev);
8301         if (!r) {
8302                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8303                 ring->ring_obj = NULL;
8304                 r600_ring_init(rdev, ring, 4096);
8305
8306                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8307                 ring->ring_obj = NULL;
8308                 r600_ring_init(rdev, ring, 4096);
8309         }
8310
8311         rdev->ih.ring_obj = NULL;
8312         r600_ih_ring_init(rdev, 64 * 1024);
8313
8314         r = r600_pcie_gart_init(rdev);
8315         if (r)
8316                 return r;
8317
8318         rdev->accel_working = true;
8319         r = cik_startup(rdev);
8320         if (r) {
8321                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8322                 cik_cp_fini(rdev);
8323                 cik_sdma_fini(rdev);
8324                 cik_irq_fini(rdev);
8325                 sumo_rlc_fini(rdev);
8326                 cik_mec_fini(rdev);
8327                 radeon_wb_fini(rdev);
8328                 radeon_ib_pool_fini(rdev);
8329                 radeon_vm_manager_fini(rdev);
8330                 radeon_irq_kms_fini(rdev);
8331                 cik_pcie_gart_fini(rdev);
8332                 rdev->accel_working = false;
8333         }
8334
8335         /* Don't start up if the MC ucode is missing.
8336          * The default clocks and voltages before the MC ucode
8337          * is loaded are not suffient for advanced operations.
8338          */
8339         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8340                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8341                 return -EINVAL;
8342         }
8343
8344         return 0;
8345 }
8346
8347 /**
8348  * cik_fini - asic specific driver and hw fini
8349  *
8350  * @rdev: radeon_device pointer
8351  *
8352  * Tear down the asic specific driver variables and program the hw
8353  * to an idle state (CIK).
8354  * Called at driver unload.
8355  */
8356 void cik_fini(struct radeon_device *rdev)
8357 {
8358         radeon_pm_fini(rdev);
8359         cik_cp_fini(rdev);
8360         cik_sdma_fini(rdev);
8361         cik_fini_pg(rdev);
8362         cik_fini_cg(rdev);
8363         cik_irq_fini(rdev);
8364         sumo_rlc_fini(rdev);
8365         cik_mec_fini(rdev);
8366         radeon_wb_fini(rdev);
8367         radeon_vm_manager_fini(rdev);
8368         radeon_ib_pool_fini(rdev);
8369         radeon_irq_kms_fini(rdev);
8370         uvd_v1_0_fini(rdev);
8371         radeon_uvd_fini(rdev);
8372         radeon_vce_fini(rdev);
8373         cik_pcie_gart_fini(rdev);
8374         r600_vram_scratch_fini(rdev);
8375         radeon_gem_fini(rdev);
8376         radeon_fence_driver_fini(rdev);
8377         radeon_bo_fini(rdev);
8378         radeon_atombios_fini(rdev);
8379         kfree(rdev->bios);
8380         rdev->bios = NULL;
8381 }
8382
8383 void dce8_program_fmt(struct drm_encoder *encoder)
8384 {
8385         struct drm_device *dev = encoder->dev;
8386         struct radeon_device *rdev = dev->dev_private;
8387         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8388         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8389         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8390         int bpc = 0;
8391         u32 tmp = 0;
8392         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8393
8394         if (connector) {
8395                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8396                 bpc = radeon_get_monitor_bpc(connector);
8397                 dither = radeon_connector->dither;
8398         }
8399
8400         /* LVDS/eDP FMT is set up by atom */
8401         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8402                 return;
8403
8404         /* not needed for analog */
8405         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8406             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8407                 return;
8408
8409         if (bpc == 0)
8410                 return;
8411
8412         switch (bpc) {
8413         case 6:
8414                 if (dither == RADEON_FMT_DITHER_ENABLE)
8415                         /* XXX sort out optimal dither settings */
8416                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8417                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8418                 else
8419                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8420                 break;
8421         case 8:
8422                 if (dither == RADEON_FMT_DITHER_ENABLE)
8423                         /* XXX sort out optimal dither settings */
8424                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8425                                 FMT_RGB_RANDOM_ENABLE |
8426                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8427                 else
8428                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8429                 break;
8430         case 10:
8431                 if (dither == RADEON_FMT_DITHER_ENABLE)
8432                         /* XXX sort out optimal dither settings */
8433                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8434                                 FMT_RGB_RANDOM_ENABLE |
8435                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8436                 else
8437                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8438                 break;
8439         default:
8440                 /* not needed */
8441                 break;
8442         }
8443
8444         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8445 }
8446
8447 /* display watermark setup */
8448 /**
8449  * dce8_line_buffer_adjust - Set up the line buffer
8450  *
8451  * @rdev: radeon_device pointer
8452  * @radeon_crtc: the selected display controller
8453  * @mode: the current display mode on the selected display
8454  * controller
8455  *
8456  * Setup up the line buffer allocation for
8457  * the selected display controller (CIK).
8458  * Returns the line buffer size in pixels.
8459  */
8460 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8461                                    struct radeon_crtc *radeon_crtc,
8462                                    struct drm_display_mode *mode)
8463 {
8464         u32 tmp, buffer_alloc, i;
8465         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8466         /*
8467          * Line Buffer Setup
8468          * There are 6 line buffers, one for each display controllers.
8469          * There are 3 partitions per LB. Select the number of partitions
8470          * to enable based on the display width.  For display widths larger
8471          * than 4096, you need use to use 2 display controllers and combine
8472          * them using the stereo blender.
8473          */
8474         if (radeon_crtc->base.enabled && mode) {
8475                 if (mode->crtc_hdisplay < 1920) {
8476                         tmp = 1;
8477                         buffer_alloc = 2;
8478                 } else if (mode->crtc_hdisplay < 2560) {
8479                         tmp = 2;
8480                         buffer_alloc = 2;
8481                 } else if (mode->crtc_hdisplay < 4096) {
8482                         tmp = 0;
8483                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8484                 } else {
8485                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8486                         tmp = 0;
8487                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8488                 }
8489         } else {
8490                 tmp = 1;
8491                 buffer_alloc = 0;
8492         }
8493
8494         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8495                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8496
8497         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8498                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8499         for (i = 0; i < rdev->usec_timeout; i++) {
8500                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8501                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8502                         break;
8503                 udelay(1);
8504         }
8505
8506         if (radeon_crtc->base.enabled && mode) {
8507                 switch (tmp) {
8508                 case 0:
8509                 default:
8510                         return 4096 * 2;
8511                 case 1:
8512                         return 1920 * 2;
8513                 case 2:
8514                         return 2560 * 2;
8515                 }
8516         }
8517
8518         /* controller not enabled, so no lb used */
8519         return 0;
8520 }
8521
8522 /**
8523  * cik_get_number_of_dram_channels - get the number of dram channels
8524  *
8525  * @rdev: radeon_device pointer
8526  *
8527  * Look up the number of video ram channels (CIK).
8528  * Used for display watermark bandwidth calculations
8529  * Returns the number of dram channels
8530  */
8531 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8532 {
8533         u32 tmp = RREG32(MC_SHARED_CHMAP);
8534
8535         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8536         case 0:
8537         default:
8538                 return 1;
8539         case 1:
8540                 return 2;
8541         case 2:
8542                 return 4;
8543         case 3:
8544                 return 8;
8545         case 4:
8546                 return 3;
8547         case 5:
8548                 return 6;
8549         case 6:
8550                 return 10;
8551         case 7:
8552                 return 12;
8553         case 8:
8554                 return 16;
8555         }
8556 }
8557
8558 struct dce8_wm_params {
8559         u32 dram_channels; /* number of dram channels */
8560         u32 yclk;          /* bandwidth per dram data pin in kHz */
8561         u32 sclk;          /* engine clock in kHz */
8562         u32 disp_clk;      /* display clock in kHz */
8563         u32 src_width;     /* viewport width */
8564         u32 active_time;   /* active display time in ns */
8565         u32 blank_time;    /* blank time in ns */
8566         bool interlaced;    /* mode is interlaced */
8567         fixed20_12 vsc;    /* vertical scale ratio */
8568         u32 num_heads;     /* number of active crtcs */
8569         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8570         u32 lb_size;       /* line buffer allocated to pipe */
8571         u32 vtaps;         /* vertical scaler taps */
8572 };
8573
8574 /**
8575  * dce8_dram_bandwidth - get the dram bandwidth
8576  *
8577  * @wm: watermark calculation data
8578  *
8579  * Calculate the raw dram bandwidth (CIK).
8580  * Used for display watermark bandwidth calculations
8581  * Returns the dram bandwidth in MBytes/s
8582  */
8583 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8584 {
8585         /* Calculate raw DRAM Bandwidth */
8586         fixed20_12 dram_efficiency; /* 0.7 */
8587         fixed20_12 yclk, dram_channels, bandwidth;
8588         fixed20_12 a;
8589
8590         a.full = dfixed_const(1000);
8591         yclk.full = dfixed_const(wm->yclk);
8592         yclk.full = dfixed_div(yclk, a);
8593         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8594         a.full = dfixed_const(10);
8595         dram_efficiency.full = dfixed_const(7);
8596         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8597         bandwidth.full = dfixed_mul(dram_channels, yclk);
8598         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8599
8600         return dfixed_trunc(bandwidth);
8601 }
8602
8603 /**
8604  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8605  *
8606  * @wm: watermark calculation data
8607  *
8608  * Calculate the dram bandwidth used for display (CIK).
8609  * Used for display watermark bandwidth calculations
8610  * Returns the dram bandwidth for display in MBytes/s
8611  */
8612 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8613 {
8614         /* Calculate DRAM Bandwidth and the part allocated to display. */
8615         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8616         fixed20_12 yclk, dram_channels, bandwidth;
8617         fixed20_12 a;
8618
8619         a.full = dfixed_const(1000);
8620         yclk.full = dfixed_const(wm->yclk);
8621         yclk.full = dfixed_div(yclk, a);
8622         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8623         a.full = dfixed_const(10);
8624         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8625         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8626         bandwidth.full = dfixed_mul(dram_channels, yclk);
8627         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8628
8629         return dfixed_trunc(bandwidth);
8630 }
8631
8632 /**
8633  * dce8_data_return_bandwidth - get the data return bandwidth
8634  *
8635  * @wm: watermark calculation data
8636  *
8637  * Calculate the data return bandwidth used for display (CIK).
8638  * Used for display watermark bandwidth calculations
8639  * Returns the data return bandwidth in MBytes/s
8640  */
8641 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8642 {
8643         /* Calculate the display Data return Bandwidth */
8644         fixed20_12 return_efficiency; /* 0.8 */
8645         fixed20_12 sclk, bandwidth;
8646         fixed20_12 a;
8647
8648         a.full = dfixed_const(1000);
8649         sclk.full = dfixed_const(wm->sclk);
8650         sclk.full = dfixed_div(sclk, a);
8651         a.full = dfixed_const(10);
8652         return_efficiency.full = dfixed_const(8);
8653         return_efficiency.full = dfixed_div(return_efficiency, a);
8654         a.full = dfixed_const(32);
8655         bandwidth.full = dfixed_mul(a, sclk);
8656         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8657
8658         return dfixed_trunc(bandwidth);
8659 }
8660
8661 /**
8662  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8663  *
8664  * @wm: watermark calculation data
8665  *
8666  * Calculate the dmif bandwidth used for display (CIK).
8667  * Used for display watermark bandwidth calculations
8668  * Returns the dmif bandwidth in MBytes/s
8669  */
8670 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8671 {
8672         /* Calculate the DMIF Request Bandwidth */
8673         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8674         fixed20_12 disp_clk, bandwidth;
8675         fixed20_12 a, b;
8676
8677         a.full = dfixed_const(1000);
8678         disp_clk.full = dfixed_const(wm->disp_clk);
8679         disp_clk.full = dfixed_div(disp_clk, a);
8680         a.full = dfixed_const(32);
8681         b.full = dfixed_mul(a, disp_clk);
8682
8683         a.full = dfixed_const(10);
8684         disp_clk_request_efficiency.full = dfixed_const(8);
8685         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8686
8687         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8688
8689         return dfixed_trunc(bandwidth);
8690 }
8691
8692 /**
8693  * dce8_available_bandwidth - get the min available bandwidth
8694  *
8695  * @wm: watermark calculation data
8696  *
8697  * Calculate the min available bandwidth used for display (CIK).
8698  * Used for display watermark bandwidth calculations
8699  * Returns the min available bandwidth in MBytes/s
8700  */
8701 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8702 {
8703         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8704         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8705         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8706         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8707
8708         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8709 }
8710
8711 /**
8712  * dce8_average_bandwidth - get the average available bandwidth
8713  *
8714  * @wm: watermark calculation data
8715  *
8716  * Calculate the average available bandwidth used for display (CIK).
8717  * Used for display watermark bandwidth calculations
8718  * Returns the average available bandwidth in MBytes/s
8719  */
8720 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8721 {
8722         /* Calculate the display mode Average Bandwidth
8723          * DisplayMode should contain the source and destination dimensions,
8724          * timing, etc.
8725          */
8726         fixed20_12 bpp;
8727         fixed20_12 line_time;
8728         fixed20_12 src_width;
8729         fixed20_12 bandwidth;
8730         fixed20_12 a;
8731
8732         a.full = dfixed_const(1000);
8733         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8734         line_time.full = dfixed_div(line_time, a);
8735         bpp.full = dfixed_const(wm->bytes_per_pixel);
8736         src_width.full = dfixed_const(wm->src_width);
8737         bandwidth.full = dfixed_mul(src_width, bpp);
8738         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8739         bandwidth.full = dfixed_div(bandwidth, line_time);
8740
8741         return dfixed_trunc(bandwidth);
8742 }
8743
8744 /**
8745  * dce8_latency_watermark - get the latency watermark
8746  *
8747  * @wm: watermark calculation data
8748  *
8749  * Calculate the latency watermark (CIK).
8750  * Used for display watermark bandwidth calculations
8751  * Returns the latency watermark in ns
8752  */
8753 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8754 {
8755         /* First calculate the latency in ns */
8756         u32 mc_latency = 2000; /* 2000 ns. */
8757         u32 available_bandwidth = dce8_available_bandwidth(wm);
8758         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8759         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8760         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8761         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8762                 (wm->num_heads * cursor_line_pair_return_time);
8763         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8764         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8765         u32 tmp, dmif_size = 12288;
8766         fixed20_12 a, b, c;
8767
8768         if (wm->num_heads == 0)
8769                 return 0;
8770
8771         a.full = dfixed_const(2);
8772         b.full = dfixed_const(1);
8773         if ((wm->vsc.full > a.full) ||
8774             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8775             (wm->vtaps >= 5) ||
8776             ((wm->vsc.full >= a.full) && wm->interlaced))
8777                 max_src_lines_per_dst_line = 4;
8778         else
8779                 max_src_lines_per_dst_line = 2;
8780
8781         a.full = dfixed_const(available_bandwidth);
8782         b.full = dfixed_const(wm->num_heads);
8783         a.full = dfixed_div(a, b);
8784
8785         b.full = dfixed_const(mc_latency + 512);
8786         c.full = dfixed_const(wm->disp_clk);
8787         b.full = dfixed_div(b, c);
8788
8789         c.full = dfixed_const(dmif_size);
8790         b.full = dfixed_div(c, b);
8791
8792         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8793
8794         b.full = dfixed_const(1000);
8795         c.full = dfixed_const(wm->disp_clk);
8796         b.full = dfixed_div(c, b);
8797         c.full = dfixed_const(wm->bytes_per_pixel);
8798         b.full = dfixed_mul(b, c);
8799
8800         lb_fill_bw = min(tmp, dfixed_trunc(b));
8801
8802         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8803         b.full = dfixed_const(1000);
8804         c.full = dfixed_const(lb_fill_bw);
8805         b.full = dfixed_div(c, b);
8806         a.full = dfixed_div(a, b);
8807         line_fill_time = dfixed_trunc(a);
8808
8809         if (line_fill_time < wm->active_time)
8810                 return latency;
8811         else
8812                 return latency + (line_fill_time - wm->active_time);
8813
8814 }
8815
8816 /**
8817  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8818  * average and available dram bandwidth
8819  *
8820  * @wm: watermark calculation data
8821  *
8822  * Check if the display average bandwidth fits in the display
8823  * dram bandwidth (CIK).
8824  * Used for display watermark bandwidth calculations
8825  * Returns true if the display fits, false if not.
8826  */
8827 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8828 {
8829         if (dce8_average_bandwidth(wm) <=
8830             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8831                 return true;
8832         else
8833                 return false;
8834 }
8835
8836 /**
8837  * dce8_average_bandwidth_vs_available_bandwidth - check
8838  * average and available bandwidth
8839  *
8840  * @wm: watermark calculation data
8841  *
8842  * Check if the display average bandwidth fits in the display
8843  * available bandwidth (CIK).
8844  * Used for display watermark bandwidth calculations
8845  * Returns true if the display fits, false if not.
8846  */
8847 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8848 {
8849         if (dce8_average_bandwidth(wm) <=
8850             (dce8_available_bandwidth(wm) / wm->num_heads))
8851                 return true;
8852         else
8853                 return false;
8854 }
8855
8856 /**
8857  * dce8_check_latency_hiding - check latency hiding
8858  *
8859  * @wm: watermark calculation data
8860  *
8861  * Check latency hiding (CIK).
8862  * Used for display watermark bandwidth calculations
8863  * Returns true if the display fits, false if not.
8864  */
8865 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8866 {
8867         u32 lb_partitions = wm->lb_size / wm->src_width;
8868         u32 line_time = wm->active_time + wm->blank_time;
8869         u32 latency_tolerant_lines;
8870         u32 latency_hiding;
8871         fixed20_12 a;
8872
8873         a.full = dfixed_const(1);
8874         if (wm->vsc.full > a.full)
8875                 latency_tolerant_lines = 1;
8876         else {
8877                 if (lb_partitions <= (wm->vtaps + 1))
8878                         latency_tolerant_lines = 1;
8879                 else
8880                         latency_tolerant_lines = 2;
8881         }
8882
8883         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8884
8885         if (dce8_latency_watermark(wm) <= latency_hiding)
8886                 return true;
8887         else
8888                 return false;
8889 }
8890
8891 /**
8892  * dce8_program_watermarks - program display watermarks
8893  *
8894  * @rdev: radeon_device pointer
8895  * @radeon_crtc: the selected display controller
8896  * @lb_size: line buffer size
8897  * @num_heads: number of display controllers in use
8898  *
8899  * Calculate and program the display watermarks for the
8900  * selected display controller (CIK).
8901  */
8902 static void dce8_program_watermarks(struct radeon_device *rdev,
8903                                     struct radeon_crtc *radeon_crtc,
8904                                     u32 lb_size, u32 num_heads)
8905 {
8906         struct drm_display_mode *mode = &radeon_crtc->base.mode;
8907         struct dce8_wm_params wm_low, wm_high;
8908         u32 pixel_period;
8909         u32 line_time = 0;
8910         u32 latency_watermark_a = 0, latency_watermark_b = 0;
8911         u32 tmp, wm_mask;
8912
8913         if (radeon_crtc->base.enabled && num_heads && mode) {
8914                 pixel_period = 1000000 / (u32)mode->clock;
8915                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8916
8917                 /* watermark for high clocks */
8918                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8919                     rdev->pm.dpm_enabled) {
8920                         wm_high.yclk =
8921                                 radeon_dpm_get_mclk(rdev, false) * 10;
8922                         wm_high.sclk =
8923                                 radeon_dpm_get_sclk(rdev, false) * 10;
8924                 } else {
8925                         wm_high.yclk = rdev->pm.current_mclk * 10;
8926                         wm_high.sclk = rdev->pm.current_sclk * 10;
8927                 }
8928
8929                 wm_high.disp_clk = mode->clock;
8930                 wm_high.src_width = mode->crtc_hdisplay;
8931                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8932                 wm_high.blank_time = line_time - wm_high.active_time;
8933                 wm_high.interlaced = false;
8934                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8935                         wm_high.interlaced = true;
8936                 wm_high.vsc = radeon_crtc->vsc;
8937                 wm_high.vtaps = 1;
8938                 if (radeon_crtc->rmx_type != RMX_OFF)
8939                         wm_high.vtaps = 2;
8940                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8941                 wm_high.lb_size = lb_size;
8942                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8943                 wm_high.num_heads = num_heads;
8944
8945                 /* set for high clocks */
8946                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8947
8948                 /* possibly force display priority to high */
8949                 /* should really do this at mode validation time... */
8950                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8951                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8952                     !dce8_check_latency_hiding(&wm_high) ||
8953                     (rdev->disp_priority == 2)) {
8954                         DRM_DEBUG_KMS("force priority to high\n");
8955                 }
8956
8957                 /* watermark for low clocks */
8958                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8959                     rdev->pm.dpm_enabled) {
8960                         wm_low.yclk =
8961                                 radeon_dpm_get_mclk(rdev, true) * 10;
8962                         wm_low.sclk =
8963                                 radeon_dpm_get_sclk(rdev, true) * 10;
8964                 } else {
8965                         wm_low.yclk = rdev->pm.current_mclk * 10;
8966                         wm_low.sclk = rdev->pm.current_sclk * 10;
8967                 }
8968
8969                 wm_low.disp_clk = mode->clock;
8970                 wm_low.src_width = mode->crtc_hdisplay;
8971                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8972                 wm_low.blank_time = line_time - wm_low.active_time;
8973                 wm_low.interlaced = false;
8974                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8975                         wm_low.interlaced = true;
8976                 wm_low.vsc = radeon_crtc->vsc;
8977                 wm_low.vtaps = 1;
8978                 if (radeon_crtc->rmx_type != RMX_OFF)
8979                         wm_low.vtaps = 2;
8980                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8981                 wm_low.lb_size = lb_size;
8982                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8983                 wm_low.num_heads = num_heads;
8984
8985                 /* set for low clocks */
8986                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8987
8988                 /* possibly force display priority to high */
8989                 /* should really do this at mode validation time... */
8990                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8991                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8992                     !dce8_check_latency_hiding(&wm_low) ||
8993                     (rdev->disp_priority == 2)) {
8994                         DRM_DEBUG_KMS("force priority to high\n");
8995                 }
8996         }
8997
8998         /* select wm A */
8999         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9000         tmp = wm_mask;
9001         tmp &= ~LATENCY_WATERMARK_MASK(3);
9002         tmp |= LATENCY_WATERMARK_MASK(1);
9003         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9004         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9005                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9006                 LATENCY_HIGH_WATERMARK(line_time)));
9007         /* select wm B */
9008         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9009         tmp &= ~LATENCY_WATERMARK_MASK(3);
9010         tmp |= LATENCY_WATERMARK_MASK(2);
9011         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9012         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9013                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9014                 LATENCY_HIGH_WATERMARK(line_time)));
9015         /* restore original selection */
9016         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9017
9018         /* save values for DPM */
9019         radeon_crtc->line_time = line_time;
9020         radeon_crtc->wm_high = latency_watermark_a;
9021         radeon_crtc->wm_low = latency_watermark_b;
9022 }
9023
9024 /**
9025  * dce8_bandwidth_update - program display watermarks
9026  *
9027  * @rdev: radeon_device pointer
9028  *
9029  * Calculate and program the display watermarks and line
9030  * buffer allocation (CIK).
9031  */
9032 void dce8_bandwidth_update(struct radeon_device *rdev)
9033 {
9034         struct drm_display_mode *mode = NULL;
9035         u32 num_heads = 0, lb_size;
9036         int i;
9037
9038         radeon_update_display_priority(rdev);
9039
9040         for (i = 0; i < rdev->num_crtc; i++) {
9041                 if (rdev->mode_info.crtcs[i]->base.enabled)
9042                         num_heads++;
9043         }
9044         for (i = 0; i < rdev->num_crtc; i++) {
9045                 mode = &rdev->mode_info.crtcs[i]->base.mode;
9046                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9047                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9048         }
9049 }
9050
9051 /**
9052  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9053  *
9054  * @rdev: radeon_device pointer
9055  *
9056  * Fetches a GPU clock counter snapshot (SI).
9057  * Returns the 64 bit clock counter snapshot.
9058  */
9059 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9060 {
9061         uint64_t clock;
9062
9063         mutex_lock(&rdev->gpu_clock_mutex);
9064         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9065         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9066                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9067         mutex_unlock(&rdev->gpu_clock_mutex);
9068         return clock;
9069 }
9070
9071 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9072                               u32 cntl_reg, u32 status_reg)
9073 {
9074         int r, i;
9075         struct atom_clock_dividers dividers;
9076         uint32_t tmp;
9077
9078         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9079                                            clock, false, &dividers);
9080         if (r)
9081                 return r;
9082
9083         tmp = RREG32_SMC(cntl_reg);
9084         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9085         tmp |= dividers.post_divider;
9086         WREG32_SMC(cntl_reg, tmp);
9087
9088         for (i = 0; i < 100; i++) {
9089                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9090                         break;
9091                 mdelay(10);
9092         }
9093         if (i == 100)
9094                 return -ETIMEDOUT;
9095
9096         return 0;
9097 }
9098
9099 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9100 {
9101         int r = 0;
9102
9103         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9104         if (r)
9105                 return r;
9106
9107         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9108         return r;
9109 }
9110
9111 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9112 {
9113         int r, i;
9114         struct atom_clock_dividers dividers;
9115         u32 tmp;
9116
9117         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9118                                            ecclk, false, &dividers);
9119         if (r)
9120                 return r;
9121
9122         for (i = 0; i < 100; i++) {
9123                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9124                         break;
9125                 mdelay(10);
9126         }
9127         if (i == 100)
9128                 return -ETIMEDOUT;
9129
9130         tmp = RREG32_SMC(CG_ECLK_CNTL);
9131         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9132         tmp |= dividers.post_divider;
9133         WREG32_SMC(CG_ECLK_CNTL, tmp);
9134
9135         for (i = 0; i < 100; i++) {
9136                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9137                         break;
9138                 mdelay(10);
9139         }
9140         if (i == 100)
9141                 return -ETIMEDOUT;
9142
9143         return 0;
9144 }
9145
9146 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9147 {
9148         struct pci_dev *root = rdev->pdev->bus->self;
9149         int bridge_pos, gpu_pos;
9150         u32 speed_cntl, mask, current_data_rate;
9151         int ret, i;
9152         u16 tmp16;
9153
9154         if (radeon_pcie_gen2 == 0)
9155                 return;
9156
9157         if (rdev->flags & RADEON_IS_IGP)
9158                 return;
9159
9160         if (!(rdev->flags & RADEON_IS_PCIE))
9161                 return;
9162
9163         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9164         if (ret != 0)
9165                 return;
9166
9167         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9168                 return;
9169
9170         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9171         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9172                 LC_CURRENT_DATA_RATE_SHIFT;
9173         if (mask & DRM_PCIE_SPEED_80) {
9174                 if (current_data_rate == 2) {
9175                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9176                         return;
9177                 }
9178                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9179         } else if (mask & DRM_PCIE_SPEED_50) {
9180                 if (current_data_rate == 1) {
9181                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9182                         return;
9183                 }
9184                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9185         }
9186
9187         bridge_pos = pci_pcie_cap(root);
9188         if (!bridge_pos)
9189                 return;
9190
9191         gpu_pos = pci_pcie_cap(rdev->pdev);
9192         if (!gpu_pos)
9193                 return;
9194
9195         if (mask & DRM_PCIE_SPEED_80) {
9196                 /* re-try equalization if gen3 is not already enabled */
9197                 if (current_data_rate != 2) {
9198                         u16 bridge_cfg, gpu_cfg;
9199                         u16 bridge_cfg2, gpu_cfg2;
9200                         u32 max_lw, current_lw, tmp;
9201
9202                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9203                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9204
9205                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9206                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9207
9208                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9209                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9210
9211                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9212                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9213                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9214
9215                         if (current_lw < max_lw) {
9216                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9217                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9218                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9219                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9220                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9221                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9222                                 }
9223                         }
9224
9225                         for (i = 0; i < 10; i++) {
9226                                 /* check status */
9227                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9228                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9229                                         break;
9230
9231                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9232                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9233
9234                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9235                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9236
9237                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9238                                 tmp |= LC_SET_QUIESCE;
9239                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9240
9241                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9242                                 tmp |= LC_REDO_EQ;
9243                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9244
9245                                 mdelay(100);
9246
9247                                 /* linkctl */
9248                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9249                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9250                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9251                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9252
9253                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9254                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9255                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9256                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9257
9258                                 /* linkctl2 */
9259                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9260                                 tmp16 &= ~((1 << 4) | (7 << 9));
9261                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9262                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9263
9264                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9265                                 tmp16 &= ~((1 << 4) | (7 << 9));
9266                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9267                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9268
9269                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9270                                 tmp &= ~LC_SET_QUIESCE;
9271                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9272                         }
9273                 }
9274         }
9275
9276         /* set the link speed */
9277         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9278         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9279         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9280
9281         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9282         tmp16 &= ~0xf;
9283         if (mask & DRM_PCIE_SPEED_80)
9284                 tmp16 |= 3; /* gen3 */
9285         else if (mask & DRM_PCIE_SPEED_50)
9286                 tmp16 |= 2; /* gen2 */
9287         else
9288                 tmp16 |= 1; /* gen1 */
9289         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9290
9291         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9292         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9293         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9294
9295         for (i = 0; i < rdev->usec_timeout; i++) {
9296                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9297                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9298                         break;
9299                 udelay(1);
9300         }
9301 }
9302
9303 static void cik_program_aspm(struct radeon_device *rdev)
9304 {
9305         u32 data, orig;
9306         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9307         bool disable_clkreq = false;
9308
9309         if (radeon_aspm == 0)
9310                 return;
9311
9312         /* XXX double check IGPs */
9313         if (rdev->flags & RADEON_IS_IGP)
9314                 return;
9315
9316         if (!(rdev->flags & RADEON_IS_PCIE))
9317                 return;
9318
9319         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9320         data &= ~LC_XMIT_N_FTS_MASK;
9321         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9322         if (orig != data)
9323                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9324
9325         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9326         data |= LC_GO_TO_RECOVERY;
9327         if (orig != data)
9328                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9329
9330         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9331         data |= P_IGNORE_EDB_ERR;
9332         if (orig != data)
9333                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9334
9335         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9336         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9337         data |= LC_PMI_TO_L1_DIS;
9338         if (!disable_l0s)
9339                 data |= LC_L0S_INACTIVITY(7);
9340
9341         if (!disable_l1) {
9342                 data |= LC_L1_INACTIVITY(7);
9343                 data &= ~LC_PMI_TO_L1_DIS;
9344                 if (orig != data)
9345                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9346
9347                 if (!disable_plloff_in_l1) {
9348                         bool clk_req_support;
9349
9350                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9351                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9352                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9353                         if (orig != data)
9354                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9355
9356                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9357                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9358                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9359                         if (orig != data)
9360                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9361
9362                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9363                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9364                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9365                         if (orig != data)
9366                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9367
9368                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9369                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9370                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9371                         if (orig != data)
9372                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9373
9374                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9375                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9376                         data |= LC_DYN_LANES_PWR_STATE(3);
9377                         if (orig != data)
9378                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9379
9380                         if (!disable_clkreq) {
9381                                 struct pci_dev *root = rdev->pdev->bus->self;
9382                                 u32 lnkcap;
9383
9384                                 clk_req_support = false;
9385                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9386                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9387                                         clk_req_support = true;
9388                         } else {
9389                                 clk_req_support = false;
9390                         }
9391
9392                         if (clk_req_support) {
9393                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9394                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9395                                 if (orig != data)
9396                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9397
9398                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9399                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9400                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9401                                 if (orig != data)
9402                                         WREG32_SMC(THM_CLK_CNTL, data);
9403
9404                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9405                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9406                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9407                                 if (orig != data)
9408                                         WREG32_SMC(MISC_CLK_CTRL, data);
9409
9410                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9411                                 data &= ~BCLK_AS_XCLK;
9412                                 if (orig != data)
9413                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9414
9415                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9416                                 data &= ~FORCE_BIF_REFCLK_EN;
9417                                 if (orig != data)
9418                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9419
9420                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9421                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9422                                 data |= MPLL_CLKOUT_SEL(4);
9423                                 if (orig != data)
9424                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9425                         }
9426                 }
9427         } else {
9428                 if (orig != data)
9429                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9430         }
9431
9432         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9433         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9434         if (orig != data)
9435                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9436
9437         if (!disable_l0s) {
9438                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9439                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9440                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9441                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9442                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9443                                 data &= ~LC_L0S_INACTIVITY_MASK;
9444                                 if (orig != data)
9445                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9446                         }
9447                 }
9448         }
9449 }