drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "cikd.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36
  37 #include "cik.h"
  38 #include "gmc_v7_0.h"
  39 #include "cik_ih.h"
  40 #include "dce_v8_0.h"
  41 #include "gfx_v7_0.h"
  42 #include "cik_sdma.h"
  43 #include "uvd_v4_2.h"
  44 #include "vce_v2_0.h"
  45 #include "cik_dpm.h"
  46
  47 #include "uvd/uvd_4_2_d.h"
  48
  49 #include "smu/smu_7_0_1_d.h"
  50 #include "smu/smu_7_0_1_sh_mask.h"
  51
  52 #include "dce/dce_8_0_d.h"
  53 #include "dce/dce_8_0_sh_mask.h"
  54
  55 #include "bif/bif_4_1_d.h"
  56 #include "bif/bif_4_1_sh_mask.h"
  57
  58 #include "gca/gfx_7_2_d.h"
  59 #include "gca/gfx_7_2_enum.h"
  60 #include "gca/gfx_7_2_sh_mask.h"
  61
  62 #include "gmc/gmc_7_1_d.h"
  63 #include "gmc/gmc_7_1_sh_mask.h"
  64
  65 #include "oss/oss_2_0_d.h"
  66 #include "oss/oss_2_0_sh_mask.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_powerplay.h"
  70
  71 /*
  72  * Indirect registers accessor
  73  */
  74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  75 {
  76         unsigned long flags;
  77         u32 r;
  78
  79         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  80         WREG32(mmPCIE_INDEX, reg);
  81         (void)RREG32(mmPCIE_INDEX);
  82         r = RREG32(mmPCIE_DATA);
  83         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  84         return r;
  85 }
  86
  87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  88 {
  89         unsigned long flags;
  90
  91         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  92         WREG32(mmPCIE_INDEX, reg);
  93         (void)RREG32(mmPCIE_INDEX);
  94         WREG32(mmPCIE_DATA, v);
  95         (void)RREG32(mmPCIE_DATA);
  96         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  97 }
  98
  99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 100 {
 101         unsigned long flags;
 102         u32 r;
 103
 104         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 105         WREG32(mmSMC_IND_INDEX_0, (reg));
 106         r = RREG32(mmSMC_IND_DATA_0);
 107         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 108         return r;
 109 }
 110
 111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 112 {
 113         unsigned long flags;
 114
 115         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 116         WREG32(mmSMC_IND_INDEX_0, (reg));
 117         WREG32(mmSMC_IND_DATA_0, (v));
 118         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 119 }
 120
 121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 122 {
 123         unsigned long flags;
 124         u32 r;
 125
 126         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 127         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 128         r = RREG32(mmUVD_CTX_DATA);
 129         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 130         return r;
 131 }
 132
 133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 134 {
 135         unsigned long flags;
 136
 137         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 138         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 139         WREG32(mmUVD_CTX_DATA, (v));
 140         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 141 }
 142
 143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 144 {
 145         unsigned long flags;
 146         u32 r;
 147
 148         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 149         WREG32(mmDIDT_IND_INDEX, (reg));
 150         r = RREG32(mmDIDT_IND_DATA);
 151         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 152         return r;
 153 }
 154
 155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 160         WREG32(mmDIDT_IND_INDEX, (reg));
 161         WREG32(mmDIDT_IND_DATA, (v));
 162         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 163 }
 164
 165 static const u32 bonaire_golden_spm_registers[] =
 166 {
 167         0xc200, 0xe0ffffff, 0xe0000000
 168 };
 169
 170 static const u32 bonaire_golden_common_registers[] =
 171 {
 172         0x31dc, 0xffffffff, 0x00000800,
 173         0x31dd, 0xffffffff, 0x00000800,
 174         0x31e6, 0xffffffff, 0x00007fbf,
 175         0x31e7, 0xffffffff, 0x00007faf
 176 };
 177
 178 static const u32 bonaire_golden_registers[] =
 179 {
 180         0xcd5, 0x00000333, 0x00000333,
 181         0xcd4, 0x000c0fc0, 0x00040200,
 182         0x2684, 0x00010000, 0x00058208,
 183         0xf000, 0xffff1fff, 0x00140000,
 184         0xf080, 0xfdfc0fff, 0x00000100,
 185         0xf08d, 0x40000000, 0x40000200,
 186         0x260c, 0xffffffff, 0x00000000,
 187         0x260d, 0xf00fffff, 0x00000400,
 188         0x260e, 0x0002021c, 0x00020200,
 189         0x31e, 0x00000080, 0x00000000,
 190         0x16ec, 0x000000f0, 0x00000070,
 191         0x16f0, 0xf0311fff, 0x80300000,
 192         0x263e, 0x73773777, 0x12010001,
 193         0xd43, 0x00810000, 0x408af000,
 194         0x1c0c, 0x31000111, 0x00000011,
 195         0xbd2, 0x73773777, 0x12010001,
 196         0x883, 0x00007fb6, 0x0021a1b1,
 197         0x884, 0x00007fb6, 0x002021b1,
 198         0x860, 0x00007fb6, 0x00002191,
 199         0x886, 0x00007fb6, 0x002121b1,
 200         0x887, 0x00007fb6, 0x002021b1,
 201         0x877, 0x00007fb6, 0x00002191,
 202         0x878, 0x00007fb6, 0x00002191,
 203         0xd8a, 0x0000003f, 0x0000000a,
 204         0xd8b, 0x0000003f, 0x0000000a,
 205         0xab9, 0x00073ffe, 0x000022a2,
 206         0x903, 0x000007ff, 0x00000000,
 207         0x2285, 0xf000003f, 0x00000007,
 208         0x22fc, 0x00002001, 0x00000001,
 209         0x22c9, 0xffffffff, 0x00ffffff,
 210         0xc281, 0x0000ff0f, 0x00000000,
 211         0xa293, 0x07ffffff, 0x06000000,
 212         0x136, 0x00000fff, 0x00000100,
 213         0xf9e, 0x00000001, 0x00000002,
 214         0x2440, 0x03000000, 0x0362c688,
 215         0x2300, 0x000000ff, 0x00000001,
 216         0x390, 0x00001fff, 0x00001fff,
 217         0x2418, 0x0000007f, 0x00000020,
 218         0x2542, 0x00010000, 0x00010000,
 219         0x2b05, 0x000003ff, 0x000000f3,
 220         0x2b03, 0xffffffff, 0x00001032
 221 };
 222
 223 static const u32 bonaire_mgcg_cgcg_init[] =
 224 {
 225         0x3108, 0xffffffff, 0xfffffffc,
 226         0xc200, 0xffffffff, 0xe0000000,
 227         0xf0a8, 0xffffffff, 0x00000100,
 228         0xf082, 0xffffffff, 0x00000100,
 229         0xf0b0, 0xffffffff, 0xc0000100,
 230         0xf0b2, 0xffffffff, 0xc0000100,
 231         0xf0b1, 0xffffffff, 0xc0000100,
 232         0x1579, 0xffffffff, 0x00600100,
 233         0xf0a0, 0xffffffff, 0x00000100,
 234         0xf085, 0xffffffff, 0x06000100,
 235         0xf088, 0xffffffff, 0x00000100,
 236         0xf086, 0xffffffff, 0x06000100,
 237         0xf081, 0xffffffff, 0x00000100,
 238         0xf0b8, 0xffffffff, 0x00000100,
 239         0xf089, 0xffffffff, 0x00000100,
 240         0xf080, 0xffffffff, 0x00000100,
 241         0xf08c, 0xffffffff, 0x00000100,
 242         0xf08d, 0xffffffff, 0x00000100,
 243         0xf094, 0xffffffff, 0x00000100,
 244         0xf095, 0xffffffff, 0x00000100,
 245         0xf096, 0xffffffff, 0x00000100,
 246         0xf097, 0xffffffff, 0x00000100,
 247         0xf098, 0xffffffff, 0x00000100,
 248         0xf09f, 0xffffffff, 0x00000100,
 249         0xf09e, 0xffffffff, 0x00000100,
 250         0xf084, 0xffffffff, 0x06000100,
 251         0xf0a4, 0xffffffff, 0x00000100,
 252         0xf09d, 0xffffffff, 0x00000100,
 253         0xf0ad, 0xffffffff, 0x00000100,
 254         0xf0ac, 0xffffffff, 0x00000100,
 255         0xf09c, 0xffffffff, 0x00000100,
 256         0xc200, 0xffffffff, 0xe0000000,
 257         0xf008, 0xffffffff, 0x00010000,
 258         0xf009, 0xffffffff, 0x00030002,
 259         0xf00a, 0xffffffff, 0x00040007,
 260         0xf00b, 0xffffffff, 0x00060005,
 261         0xf00c, 0xffffffff, 0x00090008,
 262         0xf00d, 0xffffffff, 0x00010000,
 263         0xf00e, 0xffffffff, 0x00030002,
 264         0xf00f, 0xffffffff, 0x00040007,
 265         0xf010, 0xffffffff, 0x00060005,
 266         0xf011, 0xffffffff, 0x00090008,
 267         0xf012, 0xffffffff, 0x00010000,
 268         0xf013, 0xffffffff, 0x00030002,
 269         0xf014, 0xffffffff, 0x00040007,
 270         0xf015, 0xffffffff, 0x00060005,
 271         0xf016, 0xffffffff, 0x00090008,
 272         0xf017, 0xffffffff, 0x00010000,
 273         0xf018, 0xffffffff, 0x00030002,
 274         0xf019, 0xffffffff, 0x00040007,
 275         0xf01a, 0xffffffff, 0x00060005,
 276         0xf01b, 0xffffffff, 0x00090008,
 277         0xf01c, 0xffffffff, 0x00010000,
 278         0xf01d, 0xffffffff, 0x00030002,
 279         0xf01e, 0xffffffff, 0x00040007,
 280         0xf01f, 0xffffffff, 0x00060005,
 281         0xf020, 0xffffffff, 0x00090008,
 282         0xf021, 0xffffffff, 0x00010000,
 283         0xf022, 0xffffffff, 0x00030002,
 284         0xf023, 0xffffffff, 0x00040007,
 285         0xf024, 0xffffffff, 0x00060005,
 286         0xf025, 0xffffffff, 0x00090008,
 287         0xf026, 0xffffffff, 0x00010000,
 288         0xf027, 0xffffffff, 0x00030002,
 289         0xf028, 0xffffffff, 0x00040007,
 290         0xf029, 0xffffffff, 0x00060005,
 291         0xf02a, 0xffffffff, 0x00090008,
 292         0xf000, 0xffffffff, 0x96e00200,
 293         0x21c2, 0xffffffff, 0x00900100,
 294         0x3109, 0xffffffff, 0x0020003f,
 295         0xe, 0xffffffff, 0x0140001c,
 296         0xf, 0x000f0000, 0x000f0000,
 297         0x88, 0xffffffff, 0xc060000c,
 298         0x89, 0xc0000fff, 0x00000100,
 299         0x3e4, 0xffffffff, 0x00000100,
 300         0x3e6, 0x00000101, 0x00000000,
 301         0x82a, 0xffffffff, 0x00000104,
 302         0x1579, 0xff000fff, 0x00000100,
 303         0xc33, 0xc0000fff, 0x00000104,
 304         0x3079, 0x00000001, 0x00000001,
 305         0x3403, 0xff000ff0, 0x00000100,
 306         0x3603, 0xff000ff0, 0x00000100
 307 };
 308
 309 static const u32 spectre_golden_spm_registers[] =
 310 {
 311         0xc200, 0xe0ffffff, 0xe0000000
 312 };
 313
 314 static const u32 spectre_golden_common_registers[] =
 315 {
 316         0x31dc, 0xffffffff, 0x00000800,
 317         0x31dd, 0xffffffff, 0x00000800,
 318         0x31e6, 0xffffffff, 0x00007fbf,
 319         0x31e7, 0xffffffff, 0x00007faf
 320 };
 321
 322 static const u32 spectre_golden_registers[] =
 323 {
 324         0xf000, 0xffff1fff, 0x96940200,
 325         0xf003, 0xffff0001, 0xff000000,
 326         0xf080, 0xfffc0fff, 0x00000100,
 327         0x1bb6, 0x00010101, 0x00010000,
 328         0x260d, 0xf00fffff, 0x00000400,
 329         0x260e, 0xfffffffc, 0x00020200,
 330         0x16ec, 0x000000f0, 0x00000070,
 331         0x16f0, 0xf0311fff, 0x80300000,
 332         0x263e, 0x73773777, 0x12010001,
 333         0x26df, 0x00ff0000, 0x00fc0000,
 334         0xbd2, 0x73773777, 0x12010001,
 335         0x2285, 0xf000003f, 0x00000007,
 336         0x22c9, 0xffffffff, 0x00ffffff,
 337         0xa0d4, 0x3f3f3fff, 0x00000082,
 338         0xa0d5, 0x0000003f, 0x00000000,
 339         0xf9e, 0x00000001, 0x00000002,
 340         0x244f, 0xffff03df, 0x00000004,
 341         0x31da, 0x00000008, 0x00000008,
 342         0x2300, 0x000008ff, 0x00000800,
 343         0x2542, 0x00010000, 0x00010000,
 344         0x2b03, 0xffffffff, 0x54763210,
 345         0x853e, 0x01ff01ff, 0x00000002,
 346         0x8526, 0x007ff800, 0x00200000,
 347         0x8057, 0xffffffff, 0x00000f40,
 348         0xc24d, 0xffffffff, 0x00000001
 349 };
 350
 351 static const u32 spectre_mgcg_cgcg_init[] =
 352 {
 353         0x3108, 0xffffffff, 0xfffffffc,
 354         0xc200, 0xffffffff, 0xe0000000,
 355         0xf0a8, 0xffffffff, 0x00000100,
 356         0xf082, 0xffffffff, 0x00000100,
 357         0xf0b0, 0xffffffff, 0x00000100,
 358         0xf0b2, 0xffffffff, 0x00000100,
 359         0xf0b1, 0xffffffff, 0x00000100,
 360         0x1579, 0xffffffff, 0x00600100,
 361         0xf0a0, 0xffffffff, 0x00000100,
 362         0xf085, 0xffffffff, 0x06000100,
 363         0xf088, 0xffffffff, 0x00000100,
 364         0xf086, 0xffffffff, 0x06000100,
 365         0xf081, 0xffffffff, 0x00000100,
 366         0xf0b8, 0xffffffff, 0x00000100,
 367         0xf089, 0xffffffff, 0x00000100,
 368         0xf080, 0xffffffff, 0x00000100,
 369         0xf08c, 0xffffffff, 0x00000100,
 370         0xf08d, 0xffffffff, 0x00000100,
 371         0xf094, 0xffffffff, 0x00000100,
 372         0xf095, 0xffffffff, 0x00000100,
 373         0xf096, 0xffffffff, 0x00000100,
 374         0xf097, 0xffffffff, 0x00000100,
 375         0xf098, 0xffffffff, 0x00000100,
 376         0xf09f, 0xffffffff, 0x00000100,
 377         0xf09e, 0xffffffff, 0x00000100,
 378         0xf084, 0xffffffff, 0x06000100,
 379         0xf0a4, 0xffffffff, 0x00000100,
 380         0xf09d, 0xffffffff, 0x00000100,
 381         0xf0ad, 0xffffffff, 0x00000100,
 382         0xf0ac, 0xffffffff, 0x00000100,
 383         0xf09c, 0xffffffff, 0x00000100,
 384         0xc200, 0xffffffff, 0xe0000000,
 385         0xf008, 0xffffffff, 0x00010000,
 386         0xf009, 0xffffffff, 0x00030002,
 387         0xf00a, 0xffffffff, 0x00040007,
 388         0xf00b, 0xffffffff, 0x00060005,
 389         0xf00c, 0xffffffff, 0x00090008,
 390         0xf00d, 0xffffffff, 0x00010000,
 391         0xf00e, 0xffffffff, 0x00030002,
 392         0xf00f, 0xffffffff, 0x00040007,
 393         0xf010, 0xffffffff, 0x00060005,
 394         0xf011, 0xffffffff, 0x00090008,
 395         0xf012, 0xffffffff, 0x00010000,
 396         0xf013, 0xffffffff, 0x00030002,
 397         0xf014, 0xffffffff, 0x00040007,
 398         0xf015, 0xffffffff, 0x00060005,
 399         0xf016, 0xffffffff, 0x00090008,
 400         0xf017, 0xffffffff, 0x00010000,
 401         0xf018, 0xffffffff, 0x00030002,
 402         0xf019, 0xffffffff, 0x00040007,
 403         0xf01a, 0xffffffff, 0x00060005,
 404         0xf01b, 0xffffffff, 0x00090008,
 405         0xf01c, 0xffffffff, 0x00010000,
 406         0xf01d, 0xffffffff, 0x00030002,
 407         0xf01e, 0xffffffff, 0x00040007,
 408         0xf01f, 0xffffffff, 0x00060005,
 409         0xf020, 0xffffffff, 0x00090008,
 410         0xf021, 0xffffffff, 0x00010000,
 411         0xf022, 0xffffffff, 0x00030002,
 412         0xf023, 0xffffffff, 0x00040007,
 413         0xf024, 0xffffffff, 0x00060005,
 414         0xf025, 0xffffffff, 0x00090008,
 415         0xf026, 0xffffffff, 0x00010000,
 416         0xf027, 0xffffffff, 0x00030002,
 417         0xf028, 0xffffffff, 0x00040007,
 418         0xf029, 0xffffffff, 0x00060005,
 419         0xf02a, 0xffffffff, 0x00090008,
 420         0xf02b, 0xffffffff, 0x00010000,
 421         0xf02c, 0xffffffff, 0x00030002,
 422         0xf02d, 0xffffffff, 0x00040007,
 423         0xf02e, 0xffffffff, 0x00060005,
 424         0xf02f, 0xffffffff, 0x00090008,
 425         0xf000, 0xffffffff, 0x96e00200,
 426         0x21c2, 0xffffffff, 0x00900100,
 427         0x3109, 0xffffffff, 0x0020003f,
 428         0xe, 0xffffffff, 0x0140001c,
 429         0xf, 0x000f0000, 0x000f0000,
 430         0x88, 0xffffffff, 0xc060000c,
 431         0x89, 0xc0000fff, 0x00000100,
 432         0x3e4, 0xffffffff, 0x00000100,
 433         0x3e6, 0x00000101, 0x00000000,
 434         0x82a, 0xffffffff, 0x00000104,
 435         0x1579, 0xff000fff, 0x00000100,
 436         0xc33, 0xc0000fff, 0x00000104,
 437         0x3079, 0x00000001, 0x00000001,
 438         0x3403, 0xff000ff0, 0x00000100,
 439         0x3603, 0xff000ff0, 0x00000100
 440 };
 441
 442 static const u32 kalindi_golden_spm_registers[] =
 443 {
 444         0xc200, 0xe0ffffff, 0xe0000000
 445 };
 446
 447 static const u32 kalindi_golden_common_registers[] =
 448 {
 449         0x31dc, 0xffffffff, 0x00000800,
 450         0x31dd, 0xffffffff, 0x00000800,
 451         0x31e6, 0xffffffff, 0x00007fbf,
 452         0x31e7, 0xffffffff, 0x00007faf
 453 };
 454
 455 static const u32 kalindi_golden_registers[] =
 456 {
 457         0xf000, 0xffffdfff, 0x6e944040,
 458         0x1579, 0xff607fff, 0xfc000100,
 459         0xf088, 0xff000fff, 0x00000100,
 460         0xf089, 0xff000fff, 0x00000100,
 461         0xf080, 0xfffc0fff, 0x00000100,
 462         0x1bb6, 0x00010101, 0x00010000,
 463         0x260c, 0xffffffff, 0x00000000,
 464         0x260d, 0xf00fffff, 0x00000400,
 465         0x16ec, 0x000000f0, 0x00000070,
 466         0x16f0, 0xf0311fff, 0x80300000,
 467         0x263e, 0x73773777, 0x12010001,
 468         0x263f, 0xffffffff, 0x00000010,
 469         0x26df, 0x00ff0000, 0x00fc0000,
 470         0x200c, 0x00001f0f, 0x0000100a,
 471         0xbd2, 0x73773777, 0x12010001,
 472         0x902, 0x000fffff, 0x000c007f,
 473         0x2285, 0xf000003f, 0x00000007,
 474         0x22c9, 0x3fff3fff, 0x00ffcfff,
 475         0xc281, 0x0000ff0f, 0x00000000,
 476         0xa293, 0x07ffffff, 0x06000000,
 477         0x136, 0x00000fff, 0x00000100,
 478         0xf9e, 0x00000001, 0x00000002,
 479         0x31da, 0x00000008, 0x00000008,
 480         0x2300, 0x000000ff, 0x00000003,
 481         0x853e, 0x01ff01ff, 0x00000002,
 482         0x8526, 0x007ff800, 0x00200000,
 483         0x8057, 0xffffffff, 0x00000f40,
 484         0x2231, 0x001f3ae3, 0x00000082,
 485         0x2235, 0x0000001f, 0x00000010,
 486         0xc24d, 0xffffffff, 0x00000000
 487 };
 488
 489 static const u32 kalindi_mgcg_cgcg_init[] =
 490 {
 491         0x3108, 0xffffffff, 0xfffffffc,
 492         0xc200, 0xffffffff, 0xe0000000,
 493         0xf0a8, 0xffffffff, 0x00000100,
 494         0xf082, 0xffffffff, 0x00000100,
 495         0xf0b0, 0xffffffff, 0x00000100,
 496         0xf0b2, 0xffffffff, 0x00000100,
 497         0xf0b1, 0xffffffff, 0x00000100,
 498         0x1579, 0xffffffff, 0x00600100,
 499         0xf0a0, 0xffffffff, 0x00000100,
 500         0xf085, 0xffffffff, 0x06000100,
 501         0xf088, 0xffffffff, 0x00000100,
 502         0xf086, 0xffffffff, 0x06000100,
 503         0xf081, 0xffffffff, 0x00000100,
 504         0xf0b8, 0xffffffff, 0x00000100,
 505         0xf089, 0xffffffff, 0x00000100,
 506         0xf080, 0xffffffff, 0x00000100,
 507         0xf08c, 0xffffffff, 0x00000100,
 508         0xf08d, 0xffffffff, 0x00000100,
 509         0xf094, 0xffffffff, 0x00000100,
 510         0xf095, 0xffffffff, 0x00000100,
 511         0xf096, 0xffffffff, 0x00000100,
 512         0xf097, 0xffffffff, 0x00000100,
 513         0xf098, 0xffffffff, 0x00000100,
 514         0xf09f, 0xffffffff, 0x00000100,
 515         0xf09e, 0xffffffff, 0x00000100,
 516         0xf084, 0xffffffff, 0x06000100,
 517         0xf0a4, 0xffffffff, 0x00000100,
 518         0xf09d, 0xffffffff, 0x00000100,
 519         0xf0ad, 0xffffffff, 0x00000100,
 520         0xf0ac, 0xffffffff, 0x00000100,
 521         0xf09c, 0xffffffff, 0x00000100,
 522         0xc200, 0xffffffff, 0xe0000000,
 523         0xf008, 0xffffffff, 0x00010000,
 524         0xf009, 0xffffffff, 0x00030002,
 525         0xf00a, 0xffffffff, 0x00040007,
 526         0xf00b, 0xffffffff, 0x00060005,
 527         0xf00c, 0xffffffff, 0x00090008,
 528         0xf00d, 0xffffffff, 0x00010000,
 529         0xf00e, 0xffffffff, 0x00030002,
 530         0xf00f, 0xffffffff, 0x00040007,
 531         0xf010, 0xffffffff, 0x00060005,
 532         0xf011, 0xffffffff, 0x00090008,
 533         0xf000, 0xffffffff, 0x96e00200,
 534         0x21c2, 0xffffffff, 0x00900100,
 535         0x3109, 0xffffffff, 0x0020003f,
 536         0xe, 0xffffffff, 0x0140001c,
 537         0xf, 0x000f0000, 0x000f0000,
 538         0x88, 0xffffffff, 0xc060000c,
 539         0x89, 0xc0000fff, 0x00000100,
 540         0x82a, 0xffffffff, 0x00000104,
 541         0x1579, 0xff000fff, 0x00000100,
 542         0xc33, 0xc0000fff, 0x00000104,
 543         0x3079, 0x00000001, 0x00000001,
 544         0x3403, 0xff000ff0, 0x00000100,
 545         0x3603, 0xff000ff0, 0x00000100
 546 };
 547
 548 static const u32 hawaii_golden_spm_registers[] =
 549 {
 550         0xc200, 0xe0ffffff, 0xe0000000
 551 };
 552
 553 static const u32 hawaii_golden_common_registers[] =
 554 {
 555         0xc200, 0xffffffff, 0xe0000000,
 556         0xa0d4, 0xffffffff, 0x3a00161a,
 557         0xa0d5, 0xffffffff, 0x0000002e,
 558         0x2684, 0xffffffff, 0x00018208,
 559         0x263e, 0xffffffff, 0x12011003
 560 };
 561
 562 static const u32 hawaii_golden_registers[] =
 563 {
 564         0xcd5, 0x00000333, 0x00000333,
 565         0x2684, 0x00010000, 0x00058208,
 566         0x260c, 0xffffffff, 0x00000000,
 567         0x260d, 0xf00fffff, 0x00000400,
 568         0x260e, 0x0002021c, 0x00020200,
 569         0x31e, 0x00000080, 0x00000000,
 570         0x16ec, 0x000000f0, 0x00000070,
 571         0x16f0, 0xf0311fff, 0x80300000,
 572         0xd43, 0x00810000, 0x408af000,
 573         0x1c0c, 0x31000111, 0x00000011,
 574         0xbd2, 0x73773777, 0x12010001,
 575         0x848, 0x0000007f, 0x0000001b,
 576         0x877, 0x00007fb6, 0x00002191,
 577         0xd8a, 0x0000003f, 0x0000000a,
 578         0xd8b, 0x0000003f, 0x0000000a,
 579         0xab9, 0x00073ffe, 0x000022a2,
 580         0x903, 0x000007ff, 0x00000000,
 581         0x22fc, 0x00002001, 0x00000001,
 582         0x22c9, 0xffffffff, 0x00ffffff,
 583         0xc281, 0x0000ff0f, 0x00000000,
 584         0xa293, 0x07ffffff, 0x06000000,
 585         0xf9e, 0x00000001, 0x00000002,
 586         0x31da, 0x00000008, 0x00000008,
 587         0x31dc, 0x00000f00, 0x00000800,
 588         0x31dd, 0x00000f00, 0x00000800,
 589         0x31e6, 0x00ffffff, 0x00ff7fbf,
 590         0x31e7, 0x00ffffff, 0x00ff7faf,
 591         0x2300, 0x000000ff, 0x00000800,
 592         0x390, 0x00001fff, 0x00001fff,
 593         0x2418, 0x0000007f, 0x00000020,
 594         0x2542, 0x00010000, 0x00010000,
 595         0x2b80, 0x00100000, 0x000ff07c,
 596         0x2b05, 0x000003ff, 0x0000000f,
 597         0x2b04, 0xffffffff, 0x7564fdec,
 598         0x2b03, 0xffffffff, 0x3120b9a8,
 599         0x2b02, 0x20000000, 0x0f9c0000
 600 };
 601
 602 static const u32 hawaii_mgcg_cgcg_init[] =
 603 {
 604         0x3108, 0xffffffff, 0xfffffffd,
 605         0xc200, 0xffffffff, 0xe0000000,
 606         0xf0a8, 0xffffffff, 0x00000100,
 607         0xf082, 0xffffffff, 0x00000100,
 608         0xf0b0, 0xffffffff, 0x00000100,
 609         0xf0b2, 0xffffffff, 0x00000100,
 610         0xf0b1, 0xffffffff, 0x00000100,
 611         0x1579, 0xffffffff, 0x00200100,
 612         0xf0a0, 0xffffffff, 0x00000100,
 613         0xf085, 0xffffffff, 0x06000100,
 614         0xf088, 0xffffffff, 0x00000100,
 615         0xf086, 0xffffffff, 0x06000100,
 616         0xf081, 0xffffffff, 0x00000100,
 617         0xf0b8, 0xffffffff, 0x00000100,
 618         0xf089, 0xffffffff, 0x00000100,
 619         0xf080, 0xffffffff, 0x00000100,
 620         0xf08c, 0xffffffff, 0x00000100,
 621         0xf08d, 0xffffffff, 0x00000100,
 622         0xf094, 0xffffffff, 0x00000100,
 623         0xf095, 0xffffffff, 0x00000100,
 624         0xf096, 0xffffffff, 0x00000100,
 625         0xf097, 0xffffffff, 0x00000100,
 626         0xf098, 0xffffffff, 0x00000100,
 627         0xf09f, 0xffffffff, 0x00000100,
 628         0xf09e, 0xffffffff, 0x00000100,
 629         0xf084, 0xffffffff, 0x06000100,
 630         0xf0a4, 0xffffffff, 0x00000100,
 631         0xf09d, 0xffffffff, 0x00000100,
 632         0xf0ad, 0xffffffff, 0x00000100,
 633         0xf0ac, 0xffffffff, 0x00000100,
 634         0xf09c, 0xffffffff, 0x00000100,
 635         0xc200, 0xffffffff, 0xe0000000,
 636         0xf008, 0xffffffff, 0x00010000,
 637         0xf009, 0xffffffff, 0x00030002,
 638         0xf00a, 0xffffffff, 0x00040007,
 639         0xf00b, 0xffffffff, 0x00060005,
 640         0xf00c, 0xffffffff, 0x00090008,
 641         0xf00d, 0xffffffff, 0x00010000,
 642         0xf00e, 0xffffffff, 0x00030002,
 643         0xf00f, 0xffffffff, 0x00040007,
 644         0xf010, 0xffffffff, 0x00060005,
 645         0xf011, 0xffffffff, 0x00090008,
 646         0xf012, 0xffffffff, 0x00010000,
 647         0xf013, 0xffffffff, 0x00030002,
 648         0xf014, 0xffffffff, 0x00040007,
 649         0xf015, 0xffffffff, 0x00060005,
 650         0xf016, 0xffffffff, 0x00090008,
 651         0xf017, 0xffffffff, 0x00010000,
 652         0xf018, 0xffffffff, 0x00030002,
 653         0xf019, 0xffffffff, 0x00040007,
 654         0xf01a, 0xffffffff, 0x00060005,
 655         0xf01b, 0xffffffff, 0x00090008,
 656         0xf01c, 0xffffffff, 0x00010000,
 657         0xf01d, 0xffffffff, 0x00030002,
 658         0xf01e, 0xffffffff, 0x00040007,
 659         0xf01f, 0xffffffff, 0x00060005,
 660         0xf020, 0xffffffff, 0x00090008,
 661         0xf021, 0xffffffff, 0x00010000,
 662         0xf022, 0xffffffff, 0x00030002,
 663         0xf023, 0xffffffff, 0x00040007,
 664         0xf024, 0xffffffff, 0x00060005,
 665         0xf025, 0xffffffff, 0x00090008,
 666         0xf026, 0xffffffff, 0x00010000,
 667         0xf027, 0xffffffff, 0x00030002,
 668         0xf028, 0xffffffff, 0x00040007,
 669         0xf029, 0xffffffff, 0x00060005,
 670         0xf02a, 0xffffffff, 0x00090008,
 671         0xf02b, 0xffffffff, 0x00010000,
 672         0xf02c, 0xffffffff, 0x00030002,
 673         0xf02d, 0xffffffff, 0x00040007,
 674         0xf02e, 0xffffffff, 0x00060005,
 675         0xf02f, 0xffffffff, 0x00090008,
 676         0xf030, 0xffffffff, 0x00010000,
 677         0xf031, 0xffffffff, 0x00030002,
 678         0xf032, 0xffffffff, 0x00040007,
 679         0xf033, 0xffffffff, 0x00060005,
 680         0xf034, 0xffffffff, 0x00090008,
 681         0xf035, 0xffffffff, 0x00010000,
 682         0xf036, 0xffffffff, 0x00030002,
 683         0xf037, 0xffffffff, 0x00040007,
 684         0xf038, 0xffffffff, 0x00060005,
 685         0xf039, 0xffffffff, 0x00090008,
 686         0xf03a, 0xffffffff, 0x00010000,
 687         0xf03b, 0xffffffff, 0x00030002,
 688         0xf03c, 0xffffffff, 0x00040007,
 689         0xf03d, 0xffffffff, 0x00060005,
 690         0xf03e, 0xffffffff, 0x00090008,
 691         0x30c6, 0xffffffff, 0x00020200,
 692         0xcd4, 0xffffffff, 0x00000200,
 693         0x570, 0xffffffff, 0x00000400,
 694         0x157a, 0xffffffff, 0x00000000,
 695         0xbd4, 0xffffffff, 0x00000902,
 696         0xf000, 0xffffffff, 0x96940200,
 697         0x21c2, 0xffffffff, 0x00900100,
 698         0x3109, 0xffffffff, 0x0020003f,
 699         0xe, 0xffffffff, 0x0140001c,
 700         0xf, 0x000f0000, 0x000f0000,
 701         0x88, 0xffffffff, 0xc060000c,
 702         0x89, 0xc0000fff, 0x00000100,
 703         0x3e4, 0xffffffff, 0x00000100,
 704         0x3e6, 0x00000101, 0x00000000,
 705         0x82a, 0xffffffff, 0x00000104,
 706         0x1579, 0xff000fff, 0x00000100,
 707         0xc33, 0xc0000fff, 0x00000104,
 708         0x3079, 0x00000001, 0x00000001,
 709         0x3403, 0xff000ff0, 0x00000100,
 710         0x3603, 0xff000ff0, 0x00000100
 711 };
 712
 713 static const u32 godavari_golden_registers[] =
 714 {
 715         0x1579, 0xff607fff, 0xfc000100,
 716         0x1bb6, 0x00010101, 0x00010000,
 717         0x260c, 0xffffffff, 0x00000000,
 718         0x260c0, 0xf00fffff, 0x00000400,
 719         0x184c, 0xffffffff, 0x00010000,
 720         0x16ec, 0x000000f0, 0x00000070,
 721         0x16f0, 0xf0311fff, 0x80300000,
 722         0x263e, 0x73773777, 0x12010001,
 723         0x263f, 0xffffffff, 0x00000010,
 724         0x200c, 0x00001f0f, 0x0000100a,
 725         0xbd2, 0x73773777, 0x12010001,
 726         0x902, 0x000fffff, 0x000c007f,
 727         0x2285, 0xf000003f, 0x00000007,
 728         0x22c9, 0xffffffff, 0x00ff0fff,
 729         0xc281, 0x0000ff0f, 0x00000000,
 730         0xa293, 0x07ffffff, 0x06000000,
 731         0x136, 0x00000fff, 0x00000100,
 732         0x3405, 0x00010000, 0x00810001,
 733         0x3605, 0x00010000, 0x00810001,
 734         0xf9e, 0x00000001, 0x00000002,
 735         0x31da, 0x00000008, 0x00000008,
 736         0x31dc, 0x00000f00, 0x00000800,
 737         0x31dd, 0x00000f00, 0x00000800,
 738         0x31e6, 0x00ffffff, 0x00ff7fbf,
 739         0x31e7, 0x00ffffff, 0x00ff7faf,
 740         0x2300, 0x000000ff, 0x00000001,
 741         0x853e, 0x01ff01ff, 0x00000002,
 742         0x8526, 0x007ff800, 0x00200000,
 743         0x8057, 0xffffffff, 0x00000f40,
 744         0x2231, 0x001f3ae3, 0x00000082,
 745         0x2235, 0x0000001f, 0x00000010,
 746         0xc24d, 0xffffffff, 0x00000000
 747 };
 748
 749 static void cik_init_golden_registers(struct amdgpu_device *adev)
 750 {
 751         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 752         mutex_lock(&adev->grbm_idx_mutex);
 753
 754         switch (adev->asic_type) {
 755         case CHIP_BONAIRE:
 756                 amdgpu_program_register_sequence(adev,
 757                                                  bonaire_mgcg_cgcg_init,
 758                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 759                 amdgpu_program_register_sequence(adev,
 760                                                  bonaire_golden_registers,
 761                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 762                 amdgpu_program_register_sequence(adev,
 763                                                  bonaire_golden_common_registers,
 764                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 765                 amdgpu_program_register_sequence(adev,
 766                                                  bonaire_golden_spm_registers,
 767                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 768                 break;
 769         case CHIP_KABINI:
 770                 amdgpu_program_register_sequence(adev,
 771                                                  kalindi_mgcg_cgcg_init,
 772                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 773                 amdgpu_program_register_sequence(adev,
 774                                                  kalindi_golden_registers,
 775                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 776                 amdgpu_program_register_sequence(adev,
 777                                                  kalindi_golden_common_registers,
 778                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 779                 amdgpu_program_register_sequence(adev,
 780                                                  kalindi_golden_spm_registers,
 781                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 782                 break;
 783         case CHIP_MULLINS:
 784                 amdgpu_program_register_sequence(adev,
 785                                                  kalindi_mgcg_cgcg_init,
 786                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 787                 amdgpu_program_register_sequence(adev,
 788                                                  godavari_golden_registers,
 789                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
 790                 amdgpu_program_register_sequence(adev,
 791                                                  kalindi_golden_common_registers,
 792                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 793                 amdgpu_program_register_sequence(adev,
 794                                                  kalindi_golden_spm_registers,
 795                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 796                 break;
 797         case CHIP_KAVERI:
 798                 amdgpu_program_register_sequence(adev,
 799                                                  spectre_mgcg_cgcg_init,
 800                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 801                 amdgpu_program_register_sequence(adev,
 802                                                  spectre_golden_registers,
 803                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 804                 amdgpu_program_register_sequence(adev,
 805                                                  spectre_golden_common_registers,
 806                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 807                 amdgpu_program_register_sequence(adev,
 808                                                  spectre_golden_spm_registers,
 809                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 810                 break;
 811         case CHIP_HAWAII:
 812                 amdgpu_program_register_sequence(adev,
 813                                                  hawaii_mgcg_cgcg_init,
 814                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 815                 amdgpu_program_register_sequence(adev,
 816                                                  hawaii_golden_registers,
 817                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
 818                 amdgpu_program_register_sequence(adev,
 819                                                  hawaii_golden_common_registers,
 820                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 821                 amdgpu_program_register_sequence(adev,
 822                                                  hawaii_golden_spm_registers,
 823                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 824                 break;
 825         default:
 826                 break;
 827         }
 828         mutex_unlock(&adev->grbm_idx_mutex);
 829 }
 830
 831 /**
 832  * cik_get_xclk - get the xclk
 833  *
 834  * @adev: amdgpu_device pointer
 835  *
 836  * Returns the reference clock used by the gfx engine
 837  * (CIK).
 838  */
 839 static u32 cik_get_xclk(struct amdgpu_device *adev)
 840 {
 841         u32 reference_clock = adev->clock.spll.reference_freq;
 842
 843         if (adev->flags & AMD_IS_APU) {
 844                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 845                         return reference_clock / 2;
 846         } else {
 847                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 848                         return reference_clock / 4;
 849         }
 850         return reference_clock;
 851 }
 852
 853 /**
 854  * cik_srbm_select - select specific register instances
 855  *
 856  * @adev: amdgpu_device pointer
 857  * @me: selected ME (micro engine)
 858  * @pipe: pipe
 859  * @queue: queue
 860  * @vmid: VMID
 861  *
 862  * Switches the currently active registers instances.  Some
 863  * registers are instanced per VMID, others are instanced per
 864  * me/pipe/queue combination.
 865  */
 866 void cik_srbm_select(struct amdgpu_device *adev,
 867                      u32 me, u32 pipe, u32 queue, u32 vmid)
 868 {
 869         u32 srbm_gfx_cntl =
 870                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 871                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 872                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 873                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 874         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 875 }
 876
 877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 878 {
 879         uint32_t tmp;
 880
 881         tmp = RREG32(mmCONFIG_CNTL);
 882         if (!state)
 883                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 884         else
 885                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 886         WREG32(mmCONFIG_CNTL, tmp);
 887 }
 888
 889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 890 {
 891         u32 bus_cntl;
 892         u32 d1vga_control = 0;
 893         u32 d2vga_control = 0;
 894         u32 vga_render_control = 0;
 895         u32 rom_cntl;
 896         bool r;
 897
 898         bus_cntl = RREG32(mmBUS_CNTL);
 899         if (adev->mode_info.num_crtc) {
 900                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 901                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 902                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 903         }
 904         rom_cntl = RREG32_SMC(ixROM_CNTL);
 905
 906         /* enable the rom */
 907         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 908         if (adev->mode_info.num_crtc) {
 909                 /* Disable VGA mode */
 910                 WREG32(mmD1VGA_CONTROL,
 911                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 912                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 913                 WREG32(mmD2VGA_CONTROL,
 914                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 915                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 916                 WREG32(mmVGA_RENDER_CONTROL,
 917                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 918         }
 919         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 920
 921         r = amdgpu_read_bios(adev);
 922
 923         /* restore regs */
 924         WREG32(mmBUS_CNTL, bus_cntl);
 925         if (adev->mode_info.num_crtc) {
 926                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 927                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 928                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 929         }
 930         WREG32_SMC(ixROM_CNTL, rom_cntl);
 931         return r;
 932 }
 933
 934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 935                                    u8 *bios, u32 length_bytes)
 936 {
 937         u32 *dw_ptr;
 938         unsigned long flags;
 939         u32 i, length_dw;
 940
 941         if (bios == NULL)
 942                 return false;
 943         if (length_bytes == 0)
 944                 return false;
 945         /* APU vbios image is part of sbios image */
 946         if (adev->flags & AMD_IS_APU)
 947                 return false;
 948
 949         dw_ptr = (u32 *)bios;
 950         length_dw = ALIGN(length_bytes, 4) / 4;
 951         /* take the smc lock since we are using the smc index */
 952         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 953         /* set rom index to 0 */
 954         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 955         WREG32(mmSMC_IND_DATA_0, 0);
 956         /* set index to data for continous read */
 957         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 958         for (i = 0; i < length_dw; i++)
 959                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 960         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 961
 962         return true;
 963 }
 964
 965 static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
 966 {
 967         /* CIK does not support SR-IOV */
 968         return 0;
 969 }
 970
 971 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 972         {mmGRBM_STATUS, false},
 973         {mmGB_ADDR_CONFIG, false},
 974         {mmMC_ARB_RAMCFG, false},
 975         {mmGB_TILE_MODE0, false},
 976         {mmGB_TILE_MODE1, false},
 977         {mmGB_TILE_MODE2, false},
 978         {mmGB_TILE_MODE3, false},
 979         {mmGB_TILE_MODE4, false},
 980         {mmGB_TILE_MODE5, false},
 981         {mmGB_TILE_MODE6, false},
 982         {mmGB_TILE_MODE7, false},
 983         {mmGB_TILE_MODE8, false},
 984         {mmGB_TILE_MODE9, false},
 985         {mmGB_TILE_MODE10, false},
 986         {mmGB_TILE_MODE11, false},
 987         {mmGB_TILE_MODE12, false},
 988         {mmGB_TILE_MODE13, false},
 989         {mmGB_TILE_MODE14, false},
 990         {mmGB_TILE_MODE15, false},
 991         {mmGB_TILE_MODE16, false},
 992         {mmGB_TILE_MODE17, false},
 993         {mmGB_TILE_MODE18, false},
 994         {mmGB_TILE_MODE19, false},
 995         {mmGB_TILE_MODE20, false},
 996         {mmGB_TILE_MODE21, false},
 997         {mmGB_TILE_MODE22, false},
 998         {mmGB_TILE_MODE23, false},
 999         {mmGB_TILE_MODE24, false},
1000         {mmGB_TILE_MODE25, false},
1001         {mmGB_TILE_MODE26, false},
1002         {mmGB_TILE_MODE27, false},
1003         {mmGB_TILE_MODE28, false},
1004         {mmGB_TILE_MODE29, false},
1005         {mmGB_TILE_MODE30, false},
1006         {mmGB_TILE_MODE31, false},
1007         {mmGB_MACROTILE_MODE0, false},
1008         {mmGB_MACROTILE_MODE1, false},
1009         {mmGB_MACROTILE_MODE2, false},
1010         {mmGB_MACROTILE_MODE3, false},
1011         {mmGB_MACROTILE_MODE4, false},
1012         {mmGB_MACROTILE_MODE5, false},
1013         {mmGB_MACROTILE_MODE6, false},
1014         {mmGB_MACROTILE_MODE7, false},
1015         {mmGB_MACROTILE_MODE8, false},
1016         {mmGB_MACROTILE_MODE9, false},
1017         {mmGB_MACROTILE_MODE10, false},
1018         {mmGB_MACROTILE_MODE11, false},
1019         {mmGB_MACROTILE_MODE12, false},
1020         {mmGB_MACROTILE_MODE13, false},
1021         {mmGB_MACROTILE_MODE14, false},
1022         {mmGB_MACROTILE_MODE15, false},
1023         {mmCC_RB_BACKEND_DISABLE, false, true},
1024         {mmGC_USER_RB_BACKEND_DISABLE, false, true},
1025         {mmGB_BACKEND_MAP, false, false},
1026         {mmPA_SC_RASTER_CONFIG, false, true},
1027         {mmPA_SC_RASTER_CONFIG_1, false, true},
1028 };
1029
1030 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1031                                           u32 se_num, u32 sh_num,
1032                                           u32 reg_offset)
1033 {
1034         uint32_t val;
1035
1036         mutex_lock(&adev->grbm_idx_mutex);
1037         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1038                 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1039
1040         val = RREG32(reg_offset);
1041
1042         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1043                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1044         mutex_unlock(&adev->grbm_idx_mutex);
1045         return val;
1046 }
1047
1048 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1049                              u32 sh_num, u32 reg_offset, u32 *value)
1050 {
1051         uint32_t i;
1052
1053         *value = 0;
1054         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1055                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1056                         continue;
1057
1058                 if (!cik_allowed_read_registers[i].untouched)
1059                         *value = cik_allowed_read_registers[i].grbm_indexed ?
1060                                  cik_read_indexed_register(adev, se_num,
1061                                                            sh_num, reg_offset) :
1062                                  RREG32(reg_offset);
1063                 return 0;
1064         }
1065         return -EINVAL;
1066 }
1067
1068 struct kv_reset_save_regs {
1069         u32 gmcon_reng_execute;
1070         u32 gmcon_misc;
1071         u32 gmcon_misc3;
1072 };
1073
1074 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1075                                    struct kv_reset_save_regs *save)
1076 {
1077         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1078         save->gmcon_misc = RREG32(mmGMCON_MISC);
1079         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1080
1081         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1082                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1083         WREG32(mmGMCON_MISC, save->gmcon_misc &
1084                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1085                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1086 }
1087
1088 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1089                                       struct kv_reset_save_regs *save)
1090 {
1091         int i;
1092
1093         WREG32(mmGMCON_PGFSM_WRITE, 0);
1094         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1095
1096         for (i = 0; i < 5; i++)
1097                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1098
1099         WREG32(mmGMCON_PGFSM_WRITE, 0);
1100         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1101
1102         for (i = 0; i < 5; i++)
1103                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1104
1105         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1106         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1107
1108         for (i = 0; i < 5; i++)
1109                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1110
1111         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1112         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1113
1114         for (i = 0; i < 5; i++)
1115                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1116
1117         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1118         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1119
1120         for (i = 0; i < 5; i++)
1121                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1122
1123         WREG32(mmGMCON_PGFSM_WRITE, 0);
1124         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1125
1126         for (i = 0; i < 5; i++)
1127                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1128
1129         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1130         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1131
1132         for (i = 0; i < 5; i++)
1133                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1134
1135         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1136         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1137
1138         for (i = 0; i < 5; i++)
1139                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1140
1141         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1142         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1143
1144         for (i = 0; i < 5; i++)
1145                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1146
1147         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1148         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1149
1150         for (i = 0; i < 5; i++)
1151                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1152
1153         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1154         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1155
1156         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1157         WREG32(mmGMCON_MISC, save->gmcon_misc);
1158         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1159 }
1160
1161 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1162 {
1163         struct kv_reset_save_regs kv_save = { 0 };
1164         u32 i;
1165         int r = -EINVAL;
1166
1167         dev_info(adev->dev, "GPU pci config reset\n");
1168
1169         if (adev->flags & AMD_IS_APU)
1170                 kv_save_regs_for_reset(adev, &kv_save);
1171
1172         /* disable BM */
1173         pci_clear_master(adev->pdev);
1174         /* reset */
1175         amdgpu_pci_config_reset(adev);
1176
1177         udelay(100);
1178
1179         /* wait for asic to come out of reset */
1180         for (i = 0; i < adev->usec_timeout; i++) {
1181                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1182                         /* enable BM */
1183                         pci_set_master(adev->pdev);
1184                         r = 0;
1185                         break;
1186                 }
1187                 udelay(1);
1188         }
1189
1190         /* does asic init need to be run first??? */
1191         if (adev->flags & AMD_IS_APU)
1192                 kv_restore_regs_for_reset(adev, &kv_save);
1193
1194         return r;
1195 }
1196
1197 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1198 {
1199         u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1200
1201         if (hung)
1202                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1203         else
1204                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1205
1206         WREG32(mmBIOS_SCRATCH_3, tmp);
1207 }
1208
1209 /**
1210  * cik_asic_reset - soft reset GPU
1211  *
1212  * @adev: amdgpu_device pointer
1213  *
1214  * Look up which blocks are hung and attempt
1215  * to reset them.
1216  * Returns 0 for success.
1217  */
1218 static int cik_asic_reset(struct amdgpu_device *adev)
1219 {
1220         int r;
1221         cik_set_bios_scratch_engine_hung(adev, true);
1222
1223         r = cik_gpu_pci_config_reset(adev);
1224
1225         cik_set_bios_scratch_engine_hung(adev, false);
1226
1227         return r;
1228 }
1229
1230 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1231                               u32 cntl_reg, u32 status_reg)
1232 {
1233         int r, i;
1234         struct atom_clock_dividers dividers;
1235         uint32_t tmp;
1236
1237         r = amdgpu_atombios_get_clock_dividers(adev,
1238                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1239                                                clock, false, &dividers);
1240         if (r)
1241                 return r;
1242
1243         tmp = RREG32_SMC(cntl_reg);
1244         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1245                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1246         tmp |= dividers.post_divider;
1247         WREG32_SMC(cntl_reg, tmp);
1248
1249         for (i = 0; i < 100; i++) {
1250                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1251                         break;
1252                 mdelay(10);
1253         }
1254         if (i == 100)
1255                 return -ETIMEDOUT;
1256
1257         return 0;
1258 }
1259
1260 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1261 {
1262         int r = 0;
1263
1264         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1265         if (r)
1266                 return r;
1267
1268         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1269         return r;
1270 }
1271
1272 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1273 {
1274         int r, i;
1275         struct atom_clock_dividers dividers;
1276         u32 tmp;
1277
1278         r = amdgpu_atombios_get_clock_dividers(adev,
1279                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1280                                                ecclk, false, &dividers);
1281         if (r)
1282                 return r;
1283
1284         for (i = 0; i < 100; i++) {
1285                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1286                         break;
1287                 mdelay(10);
1288         }
1289         if (i == 100)
1290                 return -ETIMEDOUT;
1291
1292         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1293         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1294                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1295         tmp |= dividers.post_divider;
1296         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1297
1298         for (i = 0; i < 100; i++) {
1299                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1300                         break;
1301                 mdelay(10);
1302         }
1303         if (i == 100)
1304                 return -ETIMEDOUT;
1305
1306         return 0;
1307 }
1308
1309 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1310 {
1311         struct pci_dev *root = adev->pdev->bus->self;
1312         int bridge_pos, gpu_pos;
1313         u32 speed_cntl, current_data_rate;
1314         int i;
1315         u16 tmp16;
1316
1317         if (pci_is_root_bus(adev->pdev->bus))
1318                 return;
1319
1320         if (amdgpu_pcie_gen2 == 0)
1321                 return;
1322
1323         if (adev->flags & AMD_IS_APU)
1324                 return;
1325
1326         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1327                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1328                 return;
1329
1330         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1331         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1332                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1333         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1334                 if (current_data_rate == 2) {
1335                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1336                         return;
1337                 }
1338                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1339         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1340                 if (current_data_rate == 1) {
1341                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1342                         return;
1343                 }
1344                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1345         }
1346
1347         bridge_pos = pci_pcie_cap(root);
1348         if (!bridge_pos)
1349                 return;
1350
1351         gpu_pos = pci_pcie_cap(adev->pdev);
1352         if (!gpu_pos)
1353                 return;
1354
1355         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1356                 /* re-try equalization if gen3 is not already enabled */
1357                 if (current_data_rate != 2) {
1358                         u16 bridge_cfg, gpu_cfg;
1359                         u16 bridge_cfg2, gpu_cfg2;
1360                         u32 max_lw, current_lw, tmp;
1361
1362                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1363                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1364
1365                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1366                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1367
1368                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1369                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1370
1371                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1372                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1373                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1374                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1375                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1376
1377                         if (current_lw < max_lw) {
1378                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1379                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1380                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1381                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1382                                         tmp |= (max_lw <<
1383                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1384                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1385                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1386                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1387                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1388                                 }
1389                         }
1390
1391                         for (i = 0; i < 10; i++) {
1392                                 /* check status */
1393                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1394                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1395                                         break;
1396
1397                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1398                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1399
1400                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1401                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1402
1403                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1404                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1405                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1406
1407                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1408                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1409                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1410
1411                                 mdelay(100);
1412
1413                                 /* linkctl */
1414                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1415                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1416                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1417                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1418
1419                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1420                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1421                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1422                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1423
1424                                 /* linkctl2 */
1425                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1426                                 tmp16 &= ~((1 << 4) | (7 << 9));
1427                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1428                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1429
1430                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1431                                 tmp16 &= ~((1 << 4) | (7 << 9));
1432                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1433                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1434
1435                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1436                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1437                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1438                         }
1439                 }
1440         }
1441
1442         /* set the link speed */
1443         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1444                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1445         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1446         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1447
1448         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1449         tmp16 &= ~0xf;
1450         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1451                 tmp16 |= 3; /* gen3 */
1452         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1453                 tmp16 |= 2; /* gen2 */
1454         else
1455                 tmp16 |= 1; /* gen1 */
1456         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1457
1458         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1459         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1460         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1461
1462         for (i = 0; i < adev->usec_timeout; i++) {
1463                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1464                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1465                         break;
1466                 udelay(1);
1467         }
1468 }
1469
1470 static void cik_program_aspm(struct amdgpu_device *adev)
1471 {
1472         u32 data, orig;
1473         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1474         bool disable_clkreq = false;
1475
1476         if (amdgpu_aspm == 0)
1477                 return;
1478
1479         if (pci_is_root_bus(adev->pdev->bus))
1480                 return;
1481
1482         /* XXX double check APUs */
1483         if (adev->flags & AMD_IS_APU)
1484                 return;
1485
1486         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1487         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1488         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1489                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1490         if (orig != data)
1491                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1492
1493         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1494         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1495         if (orig != data)
1496                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1497
1498         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1499         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1500         if (orig != data)
1501                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1502
1503         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1504         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1505                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1506         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1507         if (!disable_l0s)
1508                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1509
1510         if (!disable_l1) {
1511                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1512                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1513                 if (orig != data)
1514                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1515
1516                 if (!disable_plloff_in_l1) {
1517                         bool clk_req_support;
1518
1519                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1520                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1521                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1522                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1523                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1524                         if (orig != data)
1525                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1526
1527                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1528                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1529                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1530                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1531                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1532                         if (orig != data)
1533                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1534
1535                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1536                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1537                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1538                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1539                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1540                         if (orig != data)
1541                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1542
1543                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1544                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1545                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1546                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1547                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1548                         if (orig != data)
1549                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1550
1551                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1552                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1553                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1554                         if (orig != data)
1555                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1556
1557                         if (!disable_clkreq) {
1558                                 struct pci_dev *root = adev->pdev->bus->self;
1559                                 u32 lnkcap;
1560
1561                                 clk_req_support = false;
1562                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1563                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1564                                         clk_req_support = true;
1565                         } else {
1566                                 clk_req_support = false;
1567                         }
1568
1569                         if (clk_req_support) {
1570                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1571                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1572                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1573                                 if (orig != data)
1574                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1575
1576                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1577                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1578                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1579                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1580                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1581                                 if (orig != data)
1582                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1583
1584                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1585                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1586                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1587                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1588                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1589                                 if (orig != data)
1590                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1591
1592                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1593                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1594                                 if (orig != data)
1595                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1596
1597                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1598                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1599                                 if (orig != data)
1600                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1601
1602                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1603                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1604                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1605                                 if (orig != data)
1606                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1607                         }
1608                 }
1609         } else {
1610                 if (orig != data)
1611                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1612         }
1613
1614         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1615         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1616                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1617                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1618         if (orig != data)
1619                 WREG32_PCIE(ixPCIE_CNTL2, data);
1620
1621         if (!disable_l0s) {
1622                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1623                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1624                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1625                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1626                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1627                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1628                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1629                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1630                                 if (orig != data)
1631                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1632                         }
1633                 }
1634         }
1635 }
1636
1637 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1638 {
1639         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1640                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1641 }
1642
1643 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1644 {
1645         /* ORDER MATTERS! */
1646         {
1647                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1648                 .major = 1,
1649                 .minor = 0,
1650                 .rev = 0,
1651                 .funcs = &cik_common_ip_funcs,
1652         },
1653         {
1654                 .type = AMD_IP_BLOCK_TYPE_GMC,
1655                 .major = 7,
1656                 .minor = 0,
1657                 .rev = 0,
1658                 .funcs = &gmc_v7_0_ip_funcs,
1659         },
1660         {
1661                 .type = AMD_IP_BLOCK_TYPE_IH,
1662                 .major = 2,
1663                 .minor = 0,
1664                 .rev = 0,
1665                 .funcs = &cik_ih_ip_funcs,
1666         },
1667         {
1668                 .type = AMD_IP_BLOCK_TYPE_SMC,
1669                 .major = 7,
1670                 .minor = 0,
1671                 .rev = 0,
1672                 .funcs = &amdgpu_pp_ip_funcs,
1673         },
1674         {
1675                 .type = AMD_IP_BLOCK_TYPE_DCE,
1676                 .major = 8,
1677                 .minor = 2,
1678                 .rev = 0,
1679                 .funcs = &dce_v8_0_ip_funcs,
1680         },
1681         {
1682                 .type = AMD_IP_BLOCK_TYPE_GFX,
1683                 .major = 7,
1684                 .minor = 2,
1685                 .rev = 0,
1686                 .funcs = &gfx_v7_0_ip_funcs,
1687         },
1688         {
1689                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1690                 .major = 2,
1691                 .minor = 0,
1692                 .rev = 0,
1693                 .funcs = &cik_sdma_ip_funcs,
1694         },
1695         {
1696                 .type = AMD_IP_BLOCK_TYPE_UVD,
1697                 .major = 4,
1698                 .minor = 2,
1699                 .rev = 0,
1700                 .funcs = &uvd_v4_2_ip_funcs,
1701         },
1702         {
1703                 .type = AMD_IP_BLOCK_TYPE_VCE,
1704                 .major = 2,
1705                 .minor = 0,
1706                 .rev = 0,
1707                 .funcs = &vce_v2_0_ip_funcs,
1708         },
1709 };
1710
1711 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1712 {
1713         /* ORDER MATTERS! */
1714         {
1715                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1716                 .major = 1,
1717                 .minor = 0,
1718                 .rev = 0,
1719                 .funcs = &cik_common_ip_funcs,
1720         },
1721         {
1722                 .type = AMD_IP_BLOCK_TYPE_GMC,
1723                 .major = 7,
1724                 .minor = 0,
1725                 .rev = 0,
1726                 .funcs = &gmc_v7_0_ip_funcs,
1727         },
1728         {
1729                 .type = AMD_IP_BLOCK_TYPE_IH,
1730                 .major = 2,
1731                 .minor = 0,
1732                 .rev = 0,
1733                 .funcs = &cik_ih_ip_funcs,
1734         },
1735         {
1736                 .type = AMD_IP_BLOCK_TYPE_SMC,
1737                 .major = 7,
1738                 .minor = 0,
1739                 .rev = 0,
1740                 .funcs = &amdgpu_pp_ip_funcs,
1741         },
1742         {
1743                 .type = AMD_IP_BLOCK_TYPE_DCE,
1744                 .major = 8,
1745                 .minor = 5,
1746                 .rev = 0,
1747                 .funcs = &dce_v8_0_ip_funcs,
1748         },
1749         {
1750                 .type = AMD_IP_BLOCK_TYPE_GFX,
1751                 .major = 7,
1752                 .minor = 3,
1753                 .rev = 0,
1754                 .funcs = &gfx_v7_0_ip_funcs,
1755         },
1756         {
1757                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1758                 .major = 2,
1759                 .minor = 0,
1760                 .rev = 0,
1761                 .funcs = &cik_sdma_ip_funcs,
1762         },
1763         {
1764                 .type = AMD_IP_BLOCK_TYPE_UVD,
1765                 .major = 4,
1766                 .minor = 2,
1767                 .rev = 0,
1768                 .funcs = &uvd_v4_2_ip_funcs,
1769         },
1770         {
1771                 .type = AMD_IP_BLOCK_TYPE_VCE,
1772                 .major = 2,
1773                 .minor = 0,
1774                 .rev = 0,
1775                 .funcs = &vce_v2_0_ip_funcs,
1776         },
1777 };
1778
1779 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1780 {
1781         /* ORDER MATTERS! */
1782         {
1783                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1784                 .major = 1,
1785                 .minor = 0,
1786                 .rev = 0,
1787                 .funcs = &cik_common_ip_funcs,
1788         },
1789         {
1790                 .type = AMD_IP_BLOCK_TYPE_GMC,
1791                 .major = 7,
1792                 .minor = 0,
1793                 .rev = 0,
1794                 .funcs = &gmc_v7_0_ip_funcs,
1795         },
1796         {
1797                 .type = AMD_IP_BLOCK_TYPE_IH,
1798                 .major = 2,
1799                 .minor = 0,
1800                 .rev = 0,
1801                 .funcs = &cik_ih_ip_funcs,
1802         },
1803         {
1804                 .type = AMD_IP_BLOCK_TYPE_SMC,
1805                 .major = 7,
1806                 .minor = 0,
1807                 .rev = 0,
1808                 .funcs = &amdgpu_pp_ip_funcs,
1809         },
1810         {
1811                 .type = AMD_IP_BLOCK_TYPE_DCE,
1812                 .major = 8,
1813                 .minor = 3,
1814                 .rev = 0,
1815                 .funcs = &dce_v8_0_ip_funcs,
1816         },
1817         {
1818                 .type = AMD_IP_BLOCK_TYPE_GFX,
1819                 .major = 7,
1820                 .minor = 2,
1821                 .rev = 0,
1822                 .funcs = &gfx_v7_0_ip_funcs,
1823         },
1824         {
1825                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1826                 .major = 2,
1827                 .minor = 0,
1828                 .rev = 0,
1829                 .funcs = &cik_sdma_ip_funcs,
1830         },
1831         {
1832                 .type = AMD_IP_BLOCK_TYPE_UVD,
1833                 .major = 4,
1834                 .minor = 2,
1835                 .rev = 0,
1836                 .funcs = &uvd_v4_2_ip_funcs,
1837         },
1838         {
1839                 .type = AMD_IP_BLOCK_TYPE_VCE,
1840                 .major = 2,
1841                 .minor = 0,
1842                 .rev = 0,
1843                 .funcs = &vce_v2_0_ip_funcs,
1844         },
1845 };
1846
1847 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1848 {
1849         /* ORDER MATTERS! */
1850         {
1851                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1852                 .major = 1,
1853                 .minor = 0,
1854                 .rev = 0,
1855                 .funcs = &cik_common_ip_funcs,
1856         },
1857         {
1858                 .type = AMD_IP_BLOCK_TYPE_GMC,
1859                 .major = 7,
1860                 .minor = 0,
1861                 .rev = 0,
1862                 .funcs = &gmc_v7_0_ip_funcs,
1863         },
1864         {
1865                 .type = AMD_IP_BLOCK_TYPE_IH,
1866                 .major = 2,
1867                 .minor = 0,
1868                 .rev = 0,
1869                 .funcs = &cik_ih_ip_funcs,
1870         },
1871         {
1872                 .type = AMD_IP_BLOCK_TYPE_SMC,
1873                 .major = 7,
1874                 .minor = 0,
1875                 .rev = 0,
1876                 .funcs = &amdgpu_pp_ip_funcs,
1877         },
1878         {
1879                 .type = AMD_IP_BLOCK_TYPE_DCE,
1880                 .major = 8,
1881                 .minor = 3,
1882                 .rev = 0,
1883                 .funcs = &dce_v8_0_ip_funcs,
1884         },
1885         {
1886                 .type = AMD_IP_BLOCK_TYPE_GFX,
1887                 .major = 7,
1888                 .minor = 2,
1889                 .rev = 0,
1890                 .funcs = &gfx_v7_0_ip_funcs,
1891         },
1892         {
1893                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1894                 .major = 2,
1895                 .minor = 0,
1896                 .rev = 0,
1897                 .funcs = &cik_sdma_ip_funcs,
1898         },
1899         {
1900                 .type = AMD_IP_BLOCK_TYPE_UVD,
1901                 .major = 4,
1902                 .minor = 2,
1903                 .rev = 0,
1904                 .funcs = &uvd_v4_2_ip_funcs,
1905         },
1906         {
1907                 .type = AMD_IP_BLOCK_TYPE_VCE,
1908                 .major = 2,
1909                 .minor = 0,
1910                 .rev = 0,
1911                 .funcs = &vce_v2_0_ip_funcs,
1912         },
1913 };
1914
1915 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1916 {
1917         /* ORDER MATTERS! */
1918         {
1919                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1920                 .major = 1,
1921                 .minor = 0,
1922                 .rev = 0,
1923                 .funcs = &cik_common_ip_funcs,
1924         },
1925         {
1926                 .type = AMD_IP_BLOCK_TYPE_GMC,
1927                 .major = 7,
1928                 .minor = 0,
1929                 .rev = 0,
1930                 .funcs = &gmc_v7_0_ip_funcs,
1931         },
1932         {
1933                 .type = AMD_IP_BLOCK_TYPE_IH,
1934                 .major = 2,
1935                 .minor = 0,
1936                 .rev = 0,
1937                 .funcs = &cik_ih_ip_funcs,
1938         },
1939         {
1940                 .type = AMD_IP_BLOCK_TYPE_SMC,
1941                 .major = 7,
1942                 .minor = 0,
1943                 .rev = 0,
1944                 .funcs = &amdgpu_pp_ip_funcs,
1945         },
1946         {
1947                 .type = AMD_IP_BLOCK_TYPE_DCE,
1948                 .major = 8,
1949                 .minor = 1,
1950                 .rev = 0,
1951                 .funcs = &dce_v8_0_ip_funcs,
1952         },
1953         {
1954                 .type = AMD_IP_BLOCK_TYPE_GFX,
1955                 .major = 7,
1956                 .minor = 1,
1957                 .rev = 0,
1958                 .funcs = &gfx_v7_0_ip_funcs,
1959         },
1960         {
1961                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1962                 .major = 2,
1963                 .minor = 0,
1964                 .rev = 0,
1965                 .funcs = &cik_sdma_ip_funcs,
1966         },
1967         {
1968                 .type = AMD_IP_BLOCK_TYPE_UVD,
1969                 .major = 4,
1970                 .minor = 2,
1971                 .rev = 0,
1972                 .funcs = &uvd_v4_2_ip_funcs,
1973         },
1974         {
1975                 .type = AMD_IP_BLOCK_TYPE_VCE,
1976                 .major = 2,
1977                 .minor = 0,
1978                 .rev = 0,
1979                 .funcs = &vce_v2_0_ip_funcs,
1980         },
1981 };
1982
1983 int cik_set_ip_blocks(struct amdgpu_device *adev)
1984 {
1985         switch (adev->asic_type) {
1986         case CHIP_BONAIRE:
1987                 adev->ip_blocks = bonaire_ip_blocks;
1988                 adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1989                 break;
1990         case CHIP_HAWAII:
1991                 adev->ip_blocks = hawaii_ip_blocks;
1992                 adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1993                 break;
1994         case CHIP_KAVERI:
1995                 adev->ip_blocks = kaveri_ip_blocks;
1996                 adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1997                 break;
1998         case CHIP_KABINI:
1999                 adev->ip_blocks = kabini_ip_blocks;
2000                 adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
2001                 break;
2002         case CHIP_MULLINS:
2003                 adev->ip_blocks = mullins_ip_blocks;
2004                 adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
2005                 break;
2006         default:
2007                 /* FIXME: not supported yet */
2008                 return -EINVAL;
2009         }
2010
2011         return 0;
2012 }
2013
2014 static const struct amdgpu_asic_funcs cik_asic_funcs =
2015 {
2016         .read_disabled_bios = &cik_read_disabled_bios,
2017         .read_bios_from_rom = &cik_read_bios_from_rom,
2018         .read_register = &cik_read_register,
2019         .reset = &cik_asic_reset,
2020         .set_vga_state = &cik_vga_set_state,
2021         .get_xclk = &cik_get_xclk,
2022         .set_uvd_clocks = &cik_set_uvd_clocks,
2023         .set_vce_clocks = &cik_set_vce_clocks,
2024         .get_virtual_caps = &cik_get_virtual_caps,
2025 };
2026
2027 static int cik_common_early_init(void *handle)
2028 {
2029         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2030
2031         adev->smc_rreg = &cik_smc_rreg;
2032         adev->smc_wreg = &cik_smc_wreg;
2033         adev->pcie_rreg = &cik_pcie_rreg;
2034         adev->pcie_wreg = &cik_pcie_wreg;
2035         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2036         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2037         adev->didt_rreg = &cik_didt_rreg;
2038         adev->didt_wreg = &cik_didt_wreg;
2039
2040         adev->asic_funcs = &cik_asic_funcs;
2041
2042         adev->rev_id = cik_get_rev_id(adev);
2043         adev->external_rev_id = 0xFF;
2044         switch (adev->asic_type) {
2045         case CHIP_BONAIRE:
2046                 adev->cg_flags =
2047                         AMD_CG_SUPPORT_GFX_MGCG |
2048                         AMD_CG_SUPPORT_GFX_MGLS |
2049                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2050                         AMD_CG_SUPPORT_GFX_CGLS |
2051                         AMD_CG_SUPPORT_GFX_CGTS |
2052                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2053                         AMD_CG_SUPPORT_GFX_CP_LS |
2054                         AMD_CG_SUPPORT_MC_LS |
2055                         AMD_CG_SUPPORT_MC_MGCG |
2056                         AMD_CG_SUPPORT_SDMA_MGCG |
2057                         AMD_CG_SUPPORT_SDMA_LS |
2058                         AMD_CG_SUPPORT_BIF_LS |
2059                         AMD_CG_SUPPORT_VCE_MGCG |
2060                         AMD_CG_SUPPORT_UVD_MGCG |
2061                         AMD_CG_SUPPORT_HDP_LS |
2062                         AMD_CG_SUPPORT_HDP_MGCG;
2063                 adev->pg_flags = 0;
2064                 adev->external_rev_id = adev->rev_id + 0x14;
2065                 break;
2066         case CHIP_HAWAII:
2067                 adev->cg_flags =
2068                         AMD_CG_SUPPORT_GFX_MGCG |
2069                         AMD_CG_SUPPORT_GFX_MGLS |
2070                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2071                         AMD_CG_SUPPORT_GFX_CGLS |
2072                         AMD_CG_SUPPORT_GFX_CGTS |
2073                         AMD_CG_SUPPORT_GFX_CP_LS |
2074                         AMD_CG_SUPPORT_MC_LS |
2075                         AMD_CG_SUPPORT_MC_MGCG |
2076                         AMD_CG_SUPPORT_SDMA_MGCG |
2077                         AMD_CG_SUPPORT_SDMA_LS |
2078                         AMD_CG_SUPPORT_BIF_LS |
2079                         AMD_CG_SUPPORT_VCE_MGCG |
2080                         AMD_CG_SUPPORT_UVD_MGCG |
2081                         AMD_CG_SUPPORT_HDP_LS |
2082                         AMD_CG_SUPPORT_HDP_MGCG;
2083                 adev->pg_flags = 0;
2084                 adev->external_rev_id = 0x28;
2085                 break;
2086         case CHIP_KAVERI:
2087                 adev->cg_flags =
2088                         AMD_CG_SUPPORT_GFX_MGCG |
2089                         AMD_CG_SUPPORT_GFX_MGLS |
2090                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2091                         AMD_CG_SUPPORT_GFX_CGLS |
2092                         AMD_CG_SUPPORT_GFX_CGTS |
2093                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2094                         AMD_CG_SUPPORT_GFX_CP_LS |
2095                         AMD_CG_SUPPORT_SDMA_MGCG |
2096                         AMD_CG_SUPPORT_SDMA_LS |
2097                         AMD_CG_SUPPORT_BIF_LS |
2098                         AMD_CG_SUPPORT_VCE_MGCG |
2099                         AMD_CG_SUPPORT_UVD_MGCG |
2100                         AMD_CG_SUPPORT_HDP_LS |
2101                         AMD_CG_SUPPORT_HDP_MGCG;
2102                 adev->pg_flags =
2103                         /*AMD_PG_SUPPORT_GFX_PG |
2104                           AMD_PG_SUPPORT_GFX_SMG |
2105                           AMD_PG_SUPPORT_GFX_DMG |*/
2106                         AMD_PG_SUPPORT_UVD |
2107                         /*AMD_PG_SUPPORT_VCE |
2108                           AMD_PG_SUPPORT_CP |
2109                           AMD_PG_SUPPORT_GDS |
2110                           AMD_PG_SUPPORT_RLC_SMU_HS |
2111                           AMD_PG_SUPPORT_ACP |
2112                           AMD_PG_SUPPORT_SAMU |*/
2113                         0;
2114                 if (adev->pdev->device == 0x1312 ||
2115                         adev->pdev->device == 0x1316 ||
2116                         adev->pdev->device == 0x1317)
2117                         adev->external_rev_id = 0x41;
2118                 else
2119                         adev->external_rev_id = 0x1;
2120                 break;
2121         case CHIP_KABINI:
2122         case CHIP_MULLINS:
2123                 adev->cg_flags =
2124                         AMD_CG_SUPPORT_GFX_MGCG |
2125                         AMD_CG_SUPPORT_GFX_MGLS |
2126                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2127                         AMD_CG_SUPPORT_GFX_CGLS |
2128                         AMD_CG_SUPPORT_GFX_CGTS |
2129                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2130                         AMD_CG_SUPPORT_GFX_CP_LS |
2131                         AMD_CG_SUPPORT_SDMA_MGCG |
2132                         AMD_CG_SUPPORT_SDMA_LS |
2133                         AMD_CG_SUPPORT_BIF_LS |
2134                         AMD_CG_SUPPORT_VCE_MGCG |
2135                         AMD_CG_SUPPORT_UVD_MGCG |
2136                         AMD_CG_SUPPORT_HDP_LS |
2137                         AMD_CG_SUPPORT_HDP_MGCG;
2138                 adev->pg_flags =
2139                         /*AMD_PG_SUPPORT_GFX_PG |
2140                           AMD_PG_SUPPORT_GFX_SMG | */
2141                         AMD_PG_SUPPORT_UVD |
2142                         /*AMD_PG_SUPPORT_VCE |
2143                           AMD_PG_SUPPORT_CP |
2144                           AMD_PG_SUPPORT_GDS |
2145                           AMD_PG_SUPPORT_RLC_SMU_HS |
2146                           AMD_PG_SUPPORT_SAMU |*/
2147                         0;
2148                 if (adev->asic_type == CHIP_KABINI) {
2149                         if (adev->rev_id == 0)
2150                                 adev->external_rev_id = 0x81;
2151                         else if (adev->rev_id == 1)
2152                                 adev->external_rev_id = 0x82;
2153                         else if (adev->rev_id == 2)
2154                                 adev->external_rev_id = 0x85;
2155                 } else
2156                         adev->external_rev_id = adev->rev_id + 0xa1;
2157                 break;
2158         default:
2159                 /* FIXME: not supported yet */
2160                 return -EINVAL;
2161         }
2162
2163         amdgpu_get_pcie_info(adev);
2164
2165         return 0;
2166 }
2167
2168 static int cik_common_sw_init(void *handle)
2169 {
2170         return 0;
2171 }
2172
2173 static int cik_common_sw_fini(void *handle)
2174 {
2175         return 0;
2176 }
2177
2178 static int cik_common_hw_init(void *handle)
2179 {
2180         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2181
2182         /* move the golden regs per IP block */
2183         cik_init_golden_registers(adev);
2184         /* enable pcie gen2/3 link */
2185         cik_pcie_gen3_enable(adev);
2186         /* enable aspm */
2187         cik_program_aspm(adev);
2188
2189         return 0;
2190 }
2191
2192 static int cik_common_hw_fini(void *handle)
2193 {
2194         return 0;
2195 }
2196
2197 static int cik_common_suspend(void *handle)
2198 {
2199         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2200
2201         amdgpu_amdkfd_suspend(adev);
2202
2203         return cik_common_hw_fini(adev);
2204 }
2205
2206 static int cik_common_resume(void *handle)
2207 {
2208         int r;
2209         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2210
2211         r = cik_common_hw_init(adev);
2212         if (r)
2213                 return r;
2214
2215         return amdgpu_amdkfd_resume(adev);
2216 }
2217
2218 static bool cik_common_is_idle(void *handle)
2219 {
2220         return true;
2221 }
2222
2223 static int cik_common_wait_for_idle(void *handle)
2224 {
2225         return 0;
2226 }
2227
2228 static int cik_common_soft_reset(void *handle)
2229 {
2230         /* XXX hard reset?? */
2231         return 0;
2232 }
2233
2234 static int cik_common_set_clockgating_state(void *handle,
2235                                             enum amd_clockgating_state state)
2236 {
2237         return 0;
2238 }
2239
2240 static int cik_common_set_powergating_state(void *handle,
2241                                             enum amd_powergating_state state)
2242 {
2243         return 0;
2244 }
2245
2246 const struct amd_ip_funcs cik_common_ip_funcs = {
2247         .name = "cik_common",
2248         .early_init = cik_common_early_init,
2249         .late_init = NULL,
2250         .sw_init = cik_common_sw_init,
2251         .sw_fini = cik_common_sw_fini,
2252         .hw_init = cik_common_hw_init,
2253         .hw_fini = cik_common_hw_fini,
2254         .suspend = cik_common_suspend,
2255         .resume = cik_common_resume,
2256         .is_idle = cik_common_is_idle,
2257         .wait_for_idle = cik_common_wait_for_idle,
2258         .soft_reset = cik_common_soft_reset,
2259         .set_clockgating_state = cik_common_set_clockgating_state,
2260         .set_powergating_state = cik_common_set_powergating_state,
2261 };