drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "cikd.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36
  37 #include "cik.h"
  38 #include "gmc_v7_0.h"
  39 #include "cik_ih.h"
  40 #include "dce_v8_0.h"
  41 #include "gfx_v7_0.h"
  42 #include "cik_sdma.h"
  43 #include "uvd_v4_2.h"
  44 #include "vce_v2_0.h"
  45 #include "cik_dpm.h"
  46
  47 #include "uvd/uvd_4_2_d.h"
  48
  49 #include "smu/smu_7_0_1_d.h"
  50 #include "smu/smu_7_0_1_sh_mask.h"
  51
  52 #include "dce/dce_8_0_d.h"
  53 #include "dce/dce_8_0_sh_mask.h"
  54
  55 #include "bif/bif_4_1_d.h"
  56 #include "bif/bif_4_1_sh_mask.h"
  57
  58 #include "gca/gfx_7_2_d.h"
  59 #include "gca/gfx_7_2_enum.h"
  60 #include "gca/gfx_7_2_sh_mask.h"
  61
  62 #include "gmc/gmc_7_1_d.h"
  63 #include "gmc/gmc_7_1_sh_mask.h"
  64
  65 #include "oss/oss_2_0_d.h"
  66 #include "oss/oss_2_0_sh_mask.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_powerplay.h"
  70
  71 /*
  72  * Indirect registers accessor
  73  */
  74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  75 {
  76         unsigned long flags;
  77         u32 r;
  78
  79         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  80         WREG32(mmPCIE_INDEX, reg);
  81         (void)RREG32(mmPCIE_INDEX);
  82         r = RREG32(mmPCIE_DATA);
  83         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  84         return r;
  85 }
  86
  87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  88 {
  89         unsigned long flags;
  90
  91         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  92         WREG32(mmPCIE_INDEX, reg);
  93         (void)RREG32(mmPCIE_INDEX);
  94         WREG32(mmPCIE_DATA, v);
  95         (void)RREG32(mmPCIE_DATA);
  96         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  97 }
  98
  99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 100 {
 101         unsigned long flags;
 102         u32 r;
 103
 104         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 105         WREG32(mmSMC_IND_INDEX_0, (reg));
 106         r = RREG32(mmSMC_IND_DATA_0);
 107         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 108         return r;
 109 }
 110
 111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 112 {
 113         unsigned long flags;
 114
 115         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 116         WREG32(mmSMC_IND_INDEX_0, (reg));
 117         WREG32(mmSMC_IND_DATA_0, (v));
 118         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 119 }
 120
 121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 122 {
 123         unsigned long flags;
 124         u32 r;
 125
 126         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 127         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 128         r = RREG32(mmUVD_CTX_DATA);
 129         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 130         return r;
 131 }
 132
 133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 134 {
 135         unsigned long flags;
 136
 137         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 138         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 139         WREG32(mmUVD_CTX_DATA, (v));
 140         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 141 }
 142
 143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 144 {
 145         unsigned long flags;
 146         u32 r;
 147
 148         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 149         WREG32(mmDIDT_IND_INDEX, (reg));
 150         r = RREG32(mmDIDT_IND_DATA);
 151         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 152         return r;
 153 }
 154
 155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 160         WREG32(mmDIDT_IND_INDEX, (reg));
 161         WREG32(mmDIDT_IND_DATA, (v));
 162         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 163 }
 164
 165 static const u32 bonaire_golden_spm_registers[] =
 166 {
 167         0xc200, 0xe0ffffff, 0xe0000000
 168 };
 169
 170 static const u32 bonaire_golden_common_registers[] =
 171 {
 172         0x31dc, 0xffffffff, 0x00000800,
 173         0x31dd, 0xffffffff, 0x00000800,
 174         0x31e6, 0xffffffff, 0x00007fbf,
 175         0x31e7, 0xffffffff, 0x00007faf
 176 };
 177
 178 static const u32 bonaire_golden_registers[] =
 179 {
 180         0xcd5, 0x00000333, 0x00000333,
 181         0xcd4, 0x000c0fc0, 0x00040200,
 182         0x2684, 0x00010000, 0x00058208,
 183         0xf000, 0xffff1fff, 0x00140000,
 184         0xf080, 0xfdfc0fff, 0x00000100,
 185         0xf08d, 0x40000000, 0x40000200,
 186         0x260c, 0xffffffff, 0x00000000,
 187         0x260d, 0xf00fffff, 0x00000400,
 188         0x260e, 0x0002021c, 0x00020200,
 189         0x31e, 0x00000080, 0x00000000,
 190         0x16ec, 0x000000f0, 0x00000070,
 191         0x16f0, 0xf0311fff, 0x80300000,
 192         0x263e, 0x73773777, 0x12010001,
 193         0xd43, 0x00810000, 0x408af000,
 194         0x1c0c, 0x31000111, 0x00000011,
 195         0xbd2, 0x73773777, 0x12010001,
 196         0x883, 0x00007fb6, 0x0021a1b1,
 197         0x884, 0x00007fb6, 0x002021b1,
 198         0x860, 0x00007fb6, 0x00002191,
 199         0x886, 0x00007fb6, 0x002121b1,
 200         0x887, 0x00007fb6, 0x002021b1,
 201         0x877, 0x00007fb6, 0x00002191,
 202         0x878, 0x00007fb6, 0x00002191,
 203         0xd8a, 0x0000003f, 0x0000000a,
 204         0xd8b, 0x0000003f, 0x0000000a,
 205         0xab9, 0x00073ffe, 0x000022a2,
 206         0x903, 0x000007ff, 0x00000000,
 207         0x2285, 0xf000003f, 0x00000007,
 208         0x22fc, 0x00002001, 0x00000001,
 209         0x22c9, 0xffffffff, 0x00ffffff,
 210         0xc281, 0x0000ff0f, 0x00000000,
 211         0xa293, 0x07ffffff, 0x06000000,
 212         0x136, 0x00000fff, 0x00000100,
 213         0xf9e, 0x00000001, 0x00000002,
 214         0x2440, 0x03000000, 0x0362c688,
 215         0x2300, 0x000000ff, 0x00000001,
 216         0x390, 0x00001fff, 0x00001fff,
 217         0x2418, 0x0000007f, 0x00000020,
 218         0x2542, 0x00010000, 0x00010000,
 219         0x2b05, 0x000003ff, 0x000000f3,
 220         0x2b03, 0xffffffff, 0x00001032
 221 };
 222
 223 static const u32 bonaire_mgcg_cgcg_init[] =
 224 {
 225         0x3108, 0xffffffff, 0xfffffffc,
 226         0xc200, 0xffffffff, 0xe0000000,
 227         0xf0a8, 0xffffffff, 0x00000100,
 228         0xf082, 0xffffffff, 0x00000100,
 229         0xf0b0, 0xffffffff, 0xc0000100,
 230         0xf0b2, 0xffffffff, 0xc0000100,
 231         0xf0b1, 0xffffffff, 0xc0000100,
 232         0x1579, 0xffffffff, 0x00600100,
 233         0xf0a0, 0xffffffff, 0x00000100,
 234         0xf085, 0xffffffff, 0x06000100,
 235         0xf088, 0xffffffff, 0x00000100,
 236         0xf086, 0xffffffff, 0x06000100,
 237         0xf081, 0xffffffff, 0x00000100,
 238         0xf0b8, 0xffffffff, 0x00000100,
 239         0xf089, 0xffffffff, 0x00000100,
 240         0xf080, 0xffffffff, 0x00000100,
 241         0xf08c, 0xffffffff, 0x00000100,
 242         0xf08d, 0xffffffff, 0x00000100,
 243         0xf094, 0xffffffff, 0x00000100,
 244         0xf095, 0xffffffff, 0x00000100,
 245         0xf096, 0xffffffff, 0x00000100,
 246         0xf097, 0xffffffff, 0x00000100,
 247         0xf098, 0xffffffff, 0x00000100,
 248         0xf09f, 0xffffffff, 0x00000100,
 249         0xf09e, 0xffffffff, 0x00000100,
 250         0xf084, 0xffffffff, 0x06000100,
 251         0xf0a4, 0xffffffff, 0x00000100,
 252         0xf09d, 0xffffffff, 0x00000100,
 253         0xf0ad, 0xffffffff, 0x00000100,
 254         0xf0ac, 0xffffffff, 0x00000100,
 255         0xf09c, 0xffffffff, 0x00000100,
 256         0xc200, 0xffffffff, 0xe0000000,
 257         0xf008, 0xffffffff, 0x00010000,
 258         0xf009, 0xffffffff, 0x00030002,
 259         0xf00a, 0xffffffff, 0x00040007,
 260         0xf00b, 0xffffffff, 0x00060005,
 261         0xf00c, 0xffffffff, 0x00090008,
 262         0xf00d, 0xffffffff, 0x00010000,
 263         0xf00e, 0xffffffff, 0x00030002,
 264         0xf00f, 0xffffffff, 0x00040007,
 265         0xf010, 0xffffffff, 0x00060005,
 266         0xf011, 0xffffffff, 0x00090008,
 267         0xf012, 0xffffffff, 0x00010000,
 268         0xf013, 0xffffffff, 0x00030002,
 269         0xf014, 0xffffffff, 0x00040007,
 270         0xf015, 0xffffffff, 0x00060005,
 271         0xf016, 0xffffffff, 0x00090008,
 272         0xf017, 0xffffffff, 0x00010000,
 273         0xf018, 0xffffffff, 0x00030002,
 274         0xf019, 0xffffffff, 0x00040007,
 275         0xf01a, 0xffffffff, 0x00060005,
 276         0xf01b, 0xffffffff, 0x00090008,
 277         0xf01c, 0xffffffff, 0x00010000,
 278         0xf01d, 0xffffffff, 0x00030002,
 279         0xf01e, 0xffffffff, 0x00040007,
 280         0xf01f, 0xffffffff, 0x00060005,
 281         0xf020, 0xffffffff, 0x00090008,
 282         0xf021, 0xffffffff, 0x00010000,
 283         0xf022, 0xffffffff, 0x00030002,
 284         0xf023, 0xffffffff, 0x00040007,
 285         0xf024, 0xffffffff, 0x00060005,
 286         0xf025, 0xffffffff, 0x00090008,
 287         0xf026, 0xffffffff, 0x00010000,
 288         0xf027, 0xffffffff, 0x00030002,
 289         0xf028, 0xffffffff, 0x00040007,
 290         0xf029, 0xffffffff, 0x00060005,
 291         0xf02a, 0xffffffff, 0x00090008,
 292         0xf000, 0xffffffff, 0x96e00200,
 293         0x21c2, 0xffffffff, 0x00900100,
 294         0x3109, 0xffffffff, 0x0020003f,
 295         0xe, 0xffffffff, 0x0140001c,
 296         0xf, 0x000f0000, 0x000f0000,
 297         0x88, 0xffffffff, 0xc060000c,
 298         0x89, 0xc0000fff, 0x00000100,
 299         0x3e4, 0xffffffff, 0x00000100,
 300         0x3e6, 0x00000101, 0x00000000,
 301         0x82a, 0xffffffff, 0x00000104,
 302         0x1579, 0xff000fff, 0x00000100,
 303         0xc33, 0xc0000fff, 0x00000104,
 304         0x3079, 0x00000001, 0x00000001,
 305         0x3403, 0xff000ff0, 0x00000100,
 306         0x3603, 0xff000ff0, 0x00000100
 307 };
 308
 309 static const u32 spectre_golden_spm_registers[] =
 310 {
 311         0xc200, 0xe0ffffff, 0xe0000000
 312 };
 313
 314 static const u32 spectre_golden_common_registers[] =
 315 {
 316         0x31dc, 0xffffffff, 0x00000800,
 317         0x31dd, 0xffffffff, 0x00000800,
 318         0x31e6, 0xffffffff, 0x00007fbf,
 319         0x31e7, 0xffffffff, 0x00007faf
 320 };
 321
 322 static const u32 spectre_golden_registers[] =
 323 {
 324         0xf000, 0xffff1fff, 0x96940200,
 325         0xf003, 0xffff0001, 0xff000000,
 326         0xf080, 0xfffc0fff, 0x00000100,
 327         0x1bb6, 0x00010101, 0x00010000,
 328         0x260d, 0xf00fffff, 0x00000400,
 329         0x260e, 0xfffffffc, 0x00020200,
 330         0x16ec, 0x000000f0, 0x00000070,
 331         0x16f0, 0xf0311fff, 0x80300000,
 332         0x263e, 0x73773777, 0x12010001,
 333         0x26df, 0x00ff0000, 0x00fc0000,
 334         0xbd2, 0x73773777, 0x12010001,
 335         0x2285, 0xf000003f, 0x00000007,
 336         0x22c9, 0xffffffff, 0x00ffffff,
 337         0xa0d4, 0x3f3f3fff, 0x00000082,
 338         0xa0d5, 0x0000003f, 0x00000000,
 339         0xf9e, 0x00000001, 0x00000002,
 340         0x244f, 0xffff03df, 0x00000004,
 341         0x31da, 0x00000008, 0x00000008,
 342         0x2300, 0x000008ff, 0x00000800,
 343         0x2542, 0x00010000, 0x00010000,
 344         0x2b03, 0xffffffff, 0x54763210,
 345         0x853e, 0x01ff01ff, 0x00000002,
 346         0x8526, 0x007ff800, 0x00200000,
 347         0x8057, 0xffffffff, 0x00000f40,
 348         0xc24d, 0xffffffff, 0x00000001
 349 };
 350
 351 static const u32 spectre_mgcg_cgcg_init[] =
 352 {
 353         0x3108, 0xffffffff, 0xfffffffc,
 354         0xc200, 0xffffffff, 0xe0000000,
 355         0xf0a8, 0xffffffff, 0x00000100,
 356         0xf082, 0xffffffff, 0x00000100,
 357         0xf0b0, 0xffffffff, 0x00000100,
 358         0xf0b2, 0xffffffff, 0x00000100,
 359         0xf0b1, 0xffffffff, 0x00000100,
 360         0x1579, 0xffffffff, 0x00600100,
 361         0xf0a0, 0xffffffff, 0x00000100,
 362         0xf085, 0xffffffff, 0x06000100,
 363         0xf088, 0xffffffff, 0x00000100,
 364         0xf086, 0xffffffff, 0x06000100,
 365         0xf081, 0xffffffff, 0x00000100,
 366         0xf0b8, 0xffffffff, 0x00000100,
 367         0xf089, 0xffffffff, 0x00000100,
 368         0xf080, 0xffffffff, 0x00000100,
 369         0xf08c, 0xffffffff, 0x00000100,
 370         0xf08d, 0xffffffff, 0x00000100,
 371         0xf094, 0xffffffff, 0x00000100,
 372         0xf095, 0xffffffff, 0x00000100,
 373         0xf096, 0xffffffff, 0x00000100,
 374         0xf097, 0xffffffff, 0x00000100,
 375         0xf098, 0xffffffff, 0x00000100,
 376         0xf09f, 0xffffffff, 0x00000100,
 377         0xf09e, 0xffffffff, 0x00000100,
 378         0xf084, 0xffffffff, 0x06000100,
 379         0xf0a4, 0xffffffff, 0x00000100,
 380         0xf09d, 0xffffffff, 0x00000100,
 381         0xf0ad, 0xffffffff, 0x00000100,
 382         0xf0ac, 0xffffffff, 0x00000100,
 383         0xf09c, 0xffffffff, 0x00000100,
 384         0xc200, 0xffffffff, 0xe0000000,
 385         0xf008, 0xffffffff, 0x00010000,
 386         0xf009, 0xffffffff, 0x00030002,
 387         0xf00a, 0xffffffff, 0x00040007,
 388         0xf00b, 0xffffffff, 0x00060005,
 389         0xf00c, 0xffffffff, 0x00090008,
 390         0xf00d, 0xffffffff, 0x00010000,
 391         0xf00e, 0xffffffff, 0x00030002,
 392         0xf00f, 0xffffffff, 0x00040007,
 393         0xf010, 0xffffffff, 0x00060005,
 394         0xf011, 0xffffffff, 0x00090008,
 395         0xf012, 0xffffffff, 0x00010000,
 396         0xf013, 0xffffffff, 0x00030002,
 397         0xf014, 0xffffffff, 0x00040007,
 398         0xf015, 0xffffffff, 0x00060005,
 399         0xf016, 0xffffffff, 0x00090008,
 400         0xf017, 0xffffffff, 0x00010000,
 401         0xf018, 0xffffffff, 0x00030002,
 402         0xf019, 0xffffffff, 0x00040007,
 403         0xf01a, 0xffffffff, 0x00060005,
 404         0xf01b, 0xffffffff, 0x00090008,
 405         0xf01c, 0xffffffff, 0x00010000,
 406         0xf01d, 0xffffffff, 0x00030002,
 407         0xf01e, 0xffffffff, 0x00040007,
 408         0xf01f, 0xffffffff, 0x00060005,
 409         0xf020, 0xffffffff, 0x00090008,
 410         0xf021, 0xffffffff, 0x00010000,
 411         0xf022, 0xffffffff, 0x00030002,
 412         0xf023, 0xffffffff, 0x00040007,
 413         0xf024, 0xffffffff, 0x00060005,
 414         0xf025, 0xffffffff, 0x00090008,
 415         0xf026, 0xffffffff, 0x00010000,
 416         0xf027, 0xffffffff, 0x00030002,
 417         0xf028, 0xffffffff, 0x00040007,
 418         0xf029, 0xffffffff, 0x00060005,
 419         0xf02a, 0xffffffff, 0x00090008,
 420         0xf02b, 0xffffffff, 0x00010000,
 421         0xf02c, 0xffffffff, 0x00030002,
 422         0xf02d, 0xffffffff, 0x00040007,
 423         0xf02e, 0xffffffff, 0x00060005,
 424         0xf02f, 0xffffffff, 0x00090008,
 425         0xf000, 0xffffffff, 0x96e00200,
 426         0x21c2, 0xffffffff, 0x00900100,
 427         0x3109, 0xffffffff, 0x0020003f,
 428         0xe, 0xffffffff, 0x0140001c,
 429         0xf, 0x000f0000, 0x000f0000,
 430         0x88, 0xffffffff, 0xc060000c,
 431         0x89, 0xc0000fff, 0x00000100,
 432         0x3e4, 0xffffffff, 0x00000100,
 433         0x3e6, 0x00000101, 0x00000000,
 434         0x82a, 0xffffffff, 0x00000104,
 435         0x1579, 0xff000fff, 0x00000100,
 436         0xc33, 0xc0000fff, 0x00000104,
 437         0x3079, 0x00000001, 0x00000001,
 438         0x3403, 0xff000ff0, 0x00000100,
 439         0x3603, 0xff000ff0, 0x00000100
 440 };
 441
 442 static const u32 kalindi_golden_spm_registers[] =
 443 {
 444         0xc200, 0xe0ffffff, 0xe0000000
 445 };
 446
 447 static const u32 kalindi_golden_common_registers[] =
 448 {
 449         0x31dc, 0xffffffff, 0x00000800,
 450         0x31dd, 0xffffffff, 0x00000800,
 451         0x31e6, 0xffffffff, 0x00007fbf,
 452         0x31e7, 0xffffffff, 0x00007faf
 453 };
 454
 455 static const u32 kalindi_golden_registers[] =
 456 {
 457         0xf000, 0xffffdfff, 0x6e944040,
 458         0x1579, 0xff607fff, 0xfc000100,
 459         0xf088, 0xff000fff, 0x00000100,
 460         0xf089, 0xff000fff, 0x00000100,
 461         0xf080, 0xfffc0fff, 0x00000100,
 462         0x1bb6, 0x00010101, 0x00010000,
 463         0x260c, 0xffffffff, 0x00000000,
 464         0x260d, 0xf00fffff, 0x00000400,
 465         0x16ec, 0x000000f0, 0x00000070,
 466         0x16f0, 0xf0311fff, 0x80300000,
 467         0x263e, 0x73773777, 0x12010001,
 468         0x263f, 0xffffffff, 0x00000010,
 469         0x26df, 0x00ff0000, 0x00fc0000,
 470         0x200c, 0x00001f0f, 0x0000100a,
 471         0xbd2, 0x73773777, 0x12010001,
 472         0x902, 0x000fffff, 0x000c007f,
 473         0x2285, 0xf000003f, 0x00000007,
 474         0x22c9, 0x3fff3fff, 0x00ffcfff,
 475         0xc281, 0x0000ff0f, 0x00000000,
 476         0xa293, 0x07ffffff, 0x06000000,
 477         0x136, 0x00000fff, 0x00000100,
 478         0xf9e, 0x00000001, 0x00000002,
 479         0x31da, 0x00000008, 0x00000008,
 480         0x2300, 0x000000ff, 0x00000003,
 481         0x853e, 0x01ff01ff, 0x00000002,
 482         0x8526, 0x007ff800, 0x00200000,
 483         0x8057, 0xffffffff, 0x00000f40,
 484         0x2231, 0x001f3ae3, 0x00000082,
 485         0x2235, 0x0000001f, 0x00000010,
 486         0xc24d, 0xffffffff, 0x00000000
 487 };
 488
 489 static const u32 kalindi_mgcg_cgcg_init[] =
 490 {
 491         0x3108, 0xffffffff, 0xfffffffc,
 492         0xc200, 0xffffffff, 0xe0000000,
 493         0xf0a8, 0xffffffff, 0x00000100,
 494         0xf082, 0xffffffff, 0x00000100,
 495         0xf0b0, 0xffffffff, 0x00000100,
 496         0xf0b2, 0xffffffff, 0x00000100,
 497         0xf0b1, 0xffffffff, 0x00000100,
 498         0x1579, 0xffffffff, 0x00600100,
 499         0xf0a0, 0xffffffff, 0x00000100,
 500         0xf085, 0xffffffff, 0x06000100,
 501         0xf088, 0xffffffff, 0x00000100,
 502         0xf086, 0xffffffff, 0x06000100,
 503         0xf081, 0xffffffff, 0x00000100,
 504         0xf0b8, 0xffffffff, 0x00000100,
 505         0xf089, 0xffffffff, 0x00000100,
 506         0xf080, 0xffffffff, 0x00000100,
 507         0xf08c, 0xffffffff, 0x00000100,
 508         0xf08d, 0xffffffff, 0x00000100,
 509         0xf094, 0xffffffff, 0x00000100,
 510         0xf095, 0xffffffff, 0x00000100,
 511         0xf096, 0xffffffff, 0x00000100,
 512         0xf097, 0xffffffff, 0x00000100,
 513         0xf098, 0xffffffff, 0x00000100,
 514         0xf09f, 0xffffffff, 0x00000100,
 515         0xf09e, 0xffffffff, 0x00000100,
 516         0xf084, 0xffffffff, 0x06000100,
 517         0xf0a4, 0xffffffff, 0x00000100,
 518         0xf09d, 0xffffffff, 0x00000100,
 519         0xf0ad, 0xffffffff, 0x00000100,
 520         0xf0ac, 0xffffffff, 0x00000100,
 521         0xf09c, 0xffffffff, 0x00000100,
 522         0xc200, 0xffffffff, 0xe0000000,
 523         0xf008, 0xffffffff, 0x00010000,
 524         0xf009, 0xffffffff, 0x00030002,
 525         0xf00a, 0xffffffff, 0x00040007,
 526         0xf00b, 0xffffffff, 0x00060005,
 527         0xf00c, 0xffffffff, 0x00090008,
 528         0xf00d, 0xffffffff, 0x00010000,
 529         0xf00e, 0xffffffff, 0x00030002,
 530         0xf00f, 0xffffffff, 0x00040007,
 531         0xf010, 0xffffffff, 0x00060005,
 532         0xf011, 0xffffffff, 0x00090008,
 533         0xf000, 0xffffffff, 0x96e00200,
 534         0x21c2, 0xffffffff, 0x00900100,
 535         0x3109, 0xffffffff, 0x0020003f,
 536         0xe, 0xffffffff, 0x0140001c,
 537         0xf, 0x000f0000, 0x000f0000,
 538         0x88, 0xffffffff, 0xc060000c,
 539         0x89, 0xc0000fff, 0x00000100,
 540         0x82a, 0xffffffff, 0x00000104,
 541         0x1579, 0xff000fff, 0x00000100,
 542         0xc33, 0xc0000fff, 0x00000104,
 543         0x3079, 0x00000001, 0x00000001,
 544         0x3403, 0xff000ff0, 0x00000100,
 545         0x3603, 0xff000ff0, 0x00000100
 546 };
 547
 548 static const u32 hawaii_golden_spm_registers[] =
 549 {
 550         0xc200, 0xe0ffffff, 0xe0000000
 551 };
 552
 553 static const u32 hawaii_golden_common_registers[] =
 554 {
 555         0xc200, 0xffffffff, 0xe0000000,
 556         0xa0d4, 0xffffffff, 0x3a00161a,
 557         0xa0d5, 0xffffffff, 0x0000002e,
 558         0x2684, 0xffffffff, 0x00018208,
 559         0x263e, 0xffffffff, 0x12011003
 560 };
 561
 562 static const u32 hawaii_golden_registers[] =
 563 {
 564         0xcd5, 0x00000333, 0x00000333,
 565         0x2684, 0x00010000, 0x00058208,
 566         0x260c, 0xffffffff, 0x00000000,
 567         0x260d, 0xf00fffff, 0x00000400,
 568         0x260e, 0x0002021c, 0x00020200,
 569         0x31e, 0x00000080, 0x00000000,
 570         0x16ec, 0x000000f0, 0x00000070,
 571         0x16f0, 0xf0311fff, 0x80300000,
 572         0xd43, 0x00810000, 0x408af000,
 573         0x1c0c, 0x31000111, 0x00000011,
 574         0xbd2, 0x73773777, 0x12010001,
 575         0x848, 0x0000007f, 0x0000001b,
 576         0x877, 0x00007fb6, 0x00002191,
 577         0xd8a, 0x0000003f, 0x0000000a,
 578         0xd8b, 0x0000003f, 0x0000000a,
 579         0xab9, 0x00073ffe, 0x000022a2,
 580         0x903, 0x000007ff, 0x00000000,
 581         0x22fc, 0x00002001, 0x00000001,
 582         0x22c9, 0xffffffff, 0x00ffffff,
 583         0xc281, 0x0000ff0f, 0x00000000,
 584         0xa293, 0x07ffffff, 0x06000000,
 585         0xf9e, 0x00000001, 0x00000002,
 586         0x31da, 0x00000008, 0x00000008,
 587         0x31dc, 0x00000f00, 0x00000800,
 588         0x31dd, 0x00000f00, 0x00000800,
 589         0x31e6, 0x00ffffff, 0x00ff7fbf,
 590         0x31e7, 0x00ffffff, 0x00ff7faf,
 591         0x2300, 0x000000ff, 0x00000800,
 592         0x390, 0x00001fff, 0x00001fff,
 593         0x2418, 0x0000007f, 0x00000020,
 594         0x2542, 0x00010000, 0x00010000,
 595         0x2b80, 0x00100000, 0x000ff07c,
 596         0x2b05, 0x000003ff, 0x0000000f,
 597         0x2b04, 0xffffffff, 0x7564fdec,
 598         0x2b03, 0xffffffff, 0x3120b9a8,
 599         0x2b02, 0x20000000, 0x0f9c0000
 600 };
 601
 602 static const u32 hawaii_mgcg_cgcg_init[] =
 603 {
 604         0x3108, 0xffffffff, 0xfffffffd,
 605         0xc200, 0xffffffff, 0xe0000000,
 606         0xf0a8, 0xffffffff, 0x00000100,
 607         0xf082, 0xffffffff, 0x00000100,
 608         0xf0b0, 0xffffffff, 0x00000100,
 609         0xf0b2, 0xffffffff, 0x00000100,
 610         0xf0b1, 0xffffffff, 0x00000100,
 611         0x1579, 0xffffffff, 0x00200100,
 612         0xf0a0, 0xffffffff, 0x00000100,
 613         0xf085, 0xffffffff, 0x06000100,
 614         0xf088, 0xffffffff, 0x00000100,
 615         0xf086, 0xffffffff, 0x06000100,
 616         0xf081, 0xffffffff, 0x00000100,
 617         0xf0b8, 0xffffffff, 0x00000100,
 618         0xf089, 0xffffffff, 0x00000100,
 619         0xf080, 0xffffffff, 0x00000100,
 620         0xf08c, 0xffffffff, 0x00000100,
 621         0xf08d, 0xffffffff, 0x00000100,
 622         0xf094, 0xffffffff, 0x00000100,
 623         0xf095, 0xffffffff, 0x00000100,
 624         0xf096, 0xffffffff, 0x00000100,
 625         0xf097, 0xffffffff, 0x00000100,
 626         0xf098, 0xffffffff, 0x00000100,
 627         0xf09f, 0xffffffff, 0x00000100,
 628         0xf09e, 0xffffffff, 0x00000100,
 629         0xf084, 0xffffffff, 0x06000100,
 630         0xf0a4, 0xffffffff, 0x00000100,
 631         0xf09d, 0xffffffff, 0x00000100,
 632         0xf0ad, 0xffffffff, 0x00000100,
 633         0xf0ac, 0xffffffff, 0x00000100,
 634         0xf09c, 0xffffffff, 0x00000100,
 635         0xc200, 0xffffffff, 0xe0000000,
 636         0xf008, 0xffffffff, 0x00010000,
 637         0xf009, 0xffffffff, 0x00030002,
 638         0xf00a, 0xffffffff, 0x00040007,
 639         0xf00b, 0xffffffff, 0x00060005,
 640         0xf00c, 0xffffffff, 0x00090008,
 641         0xf00d, 0xffffffff, 0x00010000,
 642         0xf00e, 0xffffffff, 0x00030002,
 643         0xf00f, 0xffffffff, 0x00040007,
 644         0xf010, 0xffffffff, 0x00060005,
 645         0xf011, 0xffffffff, 0x00090008,
 646         0xf012, 0xffffffff, 0x00010000,
 647         0xf013, 0xffffffff, 0x00030002,
 648         0xf014, 0xffffffff, 0x00040007,
 649         0xf015, 0xffffffff, 0x00060005,
 650         0xf016, 0xffffffff, 0x00090008,
 651         0xf017, 0xffffffff, 0x00010000,
 652         0xf018, 0xffffffff, 0x00030002,
 653         0xf019, 0xffffffff, 0x00040007,
 654         0xf01a, 0xffffffff, 0x00060005,
 655         0xf01b, 0xffffffff, 0x00090008,
 656         0xf01c, 0xffffffff, 0x00010000,
 657         0xf01d, 0xffffffff, 0x00030002,
 658         0xf01e, 0xffffffff, 0x00040007,
 659         0xf01f, 0xffffffff, 0x00060005,
 660         0xf020, 0xffffffff, 0x00090008,
 661         0xf021, 0xffffffff, 0x00010000,
 662         0xf022, 0xffffffff, 0x00030002,
 663         0xf023, 0xffffffff, 0x00040007,
 664         0xf024, 0xffffffff, 0x00060005,
 665         0xf025, 0xffffffff, 0x00090008,
 666         0xf026, 0xffffffff, 0x00010000,
 667         0xf027, 0xffffffff, 0x00030002,
 668         0xf028, 0xffffffff, 0x00040007,
 669         0xf029, 0xffffffff, 0x00060005,
 670         0xf02a, 0xffffffff, 0x00090008,
 671         0xf02b, 0xffffffff, 0x00010000,
 672         0xf02c, 0xffffffff, 0x00030002,
 673         0xf02d, 0xffffffff, 0x00040007,
 674         0xf02e, 0xffffffff, 0x00060005,
 675         0xf02f, 0xffffffff, 0x00090008,
 676         0xf030, 0xffffffff, 0x00010000,
 677         0xf031, 0xffffffff, 0x00030002,
 678         0xf032, 0xffffffff, 0x00040007,
 679         0xf033, 0xffffffff, 0x00060005,
 680         0xf034, 0xffffffff, 0x00090008,
 681         0xf035, 0xffffffff, 0x00010000,
 682         0xf036, 0xffffffff, 0x00030002,
 683         0xf037, 0xffffffff, 0x00040007,
 684         0xf038, 0xffffffff, 0x00060005,
 685         0xf039, 0xffffffff, 0x00090008,
 686         0xf03a, 0xffffffff, 0x00010000,
 687         0xf03b, 0xffffffff, 0x00030002,
 688         0xf03c, 0xffffffff, 0x00040007,
 689         0xf03d, 0xffffffff, 0x00060005,
 690         0xf03e, 0xffffffff, 0x00090008,
 691         0x30c6, 0xffffffff, 0x00020200,
 692         0xcd4, 0xffffffff, 0x00000200,
 693         0x570, 0xffffffff, 0x00000400,
 694         0x157a, 0xffffffff, 0x00000000,
 695         0xbd4, 0xffffffff, 0x00000902,
 696         0xf000, 0xffffffff, 0x96940200,
 697         0x21c2, 0xffffffff, 0x00900100,
 698         0x3109, 0xffffffff, 0x0020003f,
 699         0xe, 0xffffffff, 0x0140001c,
 700         0xf, 0x000f0000, 0x000f0000,
 701         0x88, 0xffffffff, 0xc060000c,
 702         0x89, 0xc0000fff, 0x00000100,
 703         0x3e4, 0xffffffff, 0x00000100,
 704         0x3e6, 0x00000101, 0x00000000,
 705         0x82a, 0xffffffff, 0x00000104,
 706         0x1579, 0xff000fff, 0x00000100,
 707         0xc33, 0xc0000fff, 0x00000104,
 708         0x3079, 0x00000001, 0x00000001,
 709         0x3403, 0xff000ff0, 0x00000100,
 710         0x3603, 0xff000ff0, 0x00000100
 711 };
 712
 713 static const u32 godavari_golden_registers[] =
 714 {
 715         0x1579, 0xff607fff, 0xfc000100,
 716         0x1bb6, 0x00010101, 0x00010000,
 717         0x260c, 0xffffffff, 0x00000000,
 718         0x260c0, 0xf00fffff, 0x00000400,
 719         0x184c, 0xffffffff, 0x00010000,
 720         0x16ec, 0x000000f0, 0x00000070,
 721         0x16f0, 0xf0311fff, 0x80300000,
 722         0x263e, 0x73773777, 0x12010001,
 723         0x263f, 0xffffffff, 0x00000010,
 724         0x200c, 0x00001f0f, 0x0000100a,
 725         0xbd2, 0x73773777, 0x12010001,
 726         0x902, 0x000fffff, 0x000c007f,
 727         0x2285, 0xf000003f, 0x00000007,
 728         0x22c9, 0xffffffff, 0x00ff0fff,
 729         0xc281, 0x0000ff0f, 0x00000000,
 730         0xa293, 0x07ffffff, 0x06000000,
 731         0x136, 0x00000fff, 0x00000100,
 732         0x3405, 0x00010000, 0x00810001,
 733         0x3605, 0x00010000, 0x00810001,
 734         0xf9e, 0x00000001, 0x00000002,
 735         0x31da, 0x00000008, 0x00000008,
 736         0x31dc, 0x00000f00, 0x00000800,
 737         0x31dd, 0x00000f00, 0x00000800,
 738         0x31e6, 0x00ffffff, 0x00ff7fbf,
 739         0x31e7, 0x00ffffff, 0x00ff7faf,
 740         0x2300, 0x000000ff, 0x00000001,
 741         0x853e, 0x01ff01ff, 0x00000002,
 742         0x8526, 0x007ff800, 0x00200000,
 743         0x8057, 0xffffffff, 0x00000f40,
 744         0x2231, 0x001f3ae3, 0x00000082,
 745         0x2235, 0x0000001f, 0x00000010,
 746         0xc24d, 0xffffffff, 0x00000000
 747 };
 748
 749 static void cik_init_golden_registers(struct amdgpu_device *adev)
 750 {
 751         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 752         mutex_lock(&adev->grbm_idx_mutex);
 753
 754         switch (adev->asic_type) {
 755         case CHIP_BONAIRE:
 756                 amdgpu_program_register_sequence(adev,
 757                                                  bonaire_mgcg_cgcg_init,
 758                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 759                 amdgpu_program_register_sequence(adev,
 760                                                  bonaire_golden_registers,
 761                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 762                 amdgpu_program_register_sequence(adev,
 763                                                  bonaire_golden_common_registers,
 764                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 765                 amdgpu_program_register_sequence(adev,
 766                                                  bonaire_golden_spm_registers,
 767                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 768                 break;
 769         case CHIP_KABINI:
 770                 amdgpu_program_register_sequence(adev,
 771                                                  kalindi_mgcg_cgcg_init,
 772                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 773                 amdgpu_program_register_sequence(adev,
 774                                                  kalindi_golden_registers,
 775                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 776                 amdgpu_program_register_sequence(adev,
 777                                                  kalindi_golden_common_registers,
 778                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 779                 amdgpu_program_register_sequence(adev,
 780                                                  kalindi_golden_spm_registers,
 781                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 782                 break;
 783         case CHIP_MULLINS:
 784                 amdgpu_program_register_sequence(adev,
 785                                                  kalindi_mgcg_cgcg_init,
 786                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 787                 amdgpu_program_register_sequence(adev,
 788                                                  godavari_golden_registers,
 789                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
 790                 amdgpu_program_register_sequence(adev,
 791                                                  kalindi_golden_common_registers,
 792                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 793                 amdgpu_program_register_sequence(adev,
 794                                                  kalindi_golden_spm_registers,
 795                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 796                 break;
 797         case CHIP_KAVERI:
 798                 amdgpu_program_register_sequence(adev,
 799                                                  spectre_mgcg_cgcg_init,
 800                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 801                 amdgpu_program_register_sequence(adev,
 802                                                  spectre_golden_registers,
 803                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 804                 amdgpu_program_register_sequence(adev,
 805                                                  spectre_golden_common_registers,
 806                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 807                 amdgpu_program_register_sequence(adev,
 808                                                  spectre_golden_spm_registers,
 809                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 810                 break;
 811         case CHIP_HAWAII:
 812                 amdgpu_program_register_sequence(adev,
 813                                                  hawaii_mgcg_cgcg_init,
 814                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 815                 amdgpu_program_register_sequence(adev,
 816                                                  hawaii_golden_registers,
 817                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
 818                 amdgpu_program_register_sequence(adev,
 819                                                  hawaii_golden_common_registers,
 820                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 821                 amdgpu_program_register_sequence(adev,
 822                                                  hawaii_golden_spm_registers,
 823                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 824                 break;
 825         default:
 826                 break;
 827         }
 828         mutex_unlock(&adev->grbm_idx_mutex);
 829 }
 830
 831 /**
 832  * cik_get_xclk - get the xclk
 833  *
 834  * @adev: amdgpu_device pointer
 835  *
 836  * Returns the reference clock used by the gfx engine
 837  * (CIK).
 838  */
 839 static u32 cik_get_xclk(struct amdgpu_device *adev)
 840 {
 841         u32 reference_clock = adev->clock.spll.reference_freq;
 842
 843         if (adev->flags & AMD_IS_APU) {
 844                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 845                         return reference_clock / 2;
 846         } else {
 847                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 848                         return reference_clock / 4;
 849         }
 850         return reference_clock;
 851 }
 852
 853 /**
 854  * cik_srbm_select - select specific register instances
 855  *
 856  * @adev: amdgpu_device pointer
 857  * @me: selected ME (micro engine)
 858  * @pipe: pipe
 859  * @queue: queue
 860  * @vmid: VMID
 861  *
 862  * Switches the currently active registers instances.  Some
 863  * registers are instanced per VMID, others are instanced per
 864  * me/pipe/queue combination.
 865  */
 866 void cik_srbm_select(struct amdgpu_device *adev,
 867                      u32 me, u32 pipe, u32 queue, u32 vmid)
 868 {
 869         u32 srbm_gfx_cntl =
 870                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 871                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 872                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 873                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 874         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 875 }
 876
 877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 878 {
 879         uint32_t tmp;
 880
 881         tmp = RREG32(mmCONFIG_CNTL);
 882         if (state == false)
 883                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 884         else
 885                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 886         WREG32(mmCONFIG_CNTL, tmp);
 887 }
 888
 889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 890 {
 891         u32 bus_cntl;
 892         u32 d1vga_control = 0;
 893         u32 d2vga_control = 0;
 894         u32 vga_render_control = 0;
 895         u32 rom_cntl;
 896         bool r;
 897
 898         bus_cntl = RREG32(mmBUS_CNTL);
 899         if (adev->mode_info.num_crtc) {
 900                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 901                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 902                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 903         }
 904         rom_cntl = RREG32_SMC(ixROM_CNTL);
 905
 906         /* enable the rom */
 907         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 908         if (adev->mode_info.num_crtc) {
 909                 /* Disable VGA mode */
 910                 WREG32(mmD1VGA_CONTROL,
 911                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 912                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 913                 WREG32(mmD2VGA_CONTROL,
 914                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 915                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 916                 WREG32(mmVGA_RENDER_CONTROL,
 917                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 918         }
 919         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 920
 921         r = amdgpu_read_bios(adev);
 922
 923         /* restore regs */
 924         WREG32(mmBUS_CNTL, bus_cntl);
 925         if (adev->mode_info.num_crtc) {
 926                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 927                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 928                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 929         }
 930         WREG32_SMC(ixROM_CNTL, rom_cntl);
 931         return r;
 932 }
 933
 934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 935                                    u8 *bios, u32 length_bytes)
 936 {
 937         u32 *dw_ptr;
 938         unsigned long flags;
 939         u32 i, length_dw;
 940
 941         if (bios == NULL)
 942                 return false;
 943         if (length_bytes == 0)
 944                 return false;
 945         /* APU vbios image is part of sbios image */
 946         if (adev->flags & AMD_IS_APU)
 947                 return false;
 948
 949         dw_ptr = (u32 *)bios;
 950         length_dw = ALIGN(length_bytes, 4) / 4;
 951         /* take the smc lock since we are using the smc index */
 952         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 953         /* set rom index to 0 */
 954         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 955         WREG32(mmSMC_IND_DATA_0, 0);
 956         /* set index to data for continous read */
 957         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 958         for (i = 0; i < length_dw; i++)
 959                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 960         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 961
 962         return true;
 963 }
 964
 965 static struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 966         {mmGRBM_STATUS, false},
 967         {mmGB_ADDR_CONFIG, false},
 968         {mmMC_ARB_RAMCFG, false},
 969         {mmGB_TILE_MODE0, false},
 970         {mmGB_TILE_MODE1, false},
 971         {mmGB_TILE_MODE2, false},
 972         {mmGB_TILE_MODE3, false},
 973         {mmGB_TILE_MODE4, false},
 974         {mmGB_TILE_MODE5, false},
 975         {mmGB_TILE_MODE6, false},
 976         {mmGB_TILE_MODE7, false},
 977         {mmGB_TILE_MODE8, false},
 978         {mmGB_TILE_MODE9, false},
 979         {mmGB_TILE_MODE10, false},
 980         {mmGB_TILE_MODE11, false},
 981         {mmGB_TILE_MODE12, false},
 982         {mmGB_TILE_MODE13, false},
 983         {mmGB_TILE_MODE14, false},
 984         {mmGB_TILE_MODE15, false},
 985         {mmGB_TILE_MODE16, false},
 986         {mmGB_TILE_MODE17, false},
 987         {mmGB_TILE_MODE18, false},
 988         {mmGB_TILE_MODE19, false},
 989         {mmGB_TILE_MODE20, false},
 990         {mmGB_TILE_MODE21, false},
 991         {mmGB_TILE_MODE22, false},
 992         {mmGB_TILE_MODE23, false},
 993         {mmGB_TILE_MODE24, false},
 994         {mmGB_TILE_MODE25, false},
 995         {mmGB_TILE_MODE26, false},
 996         {mmGB_TILE_MODE27, false},
 997         {mmGB_TILE_MODE28, false},
 998         {mmGB_TILE_MODE29, false},
 999         {mmGB_TILE_MODE30, false},
1000         {mmGB_TILE_MODE31, false},
1001         {mmGB_MACROTILE_MODE0, false},
1002         {mmGB_MACROTILE_MODE1, false},
1003         {mmGB_MACROTILE_MODE2, false},
1004         {mmGB_MACROTILE_MODE3, false},
1005         {mmGB_MACROTILE_MODE4, false},
1006         {mmGB_MACROTILE_MODE5, false},
1007         {mmGB_MACROTILE_MODE6, false},
1008         {mmGB_MACROTILE_MODE7, false},
1009         {mmGB_MACROTILE_MODE8, false},
1010         {mmGB_MACROTILE_MODE9, false},
1011         {mmGB_MACROTILE_MODE10, false},
1012         {mmGB_MACROTILE_MODE11, false},
1013         {mmGB_MACROTILE_MODE12, false},
1014         {mmGB_MACROTILE_MODE13, false},
1015         {mmGB_MACROTILE_MODE14, false},
1016         {mmGB_MACROTILE_MODE15, false},
1017         {mmCC_RB_BACKEND_DISABLE, false, true},
1018         {mmGC_USER_RB_BACKEND_DISABLE, false, true},
1019         {mmGB_BACKEND_MAP, false, false},
1020         {mmPA_SC_RASTER_CONFIG, false, true},
1021         {mmPA_SC_RASTER_CONFIG_1, false, true},
1022 };
1023
1024 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1025                                           u32 se_num, u32 sh_num,
1026                                           u32 reg_offset)
1027 {
1028         uint32_t val;
1029
1030         mutex_lock(&adev->grbm_idx_mutex);
1031         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1032                 gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1033
1034         val = RREG32(reg_offset);
1035
1036         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1037                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1038         mutex_unlock(&adev->grbm_idx_mutex);
1039         return val;
1040 }
1041
1042 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1043                              u32 sh_num, u32 reg_offset, u32 *value)
1044 {
1045         uint32_t i;
1046
1047         *value = 0;
1048         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1049                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1050                         continue;
1051
1052                 if (!cik_allowed_read_registers[i].untouched)
1053                         *value = cik_allowed_read_registers[i].grbm_indexed ?
1054                                  cik_read_indexed_register(adev, se_num,
1055                                                            sh_num, reg_offset) :
1056                                  RREG32(reg_offset);
1057                 return 0;
1058         }
1059         return -EINVAL;
1060 }
1061
1062 static void cik_print_gpu_status_regs(struct amdgpu_device *adev)
1063 {
1064         dev_info(adev->dev, "  GRBM_STATUS=0x%08X\n",
1065                 RREG32(mmGRBM_STATUS));
1066         dev_info(adev->dev, "  GRBM_STATUS2=0x%08X\n",
1067                 RREG32(mmGRBM_STATUS2));
1068         dev_info(adev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
1069                 RREG32(mmGRBM_STATUS_SE0));
1070         dev_info(adev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
1071                 RREG32(mmGRBM_STATUS_SE1));
1072         dev_info(adev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
1073                 RREG32(mmGRBM_STATUS_SE2));
1074         dev_info(adev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
1075                 RREG32(mmGRBM_STATUS_SE3));
1076         dev_info(adev->dev, "  SRBM_STATUS=0x%08X\n",
1077                 RREG32(mmSRBM_STATUS));
1078         dev_info(adev->dev, "  SRBM_STATUS2=0x%08X\n",
1079                 RREG32(mmSRBM_STATUS2));
1080         dev_info(adev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
1081                 RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
1082         dev_info(adev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
1083                  RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
1084         dev_info(adev->dev, "  CP_STAT = 0x%08x\n", RREG32(mmCP_STAT));
1085         dev_info(adev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
1086                  RREG32(mmCP_STALLED_STAT1));
1087         dev_info(adev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
1088                  RREG32(mmCP_STALLED_STAT2));
1089         dev_info(adev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
1090                  RREG32(mmCP_STALLED_STAT3));
1091         dev_info(adev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
1092                  RREG32(mmCP_CPF_BUSY_STAT));
1093         dev_info(adev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
1094                  RREG32(mmCP_CPF_STALLED_STAT1));
1095         dev_info(adev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(mmCP_CPF_STATUS));
1096         dev_info(adev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(mmCP_CPC_BUSY_STAT));
1097         dev_info(adev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
1098                  RREG32(mmCP_CPC_STALLED_STAT1));
1099         dev_info(adev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(mmCP_CPC_STATUS));
1100 }
1101
1102 /**
1103  * cik_gpu_check_soft_reset - check which blocks are busy
1104  *
1105  * @adev: amdgpu_device pointer
1106  *
1107  * Check which blocks are busy and return the relevant reset
1108  * mask to be used by cik_gpu_soft_reset().
1109  * Returns a mask of the blocks to be reset.
1110  */
1111 u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev)
1112 {
1113         u32 reset_mask = 0;
1114         u32 tmp;
1115
1116         /* GRBM_STATUS */
1117         tmp = RREG32(mmGRBM_STATUS);
1118         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
1119                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
1120                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
1121                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
1122                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
1123                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK))
1124                 reset_mask |= AMDGPU_RESET_GFX;
1125
1126         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK))
1127                 reset_mask |= AMDGPU_RESET_CP;
1128
1129         /* GRBM_STATUS2 */
1130         tmp = RREG32(mmGRBM_STATUS2);
1131         if (tmp & GRBM_STATUS2__RLC_BUSY_MASK)
1132                 reset_mask |= AMDGPU_RESET_RLC;
1133
1134         /* SDMA0_STATUS_REG */
1135         tmp = RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
1136         if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1137                 reset_mask |= AMDGPU_RESET_DMA;
1138
1139         /* SDMA1_STATUS_REG */
1140         tmp = RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
1141         if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1142                 reset_mask |= AMDGPU_RESET_DMA1;
1143
1144         /* SRBM_STATUS2 */
1145         tmp = RREG32(mmSRBM_STATUS2);
1146         if (tmp & SRBM_STATUS2__SDMA_BUSY_MASK)
1147                 reset_mask |= AMDGPU_RESET_DMA;
1148
1149         if (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)
1150                 reset_mask |= AMDGPU_RESET_DMA1;
1151
1152         /* SRBM_STATUS */
1153         tmp = RREG32(mmSRBM_STATUS);
1154
1155         if (tmp & SRBM_STATUS__IH_BUSY_MASK)
1156                 reset_mask |= AMDGPU_RESET_IH;
1157
1158         if (tmp & SRBM_STATUS__SEM_BUSY_MASK)
1159                 reset_mask |= AMDGPU_RESET_SEM;
1160
1161         if (tmp & SRBM_STATUS__GRBM_RQ_PENDING_MASK)
1162                 reset_mask |= AMDGPU_RESET_GRBM;
1163
1164         if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1165                 reset_mask |= AMDGPU_RESET_VMC;
1166
1167         if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1168                    SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK))
1169                 reset_mask |= AMDGPU_RESET_MC;
1170
1171         if (amdgpu_display_is_display_hung(adev))
1172                 reset_mask |= AMDGPU_RESET_DISPLAY;
1173
1174         /* Skip MC reset as it's mostly likely not hung, just busy */
1175         if (reset_mask & AMDGPU_RESET_MC) {
1176                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1177                 reset_mask &= ~AMDGPU_RESET_MC;
1178         }
1179
1180         return reset_mask;
1181 }
1182
1183 /**
1184  * cik_gpu_soft_reset - soft reset GPU
1185  *
1186  * @adev: amdgpu_device pointer
1187  * @reset_mask: mask of which blocks to reset
1188  *
1189  * Soft reset the blocks specified in @reset_mask.
1190  */
1191 static void cik_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask)
1192 {
1193         struct amdgpu_mode_mc_save save;
1194         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1195         u32 tmp;
1196
1197         if (reset_mask == 0)
1198                 return;
1199
1200         dev_info(adev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1201
1202         cik_print_gpu_status_regs(adev);
1203         dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1204                  RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR));
1205         dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1206                  RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS));
1207
1208         /* disable CG/PG */
1209
1210         /* stop the rlc */
1211         gfx_v7_0_rlc_stop(adev);
1212
1213         /* Disable GFX parsing/prefetching */
1214         WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
1215
1216         /* Disable MEC parsing/prefetching */
1217         WREG32(mmCP_MEC_CNTL, CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
1218
1219         if (reset_mask & AMDGPU_RESET_DMA) {
1220                 /* sdma0 */
1221                 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1222                 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1223                 WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1224         }
1225         if (reset_mask & AMDGPU_RESET_DMA1) {
1226                 /* sdma1 */
1227                 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1228                 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1229                 WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1230         }
1231
1232         gmc_v7_0_mc_stop(adev, &save);
1233         if (amdgpu_asic_wait_for_mc_idle(adev)) {
1234                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
1235         }
1236
1237         if (reset_mask & (AMDGPU_RESET_GFX | AMDGPU_RESET_COMPUTE | AMDGPU_RESET_CP))
1238                 grbm_soft_reset = GRBM_SOFT_RESET__SOFT_RESET_CP_MASK |
1239                         GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK;
1240
1241         if (reset_mask & AMDGPU_RESET_CP) {
1242                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK;
1243
1244                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
1245         }
1246
1247         if (reset_mask & AMDGPU_RESET_DMA)
1248                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;
1249
1250         if (reset_mask & AMDGPU_RESET_DMA1)
1251                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
1252
1253         if (reset_mask & AMDGPU_RESET_DISPLAY)
1254                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
1255
1256         if (reset_mask & AMDGPU_RESET_RLC)
1257                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
1258
1259         if (reset_mask & AMDGPU_RESET_SEM)
1260                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SEM_MASK;
1261
1262         if (reset_mask & AMDGPU_RESET_IH)
1263                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
1264
1265         if (reset_mask & AMDGPU_RESET_GRBM)
1266                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
1267
1268         if (reset_mask & AMDGPU_RESET_VMC)
1269                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_VMC_MASK;
1270
1271         if (!(adev->flags & AMD_IS_APU)) {
1272                 if (reset_mask & AMDGPU_RESET_MC)
1273                         srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_MC_MASK;
1274         }
1275
1276         if (grbm_soft_reset) {
1277                 tmp = RREG32(mmGRBM_SOFT_RESET);
1278                 tmp |= grbm_soft_reset;
1279                 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1280                 WREG32(mmGRBM_SOFT_RESET, tmp);
1281                 tmp = RREG32(mmGRBM_SOFT_RESET);
1282
1283                 udelay(50);
1284
1285                 tmp &= ~grbm_soft_reset;
1286                 WREG32(mmGRBM_SOFT_RESET, tmp);
1287                 tmp = RREG32(mmGRBM_SOFT_RESET);
1288         }
1289
1290         if (srbm_soft_reset) {
1291                 tmp = RREG32(mmSRBM_SOFT_RESET);
1292                 tmp |= srbm_soft_reset;
1293                 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1294                 WREG32(mmSRBM_SOFT_RESET, tmp);
1295                 tmp = RREG32(mmSRBM_SOFT_RESET);
1296
1297                 udelay(50);
1298
1299                 tmp &= ~srbm_soft_reset;
1300                 WREG32(mmSRBM_SOFT_RESET, tmp);
1301                 tmp = RREG32(mmSRBM_SOFT_RESET);
1302         }
1303
1304         /* Wait a little for things to settle down */
1305         udelay(50);
1306
1307         gmc_v7_0_mc_resume(adev, &save);
1308         udelay(50);
1309
1310         cik_print_gpu_status_regs(adev);
1311 }
1312
1313 struct kv_reset_save_regs {
1314         u32 gmcon_reng_execute;
1315         u32 gmcon_misc;
1316         u32 gmcon_misc3;
1317 };
1318
1319 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1320                                    struct kv_reset_save_regs *save)
1321 {
1322         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1323         save->gmcon_misc = RREG32(mmGMCON_MISC);
1324         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1325
1326         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1327                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1328         WREG32(mmGMCON_MISC, save->gmcon_misc &
1329                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1330                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1331 }
1332
1333 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1334                                       struct kv_reset_save_regs *save)
1335 {
1336         int i;
1337
1338         WREG32(mmGMCON_PGFSM_WRITE, 0);
1339         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1340
1341         for (i = 0; i < 5; i++)
1342                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1343
1344         WREG32(mmGMCON_PGFSM_WRITE, 0);
1345         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1346
1347         for (i = 0; i < 5; i++)
1348                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1349
1350         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1351         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1352
1353         for (i = 0; i < 5; i++)
1354                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1355
1356         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1357         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1358
1359         for (i = 0; i < 5; i++)
1360                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1361
1362         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1363         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1364
1365         for (i = 0; i < 5; i++)
1366                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1367
1368         WREG32(mmGMCON_PGFSM_WRITE, 0);
1369         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1370
1371         for (i = 0; i < 5; i++)
1372                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1373
1374         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1375         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1376
1377         for (i = 0; i < 5; i++)
1378                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1379
1380         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1381         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1382
1383         for (i = 0; i < 5; i++)
1384                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1385
1386         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1387         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1388
1389         for (i = 0; i < 5; i++)
1390                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1391
1392         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1393         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1394
1395         for (i = 0; i < 5; i++)
1396                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1397
1398         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1399         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1400
1401         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1402         WREG32(mmGMCON_MISC, save->gmcon_misc);
1403         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1404 }
1405
1406 static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1407 {
1408         struct amdgpu_mode_mc_save save;
1409         struct kv_reset_save_regs kv_save = { 0 };
1410         u32 tmp, i;
1411
1412         dev_info(adev->dev, "GPU pci config reset\n");
1413
1414         /* disable dpm? */
1415
1416         /* disable cg/pg */
1417
1418         /* Disable GFX parsing/prefetching */
1419         WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK |
1420                 CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
1421
1422         /* Disable MEC parsing/prefetching */
1423         WREG32(mmCP_MEC_CNTL,
1424                         CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
1425
1426         /* sdma0 */
1427         tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1428         tmp |= SDMA0_F32_CNTL__HALT_MASK;
1429         WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1430         /* sdma1 */
1431         tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1432         tmp |= SDMA0_F32_CNTL__HALT_MASK;
1433         WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1434         /* XXX other engines? */
1435
1436         /* halt the rlc, disable cp internal ints */
1437         gfx_v7_0_rlc_stop(adev);
1438
1439         udelay(50);
1440
1441         /* disable mem access */
1442         gmc_v7_0_mc_stop(adev, &save);
1443         if (amdgpu_asic_wait_for_mc_idle(adev)) {
1444                 dev_warn(adev->dev, "Wait for MC idle timed out !\n");
1445         }
1446
1447         if (adev->flags & AMD_IS_APU)
1448                 kv_save_regs_for_reset(adev, &kv_save);
1449
1450         /* disable BM */
1451         pci_clear_master(adev->pdev);
1452         /* reset */
1453         amdgpu_pci_config_reset(adev);
1454
1455         udelay(100);
1456
1457         /* wait for asic to come out of reset */
1458         for (i = 0; i < adev->usec_timeout; i++) {
1459                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
1460                         break;
1461                 udelay(1);
1462         }
1463
1464         /* does asic init need to be run first??? */
1465         if (adev->flags & AMD_IS_APU)
1466                 kv_restore_regs_for_reset(adev, &kv_save);
1467 }
1468
1469 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1470 {
1471         u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1472
1473         if (hung)
1474                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1475         else
1476                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1477
1478         WREG32(mmBIOS_SCRATCH_3, tmp);
1479 }
1480
1481 /**
1482  * cik_asic_reset - soft reset GPU
1483  *
1484  * @adev: amdgpu_device pointer
1485  *
1486  * Look up which blocks are hung and attempt
1487  * to reset them.
1488  * Returns 0 for success.
1489  */
1490 static int cik_asic_reset(struct amdgpu_device *adev)
1491 {
1492         u32 reset_mask;
1493
1494         reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1495
1496         if (reset_mask)
1497                 cik_set_bios_scratch_engine_hung(adev, true);
1498
1499         /* try soft reset */
1500         cik_gpu_soft_reset(adev, reset_mask);
1501
1502         reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1503
1504         /* try pci config reset */
1505         if (reset_mask && amdgpu_hard_reset)
1506                 cik_gpu_pci_config_reset(adev);
1507
1508         reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1509
1510         if (!reset_mask)
1511                 cik_set_bios_scratch_engine_hung(adev, false);
1512
1513         return 0;
1514 }
1515
1516 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1517                               u32 cntl_reg, u32 status_reg)
1518 {
1519         int r, i;
1520         struct atom_clock_dividers dividers;
1521         uint32_t tmp;
1522
1523         r = amdgpu_atombios_get_clock_dividers(adev,
1524                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1525                                                clock, false, &dividers);
1526         if (r)
1527                 return r;
1528
1529         tmp = RREG32_SMC(cntl_reg);
1530         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1531                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1532         tmp |= dividers.post_divider;
1533         WREG32_SMC(cntl_reg, tmp);
1534
1535         for (i = 0; i < 100; i++) {
1536                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1537                         break;
1538                 mdelay(10);
1539         }
1540         if (i == 100)
1541                 return -ETIMEDOUT;
1542
1543         return 0;
1544 }
1545
1546 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1547 {
1548         int r = 0;
1549
1550         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1551         if (r)
1552                 return r;
1553
1554         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1555         return r;
1556 }
1557
1558 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1559 {
1560         int r, i;
1561         struct atom_clock_dividers dividers;
1562         u32 tmp;
1563
1564         r = amdgpu_atombios_get_clock_dividers(adev,
1565                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1566                                                ecclk, false, &dividers);
1567         if (r)
1568                 return r;
1569
1570         for (i = 0; i < 100; i++) {
1571                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1572                         break;
1573                 mdelay(10);
1574         }
1575         if (i == 100)
1576                 return -ETIMEDOUT;
1577
1578         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1579         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1580                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1581         tmp |= dividers.post_divider;
1582         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1583
1584         for (i = 0; i < 100; i++) {
1585                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1586                         break;
1587                 mdelay(10);
1588         }
1589         if (i == 100)
1590                 return -ETIMEDOUT;
1591
1592         return 0;
1593 }
1594
1595 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1596 {
1597         struct pci_dev *root = adev->pdev->bus->self;
1598         int bridge_pos, gpu_pos;
1599         u32 speed_cntl, current_data_rate;
1600         int i;
1601         u16 tmp16;
1602
1603         if (pci_is_root_bus(adev->pdev->bus))
1604                 return;
1605
1606         if (amdgpu_pcie_gen2 == 0)
1607                 return;
1608
1609         if (adev->flags & AMD_IS_APU)
1610                 return;
1611
1612         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1613                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1614                 return;
1615
1616         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1617         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1618                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1619         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1620                 if (current_data_rate == 2) {
1621                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1622                         return;
1623                 }
1624                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1625         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1626                 if (current_data_rate == 1) {
1627                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1628                         return;
1629                 }
1630                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1631         }
1632
1633         bridge_pos = pci_pcie_cap(root);
1634         if (!bridge_pos)
1635                 return;
1636
1637         gpu_pos = pci_pcie_cap(adev->pdev);
1638         if (!gpu_pos)
1639                 return;
1640
1641         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1642                 /* re-try equalization if gen3 is not already enabled */
1643                 if (current_data_rate != 2) {
1644                         u16 bridge_cfg, gpu_cfg;
1645                         u16 bridge_cfg2, gpu_cfg2;
1646                         u32 max_lw, current_lw, tmp;
1647
1648                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1649                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1650
1651                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1652                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1653
1654                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1655                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1656
1657                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1658                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1659                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1660                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1661                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1662
1663                         if (current_lw < max_lw) {
1664                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1665                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1666                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1667                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1668                                         tmp |= (max_lw <<
1669                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1670                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1671                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1672                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1673                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1674                                 }
1675                         }
1676
1677                         for (i = 0; i < 10; i++) {
1678                                 /* check status */
1679                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1680                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1681                                         break;
1682
1683                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1684                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1685
1686                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1687                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1688
1689                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1690                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1691                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1692
1693                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1694                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1695                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1696
1697                                 mdelay(100);
1698
1699                                 /* linkctl */
1700                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1701                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1702                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1703                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1704
1705                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1706                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1707                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1708                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1709
1710                                 /* linkctl2 */
1711                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1712                                 tmp16 &= ~((1 << 4) | (7 << 9));
1713                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1714                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1715
1716                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1717                                 tmp16 &= ~((1 << 4) | (7 << 9));
1718                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1719                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1720
1721                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1722                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1723                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1724                         }
1725                 }
1726         }
1727
1728         /* set the link speed */
1729         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1730                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1731         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1732         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1733
1734         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1735         tmp16 &= ~0xf;
1736         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1737                 tmp16 |= 3; /* gen3 */
1738         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1739                 tmp16 |= 2; /* gen2 */
1740         else
1741                 tmp16 |= 1; /* gen1 */
1742         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1743
1744         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1745         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1746         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1747
1748         for (i = 0; i < adev->usec_timeout; i++) {
1749                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1750                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1751                         break;
1752                 udelay(1);
1753         }
1754 }
1755
1756 static void cik_program_aspm(struct amdgpu_device *adev)
1757 {
1758         u32 data, orig;
1759         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1760         bool disable_clkreq = false;
1761
1762         if (amdgpu_aspm == 0)
1763                 return;
1764
1765         /* XXX double check APUs */
1766         if (adev->flags & AMD_IS_APU)
1767                 return;
1768
1769         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1770         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1771         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1772                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1773         if (orig != data)
1774                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1775
1776         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1777         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1778         if (orig != data)
1779                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1780
1781         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1782         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1783         if (orig != data)
1784                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1785
1786         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1787         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1788                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1789         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1790         if (!disable_l0s)
1791                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1792
1793         if (!disable_l1) {
1794                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1795                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1796                 if (orig != data)
1797                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1798
1799                 if (!disable_plloff_in_l1) {
1800                         bool clk_req_support;
1801
1802                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1803                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1804                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1805                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1806                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1807                         if (orig != data)
1808                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1809
1810                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1811                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1812                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1813                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1814                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1815                         if (orig != data)
1816                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1817
1818                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1819                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1820                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1821                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1822                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1823                         if (orig != data)
1824                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1825
1826                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1827                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1828                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1829                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1830                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1831                         if (orig != data)
1832                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1833
1834                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1835                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1836                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1837                         if (orig != data)
1838                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1839
1840                         if (!disable_clkreq) {
1841                                 struct pci_dev *root = adev->pdev->bus->self;
1842                                 u32 lnkcap;
1843
1844                                 clk_req_support = false;
1845                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1846                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1847                                         clk_req_support = true;
1848                         } else {
1849                                 clk_req_support = false;
1850                         }
1851
1852                         if (clk_req_support) {
1853                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1854                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1855                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1856                                 if (orig != data)
1857                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1858
1859                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1860                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1861                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1862                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1863                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1864                                 if (orig != data)
1865                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1866
1867                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1868                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1869                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1870                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1871                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1872                                 if (orig != data)
1873                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1874
1875                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1876                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1877                                 if (orig != data)
1878                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1879
1880                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1881                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1882                                 if (orig != data)
1883                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1884
1885                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1886                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1887                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1888                                 if (orig != data)
1889                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1890                         }
1891                 }
1892         } else {
1893                 if (orig != data)
1894                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1895         }
1896
1897         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1898         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1899                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1900                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1901         if (orig != data)
1902                 WREG32_PCIE(ixPCIE_CNTL2, data);
1903
1904         if (!disable_l0s) {
1905                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1906                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1907                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1908                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1909                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1910                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1911                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1912                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1913                                 if (orig != data)
1914                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1915                         }
1916                 }
1917         }
1918 }
1919
1920 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1921 {
1922         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1923                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1924 }
1925
1926 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1927 {
1928         /* ORDER MATTERS! */
1929         {
1930                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1931                 .major = 1,
1932                 .minor = 0,
1933                 .rev = 0,
1934                 .funcs = &cik_common_ip_funcs,
1935         },
1936         {
1937                 .type = AMD_IP_BLOCK_TYPE_GMC,
1938                 .major = 7,
1939                 .minor = 0,
1940                 .rev = 0,
1941                 .funcs = &gmc_v7_0_ip_funcs,
1942         },
1943         {
1944                 .type = AMD_IP_BLOCK_TYPE_IH,
1945                 .major = 2,
1946                 .minor = 0,
1947                 .rev = 0,
1948                 .funcs = &cik_ih_ip_funcs,
1949         },
1950         {
1951                 .type = AMD_IP_BLOCK_TYPE_SMC,
1952                 .major = 7,
1953                 .minor = 0,
1954                 .rev = 0,
1955                 .funcs = &amdgpu_pp_ip_funcs,
1956         },
1957         {
1958                 .type = AMD_IP_BLOCK_TYPE_DCE,
1959                 .major = 8,
1960                 .minor = 2,
1961                 .rev = 0,
1962                 .funcs = &dce_v8_0_ip_funcs,
1963         },
1964         {
1965                 .type = AMD_IP_BLOCK_TYPE_GFX,
1966                 .major = 7,
1967                 .minor = 2,
1968                 .rev = 0,
1969                 .funcs = &gfx_v7_0_ip_funcs,
1970         },
1971         {
1972                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1973                 .major = 2,
1974                 .minor = 0,
1975                 .rev = 0,
1976                 .funcs = &cik_sdma_ip_funcs,
1977         },
1978         {
1979                 .type = AMD_IP_BLOCK_TYPE_UVD,
1980                 .major = 4,
1981                 .minor = 2,
1982                 .rev = 0,
1983                 .funcs = &uvd_v4_2_ip_funcs,
1984         },
1985         {
1986                 .type = AMD_IP_BLOCK_TYPE_VCE,
1987                 .major = 2,
1988                 .minor = 0,
1989                 .rev = 0,
1990                 .funcs = &vce_v2_0_ip_funcs,
1991         },
1992 };
1993
1994 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1995 {
1996         /* ORDER MATTERS! */
1997         {
1998                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1999                 .major = 1,
2000                 .minor = 0,
2001                 .rev = 0,
2002                 .funcs = &cik_common_ip_funcs,
2003         },
2004         {
2005                 .type = AMD_IP_BLOCK_TYPE_GMC,
2006                 .major = 7,
2007                 .minor = 0,
2008                 .rev = 0,
2009                 .funcs = &gmc_v7_0_ip_funcs,
2010         },
2011         {
2012                 .type = AMD_IP_BLOCK_TYPE_IH,
2013                 .major = 2,
2014                 .minor = 0,
2015                 .rev = 0,
2016                 .funcs = &cik_ih_ip_funcs,
2017         },
2018         {
2019                 .type = AMD_IP_BLOCK_TYPE_SMC,
2020                 .major = 7,
2021                 .minor = 0,
2022                 .rev = 0,
2023                 .funcs = &amdgpu_pp_ip_funcs,
2024         },
2025         {
2026                 .type = AMD_IP_BLOCK_TYPE_DCE,
2027                 .major = 8,
2028                 .minor = 5,
2029                 .rev = 0,
2030                 .funcs = &dce_v8_0_ip_funcs,
2031         },
2032         {
2033                 .type = AMD_IP_BLOCK_TYPE_GFX,
2034                 .major = 7,
2035                 .minor = 3,
2036                 .rev = 0,
2037                 .funcs = &gfx_v7_0_ip_funcs,
2038         },
2039         {
2040                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2041                 .major = 2,
2042                 .minor = 0,
2043                 .rev = 0,
2044                 .funcs = &cik_sdma_ip_funcs,
2045         },
2046         {
2047                 .type = AMD_IP_BLOCK_TYPE_UVD,
2048                 .major = 4,
2049                 .minor = 2,
2050                 .rev = 0,
2051                 .funcs = &uvd_v4_2_ip_funcs,
2052         },
2053         {
2054                 .type = AMD_IP_BLOCK_TYPE_VCE,
2055                 .major = 2,
2056                 .minor = 0,
2057                 .rev = 0,
2058                 .funcs = &vce_v2_0_ip_funcs,
2059         },
2060 };
2061
2062 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
2063 {
2064         /* ORDER MATTERS! */
2065         {
2066                 .type = AMD_IP_BLOCK_TYPE_COMMON,
2067                 .major = 1,
2068                 .minor = 0,
2069                 .rev = 0,
2070                 .funcs = &cik_common_ip_funcs,
2071         },
2072         {
2073                 .type = AMD_IP_BLOCK_TYPE_GMC,
2074                 .major = 7,
2075                 .minor = 0,
2076                 .rev = 0,
2077                 .funcs = &gmc_v7_0_ip_funcs,
2078         },
2079         {
2080                 .type = AMD_IP_BLOCK_TYPE_IH,
2081                 .major = 2,
2082                 .minor = 0,
2083                 .rev = 0,
2084                 .funcs = &cik_ih_ip_funcs,
2085         },
2086         {
2087                 .type = AMD_IP_BLOCK_TYPE_SMC,
2088                 .major = 7,
2089                 .minor = 0,
2090                 .rev = 0,
2091                 .funcs = &amdgpu_pp_ip_funcs,
2092         },
2093         {
2094                 .type = AMD_IP_BLOCK_TYPE_DCE,
2095                 .major = 8,
2096                 .minor = 3,
2097                 .rev = 0,
2098                 .funcs = &dce_v8_0_ip_funcs,
2099         },
2100         {
2101                 .type = AMD_IP_BLOCK_TYPE_GFX,
2102                 .major = 7,
2103                 .minor = 2,
2104                 .rev = 0,
2105                 .funcs = &gfx_v7_0_ip_funcs,
2106         },
2107         {
2108                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2109                 .major = 2,
2110                 .minor = 0,
2111                 .rev = 0,
2112                 .funcs = &cik_sdma_ip_funcs,
2113         },
2114         {
2115                 .type = AMD_IP_BLOCK_TYPE_UVD,
2116                 .major = 4,
2117                 .minor = 2,
2118                 .rev = 0,
2119                 .funcs = &uvd_v4_2_ip_funcs,
2120         },
2121         {
2122                 .type = AMD_IP_BLOCK_TYPE_VCE,
2123                 .major = 2,
2124                 .minor = 0,
2125                 .rev = 0,
2126                 .funcs = &vce_v2_0_ip_funcs,
2127         },
2128 };
2129
2130 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
2131 {
2132         /* ORDER MATTERS! */
2133         {
2134                 .type = AMD_IP_BLOCK_TYPE_COMMON,
2135                 .major = 1,
2136                 .minor = 0,
2137                 .rev = 0,
2138                 .funcs = &cik_common_ip_funcs,
2139         },
2140         {
2141                 .type = AMD_IP_BLOCK_TYPE_GMC,
2142                 .major = 7,
2143                 .minor = 0,
2144                 .rev = 0,
2145                 .funcs = &gmc_v7_0_ip_funcs,
2146         },
2147         {
2148                 .type = AMD_IP_BLOCK_TYPE_IH,
2149                 .major = 2,
2150                 .minor = 0,
2151                 .rev = 0,
2152                 .funcs = &cik_ih_ip_funcs,
2153         },
2154         {
2155                 .type = AMD_IP_BLOCK_TYPE_SMC,
2156                 .major = 7,
2157                 .minor = 0,
2158                 .rev = 0,
2159                 .funcs = &amdgpu_pp_ip_funcs,
2160         },
2161         {
2162                 .type = AMD_IP_BLOCK_TYPE_DCE,
2163                 .major = 8,
2164                 .minor = 3,
2165                 .rev = 0,
2166                 .funcs = &dce_v8_0_ip_funcs,
2167         },
2168         {
2169                 .type = AMD_IP_BLOCK_TYPE_GFX,
2170                 .major = 7,
2171                 .minor = 2,
2172                 .rev = 0,
2173                 .funcs = &gfx_v7_0_ip_funcs,
2174         },
2175         {
2176                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2177                 .major = 2,
2178                 .minor = 0,
2179                 .rev = 0,
2180                 .funcs = &cik_sdma_ip_funcs,
2181         },
2182         {
2183                 .type = AMD_IP_BLOCK_TYPE_UVD,
2184                 .major = 4,
2185                 .minor = 2,
2186                 .rev = 0,
2187                 .funcs = &uvd_v4_2_ip_funcs,
2188         },
2189         {
2190                 .type = AMD_IP_BLOCK_TYPE_VCE,
2191                 .major = 2,
2192                 .minor = 0,
2193                 .rev = 0,
2194                 .funcs = &vce_v2_0_ip_funcs,
2195         },
2196 };
2197
2198 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
2199 {
2200         /* ORDER MATTERS! */
2201         {
2202                 .type = AMD_IP_BLOCK_TYPE_COMMON,
2203                 .major = 1,
2204                 .minor = 0,
2205                 .rev = 0,
2206                 .funcs = &cik_common_ip_funcs,
2207         },
2208         {
2209                 .type = AMD_IP_BLOCK_TYPE_GMC,
2210                 .major = 7,
2211                 .minor = 0,
2212                 .rev = 0,
2213                 .funcs = &gmc_v7_0_ip_funcs,
2214         },
2215         {
2216                 .type = AMD_IP_BLOCK_TYPE_IH,
2217                 .major = 2,
2218                 .minor = 0,
2219                 .rev = 0,
2220                 .funcs = &cik_ih_ip_funcs,
2221         },
2222         {
2223                 .type = AMD_IP_BLOCK_TYPE_SMC,
2224                 .major = 7,
2225                 .minor = 0,
2226                 .rev = 0,
2227                 .funcs = &amdgpu_pp_ip_funcs,
2228         },
2229         {
2230                 .type = AMD_IP_BLOCK_TYPE_DCE,
2231                 .major = 8,
2232                 .minor = 1,
2233                 .rev = 0,
2234                 .funcs = &dce_v8_0_ip_funcs,
2235         },
2236         {
2237                 .type = AMD_IP_BLOCK_TYPE_GFX,
2238                 .major = 7,
2239                 .minor = 1,
2240                 .rev = 0,
2241                 .funcs = &gfx_v7_0_ip_funcs,
2242         },
2243         {
2244                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2245                 .major = 2,
2246                 .minor = 0,
2247                 .rev = 0,
2248                 .funcs = &cik_sdma_ip_funcs,
2249         },
2250         {
2251                 .type = AMD_IP_BLOCK_TYPE_UVD,
2252                 .major = 4,
2253                 .minor = 2,
2254                 .rev = 0,
2255                 .funcs = &uvd_v4_2_ip_funcs,
2256         },
2257         {
2258                 .type = AMD_IP_BLOCK_TYPE_VCE,
2259                 .major = 2,
2260                 .minor = 0,
2261                 .rev = 0,
2262                 .funcs = &vce_v2_0_ip_funcs,
2263         },
2264 };
2265
2266 int cik_set_ip_blocks(struct amdgpu_device *adev)
2267 {
2268         switch (adev->asic_type) {
2269         case CHIP_BONAIRE:
2270                 adev->ip_blocks = bonaire_ip_blocks;
2271                 adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
2272                 break;
2273         case CHIP_HAWAII:
2274                 adev->ip_blocks = hawaii_ip_blocks;
2275                 adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
2276                 break;
2277         case CHIP_KAVERI:
2278                 adev->ip_blocks = kaveri_ip_blocks;
2279                 adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
2280                 break;
2281         case CHIP_KABINI:
2282                 adev->ip_blocks = kabini_ip_blocks;
2283                 adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
2284                 break;
2285         case CHIP_MULLINS:
2286                 adev->ip_blocks = mullins_ip_blocks;
2287                 adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
2288                 break;
2289         default:
2290                 /* FIXME: not supported yet */
2291                 return -EINVAL;
2292         }
2293
2294         return 0;
2295 }
2296
2297 static const struct amdgpu_asic_funcs cik_asic_funcs =
2298 {
2299         .read_disabled_bios = &cik_read_disabled_bios,
2300         .read_bios_from_rom = &cik_read_bios_from_rom,
2301         .read_register = &cik_read_register,
2302         .reset = &cik_asic_reset,
2303         .set_vga_state = &cik_vga_set_state,
2304         .get_xclk = &cik_get_xclk,
2305         .set_uvd_clocks = &cik_set_uvd_clocks,
2306         .set_vce_clocks = &cik_set_vce_clocks,
2307         .get_cu_info = &gfx_v7_0_get_cu_info,
2308         /* these should be moved to their own ip modules */
2309         .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2310         .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2311 };
2312
2313 static int cik_common_early_init(void *handle)
2314 {
2315         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2316
2317         adev->smc_rreg = &cik_smc_rreg;
2318         adev->smc_wreg = &cik_smc_wreg;
2319         adev->pcie_rreg = &cik_pcie_rreg;
2320         adev->pcie_wreg = &cik_pcie_wreg;
2321         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2322         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2323         adev->didt_rreg = &cik_didt_rreg;
2324         adev->didt_wreg = &cik_didt_wreg;
2325
2326         adev->asic_funcs = &cik_asic_funcs;
2327
2328         adev->has_uvd = true;
2329
2330         adev->rev_id = cik_get_rev_id(adev);
2331         adev->external_rev_id = 0xFF;
2332         switch (adev->asic_type) {
2333         case CHIP_BONAIRE:
2334                 adev->cg_flags =
2335                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2336                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2337                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2338                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2339                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2340                         AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2341                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2342                         AMDGPU_CG_SUPPORT_MC_LS |
2343                         AMDGPU_CG_SUPPORT_MC_MGCG |
2344                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2345                         AMDGPU_CG_SUPPORT_SDMA_LS |
2346                         AMDGPU_CG_SUPPORT_BIF_LS |
2347                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2348                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2349                         AMDGPU_CG_SUPPORT_HDP_LS |
2350                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2351                 adev->pg_flags = 0;
2352                 adev->external_rev_id = adev->rev_id + 0x14;
2353                 break;
2354         case CHIP_HAWAII:
2355                 adev->cg_flags =
2356                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2357                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2358                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2359                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2360                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2361                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2362                         AMDGPU_CG_SUPPORT_MC_LS |
2363                         AMDGPU_CG_SUPPORT_MC_MGCG |
2364                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2365                         AMDGPU_CG_SUPPORT_SDMA_LS |
2366                         AMDGPU_CG_SUPPORT_BIF_LS |
2367                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2368                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2369                         AMDGPU_CG_SUPPORT_HDP_LS |
2370                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2371                 adev->pg_flags = 0;
2372                 adev->external_rev_id = 0x28;
2373                 break;
2374         case CHIP_KAVERI:
2375                 adev->cg_flags =
2376                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2377                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2378                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2379                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2380                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2381                         AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2382                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2383                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2384                         AMDGPU_CG_SUPPORT_SDMA_LS |
2385                         AMDGPU_CG_SUPPORT_BIF_LS |
2386                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2387                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2388                         AMDGPU_CG_SUPPORT_HDP_LS |
2389                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2390                 adev->pg_flags =
2391                         /*AMDGPU_PG_SUPPORT_GFX_PG |
2392                           AMDGPU_PG_SUPPORT_GFX_SMG |
2393                           AMDGPU_PG_SUPPORT_GFX_DMG |*/
2394                         AMDGPU_PG_SUPPORT_UVD |
2395                         /*AMDGPU_PG_SUPPORT_VCE |
2396                           AMDGPU_PG_SUPPORT_CP |
2397                           AMDGPU_PG_SUPPORT_GDS |
2398                           AMDGPU_PG_SUPPORT_RLC_SMU_HS |
2399                           AMDGPU_PG_SUPPORT_ACP |
2400                           AMDGPU_PG_SUPPORT_SAMU |*/
2401                         0;
2402                 if (adev->pdev->device == 0x1312 ||
2403                         adev->pdev->device == 0x1316 ||
2404                         adev->pdev->device == 0x1317)
2405                         adev->external_rev_id = 0x41;
2406                 else
2407                         adev->external_rev_id = 0x1;
2408                 break;
2409         case CHIP_KABINI:
2410         case CHIP_MULLINS:
2411                 adev->cg_flags =
2412                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2413                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2414                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2415                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2416                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2417                         AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2418                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2419                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2420                         AMDGPU_CG_SUPPORT_SDMA_LS |
2421                         AMDGPU_CG_SUPPORT_BIF_LS |
2422                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2423                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2424                         AMDGPU_CG_SUPPORT_HDP_LS |
2425                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2426                 adev->pg_flags =
2427                         /*AMDGPU_PG_SUPPORT_GFX_PG |
2428                           AMDGPU_PG_SUPPORT_GFX_SMG | */
2429                         AMDGPU_PG_SUPPORT_UVD |
2430                         /*AMDGPU_PG_SUPPORT_VCE |
2431                           AMDGPU_PG_SUPPORT_CP |
2432                           AMDGPU_PG_SUPPORT_GDS |
2433                           AMDGPU_PG_SUPPORT_RLC_SMU_HS |
2434                           AMDGPU_PG_SUPPORT_SAMU |*/
2435                         0;
2436                 if (adev->asic_type == CHIP_KABINI) {
2437                         if (adev->rev_id == 0)
2438                                 adev->external_rev_id = 0x81;
2439                         else if (adev->rev_id == 1)
2440                                 adev->external_rev_id = 0x82;
2441                         else if (adev->rev_id == 2)
2442                                 adev->external_rev_id = 0x85;
2443                 } else
2444                         adev->external_rev_id = adev->rev_id + 0xa1;
2445                 break;
2446         default:
2447                 /* FIXME: not supported yet */
2448                 return -EINVAL;
2449         }
2450
2451         amdgpu_get_pcie_info(adev);
2452
2453         return 0;
2454 }
2455
2456 static int cik_common_sw_init(void *handle)
2457 {
2458         return 0;
2459 }
2460
2461 static int cik_common_sw_fini(void *handle)
2462 {
2463         return 0;
2464 }
2465
2466 static int cik_common_hw_init(void *handle)
2467 {
2468         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2469
2470         /* move the golden regs per IP block */
2471         cik_init_golden_registers(adev);
2472         /* enable pcie gen2/3 link */
2473         cik_pcie_gen3_enable(adev);
2474         /* enable aspm */
2475         cik_program_aspm(adev);
2476
2477         return 0;
2478 }
2479
2480 static int cik_common_hw_fini(void *handle)
2481 {
2482         return 0;
2483 }
2484
2485 static int cik_common_suspend(void *handle)
2486 {
2487         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2488
2489         amdgpu_amdkfd_suspend(adev);
2490
2491         return cik_common_hw_fini(adev);
2492 }
2493
2494 static int cik_common_resume(void *handle)
2495 {
2496         int r;
2497         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2498
2499         r = cik_common_hw_init(adev);
2500         if (r)
2501                 return r;
2502
2503         return amdgpu_amdkfd_resume(adev);
2504 }
2505
2506 static bool cik_common_is_idle(void *handle)
2507 {
2508         return true;
2509 }
2510
2511 static int cik_common_wait_for_idle(void *handle)
2512 {
2513         return 0;
2514 }
2515
2516 static void cik_common_print_status(void *handle)
2517 {
2518
2519 }
2520
2521 static int cik_common_soft_reset(void *handle)
2522 {
2523         /* XXX hard reset?? */
2524         return 0;
2525 }
2526
2527 static int cik_common_set_clockgating_state(void *handle,
2528                                             enum amd_clockgating_state state)
2529 {
2530         return 0;
2531 }
2532
2533 static int cik_common_set_powergating_state(void *handle,
2534                                             enum amd_powergating_state state)
2535 {
2536         return 0;
2537 }
2538
2539 const struct amd_ip_funcs cik_common_ip_funcs = {
2540         .early_init = cik_common_early_init,
2541         .late_init = NULL,
2542         .sw_init = cik_common_sw_init,
2543         .sw_fini = cik_common_sw_fini,
2544         .hw_init = cik_common_hw_init,
2545         .hw_fini = cik_common_hw_fini,
2546         .suspend = cik_common_suspend,
2547         .resume = cik_common_resume,
2548         .is_idle = cik_common_is_idle,
2549         .wait_for_idle = cik_common_wait_for_idle,
2550         .soft_reset = cik_common_soft_reset,
2551         .print_status = cik_common_print_status,
2552         .set_clockgating_state = cik_common_set_clockgating_state,
2553         .set_powergating_state = cik_common_set_powergating_state,
2554 };