drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 static const u32 crtc_offsets[6] =
  39 {
  40         EVERGREEN_CRTC0_REGISTER_OFFSET,
  41         EVERGREEN_CRTC1_REGISTER_OFFSET,
  42         EVERGREEN_CRTC2_REGISTER_OFFSET,
  43         EVERGREEN_CRTC3_REGISTER_OFFSET,
  44         EVERGREEN_CRTC4_REGISTER_OFFSET,
  45         EVERGREEN_CRTC5_REGISTER_OFFSET
  46 };
  47
  48 #include "clearstate_evergreen.h"
  49
  50 static const u32 sumo_rlc_save_restore_register_list[] =
  51 {
  52         0x98fc,
  53         0x9830,
  54         0x9834,
  55         0x9838,
  56         0x9870,
  57         0x9874,
  58         0x8a14,
  59         0x8b24,
  60         0x8bcc,
  61         0x8b10,
  62         0x8d00,
  63         0x8d04,
  64         0x8c00,
  65         0x8c04,
  66         0x8c08,
  67         0x8c0c,
  68         0x8d8c,
  69         0x8c20,
  70         0x8c24,
  71         0x8c28,
  72         0x8c18,
  73         0x8c1c,
  74         0x8cf0,
  75         0x8e2c,
  76         0x8e38,
  77         0x8c30,
  78         0x9508,
  79         0x9688,
  80         0x9608,
  81         0x960c,
  82         0x9610,
  83         0x9614,
  84         0x88c4,
  85         0x88d4,
  86         0xa008,
  87         0x900c,
  88         0x9100,
  89         0x913c,
  90         0x98f8,
  91         0x98f4,
  92         0x9b7c,
  93         0x3f8c,
  94         0x8950,
  95         0x8954,
  96         0x8a18,
  97         0x8b28,
  98         0x9144,
  99         0x9148,
 100         0x914c,
 101         0x3f90,
 102         0x3f94,
 103         0x915c,
 104         0x9160,
 105         0x9178,
 106         0x917c,
 107         0x9180,
 108         0x918c,
 109         0x9190,
 110         0x9194,
 111         0x9198,
 112         0x919c,
 113         0x91a8,
 114         0x91ac,
 115         0x91b0,
 116         0x91b4,
 117         0x91b8,
 118         0x91c4,
 119         0x91c8,
 120         0x91cc,
 121         0x91d0,
 122         0x91d4,
 123         0x91e0,
 124         0x91e4,
 125         0x91ec,
 126         0x91f0,
 127         0x91f4,
 128         0x9200,
 129         0x9204,
 130         0x929c,
 131         0x9150,
 132         0x802c,
 133 };
 134
 135 static void evergreen_gpu_init(struct radeon_device *rdev);
 136 void evergreen_fini(struct radeon_device *rdev);
 137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 138 void evergreen_program_aspm(struct radeon_device *rdev);
 139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 140                                      int ring, u32 cp_int_cntl);
 141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 142                                    u32 status, u32 addr);
 143 void cik_init_cp_pg_table(struct radeon_device *rdev);
 144
 145 extern u32 si_get_csb_size(struct radeon_device *rdev);
 146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
 148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 149
 150 static const u32 evergreen_golden_registers[] =
 151 {
 152         0x3f90, 0xffff0000, 0xff000000,
 153         0x9148, 0xffff0000, 0xff000000,
 154         0x3f94, 0xffff0000, 0xff000000,
 155         0x914c, 0xffff0000, 0xff000000,
 156         0x9b7c, 0xffffffff, 0x00000000,
 157         0x8a14, 0xffffffff, 0x00000007,
 158         0x8b10, 0xffffffff, 0x00000000,
 159         0x960c, 0xffffffff, 0x54763210,
 160         0x88c4, 0xffffffff, 0x000000c2,
 161         0x88d4, 0xffffffff, 0x00000010,
 162         0x8974, 0xffffffff, 0x00000000,
 163         0xc78, 0x00000080, 0x00000080,
 164         0x5eb4, 0xffffffff, 0x00000002,
 165         0x5e78, 0xffffffff, 0x001000f0,
 166         0x6104, 0x01000300, 0x00000000,
 167         0x5bc0, 0x00300000, 0x00000000,
 168         0x7030, 0xffffffff, 0x00000011,
 169         0x7c30, 0xffffffff, 0x00000011,
 170         0x10830, 0xffffffff, 0x00000011,
 171         0x11430, 0xffffffff, 0x00000011,
 172         0x12030, 0xffffffff, 0x00000011,
 173         0x12c30, 0xffffffff, 0x00000011,
 174         0xd02c, 0xffffffff, 0x08421000,
 175         0x240c, 0xffffffff, 0x00000380,
 176         0x8b24, 0xffffffff, 0x00ff0fff,
 177         0x28a4c, 0x06000000, 0x06000000,
 178         0x10c, 0x00000001, 0x00000001,
 179         0x8d00, 0xffffffff, 0x100e4848,
 180         0x8d04, 0xffffffff, 0x00164745,
 181         0x8c00, 0xffffffff, 0xe4000003,
 182         0x8c04, 0xffffffff, 0x40600060,
 183         0x8c08, 0xffffffff, 0x001c001c,
 184         0x8cf0, 0xffffffff, 0x08e00620,
 185         0x8c20, 0xffffffff, 0x00800080,
 186         0x8c24, 0xffffffff, 0x00800080,
 187         0x8c18, 0xffffffff, 0x20202078,
 188         0x8c1c, 0xffffffff, 0x00001010,
 189         0x28350, 0xffffffff, 0x00000000,
 190         0xa008, 0xffffffff, 0x00010000,
 191         0x5cc, 0xffffffff, 0x00000001,
 192         0x9508, 0xffffffff, 0x00000002,
 193         0x913c, 0x0000000f, 0x0000000a
 194 };
 195
 196 static const u32 evergreen_golden_registers2[] =
 197 {
 198         0x2f4c, 0xffffffff, 0x00000000,
 199         0x54f4, 0xffffffff, 0x00000000,
 200         0x54f0, 0xffffffff, 0x00000000,
 201         0x5498, 0xffffffff, 0x00000000,
 202         0x549c, 0xffffffff, 0x00000000,
 203         0x5494, 0xffffffff, 0x00000000,
 204         0x53cc, 0xffffffff, 0x00000000,
 205         0x53c8, 0xffffffff, 0x00000000,
 206         0x53c4, 0xffffffff, 0x00000000,
 207         0x53c0, 0xffffffff, 0x00000000,
 208         0x53bc, 0xffffffff, 0x00000000,
 209         0x53b8, 0xffffffff, 0x00000000,
 210         0x53b4, 0xffffffff, 0x00000000,
 211         0x53b0, 0xffffffff, 0x00000000
 212 };
 213
 214 static const u32 cypress_mgcg_init[] =
 215 {
 216         0x802c, 0xffffffff, 0xc0000000,
 217         0x5448, 0xffffffff, 0x00000100,
 218         0x55e4, 0xffffffff, 0x00000100,
 219         0x160c, 0xffffffff, 0x00000100,
 220         0x5644, 0xffffffff, 0x00000100,
 221         0xc164, 0xffffffff, 0x00000100,
 222         0x8a18, 0xffffffff, 0x00000100,
 223         0x897c, 0xffffffff, 0x06000100,
 224         0x8b28, 0xffffffff, 0x00000100,
 225         0x9144, 0xffffffff, 0x00000100,
 226         0x9a60, 0xffffffff, 0x00000100,
 227         0x9868, 0xffffffff, 0x00000100,
 228         0x8d58, 0xffffffff, 0x00000100,
 229         0x9510, 0xffffffff, 0x00000100,
 230         0x949c, 0xffffffff, 0x00000100,
 231         0x9654, 0xffffffff, 0x00000100,
 232         0x9030, 0xffffffff, 0x00000100,
 233         0x9034, 0xffffffff, 0x00000100,
 234         0x9038, 0xffffffff, 0x00000100,
 235         0x903c, 0xffffffff, 0x00000100,
 236         0x9040, 0xffffffff, 0x00000100,
 237         0xa200, 0xffffffff, 0x00000100,
 238         0xa204, 0xffffffff, 0x00000100,
 239         0xa208, 0xffffffff, 0x00000100,
 240         0xa20c, 0xffffffff, 0x00000100,
 241         0x971c, 0xffffffff, 0x00000100,
 242         0x977c, 0xffffffff, 0x00000100,
 243         0x3f80, 0xffffffff, 0x00000100,
 244         0xa210, 0xffffffff, 0x00000100,
 245         0xa214, 0xffffffff, 0x00000100,
 246         0x4d8, 0xffffffff, 0x00000100,
 247         0x9784, 0xffffffff, 0x00000100,
 248         0x9698, 0xffffffff, 0x00000100,
 249         0x4d4, 0xffffffff, 0x00000200,
 250         0x30cc, 0xffffffff, 0x00000100,
 251         0xd0c0, 0xffffffff, 0xff000100,
 252         0x802c, 0xffffffff, 0x40000000,
 253         0x915c, 0xffffffff, 0x00010000,
 254         0x9160, 0xffffffff, 0x00030002,
 255         0x9178, 0xffffffff, 0x00070000,
 256         0x917c, 0xffffffff, 0x00030002,
 257         0x9180, 0xffffffff, 0x00050004,
 258         0x918c, 0xffffffff, 0x00010006,
 259         0x9190, 0xffffffff, 0x00090008,
 260         0x9194, 0xffffffff, 0x00070000,
 261         0x9198, 0xffffffff, 0x00030002,
 262         0x919c, 0xffffffff, 0x00050004,
 263         0x91a8, 0xffffffff, 0x00010006,
 264         0x91ac, 0xffffffff, 0x00090008,
 265         0x91b0, 0xffffffff, 0x00070000,
 266         0x91b4, 0xffffffff, 0x00030002,
 267         0x91b8, 0xffffffff, 0x00050004,
 268         0x91c4, 0xffffffff, 0x00010006,
 269         0x91c8, 0xffffffff, 0x00090008,
 270         0x91cc, 0xffffffff, 0x00070000,
 271         0x91d0, 0xffffffff, 0x00030002,
 272         0x91d4, 0xffffffff, 0x00050004,
 273         0x91e0, 0xffffffff, 0x00010006,
 274         0x91e4, 0xffffffff, 0x00090008,
 275         0x91e8, 0xffffffff, 0x00000000,
 276         0x91ec, 0xffffffff, 0x00070000,
 277         0x91f0, 0xffffffff, 0x00030002,
 278         0x91f4, 0xffffffff, 0x00050004,
 279         0x9200, 0xffffffff, 0x00010006,
 280         0x9204, 0xffffffff, 0x00090008,
 281         0x9208, 0xffffffff, 0x00070000,
 282         0x920c, 0xffffffff, 0x00030002,
 283         0x9210, 0xffffffff, 0x00050004,
 284         0x921c, 0xffffffff, 0x00010006,
 285         0x9220, 0xffffffff, 0x00090008,
 286         0x9224, 0xffffffff, 0x00070000,
 287         0x9228, 0xffffffff, 0x00030002,
 288         0x922c, 0xffffffff, 0x00050004,
 289         0x9238, 0xffffffff, 0x00010006,
 290         0x923c, 0xffffffff, 0x00090008,
 291         0x9240, 0xffffffff, 0x00070000,
 292         0x9244, 0xffffffff, 0x00030002,
 293         0x9248, 0xffffffff, 0x00050004,
 294         0x9254, 0xffffffff, 0x00010006,
 295         0x9258, 0xffffffff, 0x00090008,
 296         0x925c, 0xffffffff, 0x00070000,
 297         0x9260, 0xffffffff, 0x00030002,
 298         0x9264, 0xffffffff, 0x00050004,
 299         0x9270, 0xffffffff, 0x00010006,
 300         0x9274, 0xffffffff, 0x00090008,
 301         0x9278, 0xffffffff, 0x00070000,
 302         0x927c, 0xffffffff, 0x00030002,
 303         0x9280, 0xffffffff, 0x00050004,
 304         0x928c, 0xffffffff, 0x00010006,
 305         0x9290, 0xffffffff, 0x00090008,
 306         0x9294, 0xffffffff, 0x00000000,
 307         0x929c, 0xffffffff, 0x00000001,
 308         0x802c, 0xffffffff, 0x40010000,
 309         0x915c, 0xffffffff, 0x00010000,
 310         0x9160, 0xffffffff, 0x00030002,
 311         0x9178, 0xffffffff, 0x00070000,
 312         0x917c, 0xffffffff, 0x00030002,
 313         0x9180, 0xffffffff, 0x00050004,
 314         0x918c, 0xffffffff, 0x00010006,
 315         0x9190, 0xffffffff, 0x00090008,
 316         0x9194, 0xffffffff, 0x00070000,
 317         0x9198, 0xffffffff, 0x00030002,
 318         0x919c, 0xffffffff, 0x00050004,
 319         0x91a8, 0xffffffff, 0x00010006,
 320         0x91ac, 0xffffffff, 0x00090008,
 321         0x91b0, 0xffffffff, 0x00070000,
 322         0x91b4, 0xffffffff, 0x00030002,
 323         0x91b8, 0xffffffff, 0x00050004,
 324         0x91c4, 0xffffffff, 0x00010006,
 325         0x91c8, 0xffffffff, 0x00090008,
 326         0x91cc, 0xffffffff, 0x00070000,
 327         0x91d0, 0xffffffff, 0x00030002,
 328         0x91d4, 0xffffffff, 0x00050004,
 329         0x91e0, 0xffffffff, 0x00010006,
 330         0x91e4, 0xffffffff, 0x00090008,
 331         0x91e8, 0xffffffff, 0x00000000,
 332         0x91ec, 0xffffffff, 0x00070000,
 333         0x91f0, 0xffffffff, 0x00030002,
 334         0x91f4, 0xffffffff, 0x00050004,
 335         0x9200, 0xffffffff, 0x00010006,
 336         0x9204, 0xffffffff, 0x00090008,
 337         0x9208, 0xffffffff, 0x00070000,
 338         0x920c, 0xffffffff, 0x00030002,
 339         0x9210, 0xffffffff, 0x00050004,
 340         0x921c, 0xffffffff, 0x00010006,
 341         0x9220, 0xffffffff, 0x00090008,
 342         0x9224, 0xffffffff, 0x00070000,
 343         0x9228, 0xffffffff, 0x00030002,
 344         0x922c, 0xffffffff, 0x00050004,
 345         0x9238, 0xffffffff, 0x00010006,
 346         0x923c, 0xffffffff, 0x00090008,
 347         0x9240, 0xffffffff, 0x00070000,
 348         0x9244, 0xffffffff, 0x00030002,
 349         0x9248, 0xffffffff, 0x00050004,
 350         0x9254, 0xffffffff, 0x00010006,
 351         0x9258, 0xffffffff, 0x00090008,
 352         0x925c, 0xffffffff, 0x00070000,
 353         0x9260, 0xffffffff, 0x00030002,
 354         0x9264, 0xffffffff, 0x00050004,
 355         0x9270, 0xffffffff, 0x00010006,
 356         0x9274, 0xffffffff, 0x00090008,
 357         0x9278, 0xffffffff, 0x00070000,
 358         0x927c, 0xffffffff, 0x00030002,
 359         0x9280, 0xffffffff, 0x00050004,
 360         0x928c, 0xffffffff, 0x00010006,
 361         0x9290, 0xffffffff, 0x00090008,
 362         0x9294, 0xffffffff, 0x00000000,
 363         0x929c, 0xffffffff, 0x00000001,
 364         0x802c, 0xffffffff, 0xc0000000
 365 };
 366
 367 static const u32 redwood_mgcg_init[] =
 368 {
 369         0x802c, 0xffffffff, 0xc0000000,
 370         0x5448, 0xffffffff, 0x00000100,
 371         0x55e4, 0xffffffff, 0x00000100,
 372         0x160c, 0xffffffff, 0x00000100,
 373         0x5644, 0xffffffff, 0x00000100,
 374         0xc164, 0xffffffff, 0x00000100,
 375         0x8a18, 0xffffffff, 0x00000100,
 376         0x897c, 0xffffffff, 0x06000100,
 377         0x8b28, 0xffffffff, 0x00000100,
 378         0x9144, 0xffffffff, 0x00000100,
 379         0x9a60, 0xffffffff, 0x00000100,
 380         0x9868, 0xffffffff, 0x00000100,
 381         0x8d58, 0xffffffff, 0x00000100,
 382         0x9510, 0xffffffff, 0x00000100,
 383         0x949c, 0xffffffff, 0x00000100,
 384         0x9654, 0xffffffff, 0x00000100,
 385         0x9030, 0xffffffff, 0x00000100,
 386         0x9034, 0xffffffff, 0x00000100,
 387         0x9038, 0xffffffff, 0x00000100,
 388         0x903c, 0xffffffff, 0x00000100,
 389         0x9040, 0xffffffff, 0x00000100,
 390         0xa200, 0xffffffff, 0x00000100,
 391         0xa204, 0xffffffff, 0x00000100,
 392         0xa208, 0xffffffff, 0x00000100,
 393         0xa20c, 0xffffffff, 0x00000100,
 394         0x971c, 0xffffffff, 0x00000100,
 395         0x977c, 0xffffffff, 0x00000100,
 396         0x3f80, 0xffffffff, 0x00000100,
 397         0xa210, 0xffffffff, 0x00000100,
 398         0xa214, 0xffffffff, 0x00000100,
 399         0x4d8, 0xffffffff, 0x00000100,
 400         0x9784, 0xffffffff, 0x00000100,
 401         0x9698, 0xffffffff, 0x00000100,
 402         0x4d4, 0xffffffff, 0x00000200,
 403         0x30cc, 0xffffffff, 0x00000100,
 404         0xd0c0, 0xffffffff, 0xff000100,
 405         0x802c, 0xffffffff, 0x40000000,
 406         0x915c, 0xffffffff, 0x00010000,
 407         0x9160, 0xffffffff, 0x00030002,
 408         0x9178, 0xffffffff, 0x00070000,
 409         0x917c, 0xffffffff, 0x00030002,
 410         0x9180, 0xffffffff, 0x00050004,
 411         0x918c, 0xffffffff, 0x00010006,
 412         0x9190, 0xffffffff, 0x00090008,
 413         0x9194, 0xffffffff, 0x00070000,
 414         0x9198, 0xffffffff, 0x00030002,
 415         0x919c, 0xffffffff, 0x00050004,
 416         0x91a8, 0xffffffff, 0x00010006,
 417         0x91ac, 0xffffffff, 0x00090008,
 418         0x91b0, 0xffffffff, 0x00070000,
 419         0x91b4, 0xffffffff, 0x00030002,
 420         0x91b8, 0xffffffff, 0x00050004,
 421         0x91c4, 0xffffffff, 0x00010006,
 422         0x91c8, 0xffffffff, 0x00090008,
 423         0x91cc, 0xffffffff, 0x00070000,
 424         0x91d0, 0xffffffff, 0x00030002,
 425         0x91d4, 0xffffffff, 0x00050004,
 426         0x91e0, 0xffffffff, 0x00010006,
 427         0x91e4, 0xffffffff, 0x00090008,
 428         0x91e8, 0xffffffff, 0x00000000,
 429         0x91ec, 0xffffffff, 0x00070000,
 430         0x91f0, 0xffffffff, 0x00030002,
 431         0x91f4, 0xffffffff, 0x00050004,
 432         0x9200, 0xffffffff, 0x00010006,
 433         0x9204, 0xffffffff, 0x00090008,
 434         0x9294, 0xffffffff, 0x00000000,
 435         0x929c, 0xffffffff, 0x00000001,
 436         0x802c, 0xffffffff, 0xc0000000
 437 };
 438
 439 static const u32 cedar_golden_registers[] =
 440 {
 441         0x3f90, 0xffff0000, 0xff000000,
 442         0x9148, 0xffff0000, 0xff000000,
 443         0x3f94, 0xffff0000, 0xff000000,
 444         0x914c, 0xffff0000, 0xff000000,
 445         0x9b7c, 0xffffffff, 0x00000000,
 446         0x8a14, 0xffffffff, 0x00000007,
 447         0x8b10, 0xffffffff, 0x00000000,
 448         0x960c, 0xffffffff, 0x54763210,
 449         0x88c4, 0xffffffff, 0x000000c2,
 450         0x88d4, 0xffffffff, 0x00000000,
 451         0x8974, 0xffffffff, 0x00000000,
 452         0xc78, 0x00000080, 0x00000080,
 453         0x5eb4, 0xffffffff, 0x00000002,
 454         0x5e78, 0xffffffff, 0x001000f0,
 455         0x6104, 0x01000300, 0x00000000,
 456         0x5bc0, 0x00300000, 0x00000000,
 457         0x7030, 0xffffffff, 0x00000011,
 458         0x7c30, 0xffffffff, 0x00000011,
 459         0x10830, 0xffffffff, 0x00000011,
 460         0x11430, 0xffffffff, 0x00000011,
 461         0xd02c, 0xffffffff, 0x08421000,
 462         0x240c, 0xffffffff, 0x00000380,
 463         0x8b24, 0xffffffff, 0x00ff0fff,
 464         0x28a4c, 0x06000000, 0x06000000,
 465         0x10c, 0x00000001, 0x00000001,
 466         0x8d00, 0xffffffff, 0x100e4848,
 467         0x8d04, 0xffffffff, 0x00164745,
 468         0x8c00, 0xffffffff, 0xe4000003,
 469         0x8c04, 0xffffffff, 0x40600060,
 470         0x8c08, 0xffffffff, 0x001c001c,
 471         0x8cf0, 0xffffffff, 0x08e00410,
 472         0x8c20, 0xffffffff, 0x00800080,
 473         0x8c24, 0xffffffff, 0x00800080,
 474         0x8c18, 0xffffffff, 0x20202078,
 475         0x8c1c, 0xffffffff, 0x00001010,
 476         0x28350, 0xffffffff, 0x00000000,
 477         0xa008, 0xffffffff, 0x00010000,
 478         0x5cc, 0xffffffff, 0x00000001,
 479         0x9508, 0xffffffff, 0x00000002
 480 };
 481
 482 static const u32 cedar_mgcg_init[] =
 483 {
 484         0x802c, 0xffffffff, 0xc0000000,
 485         0x5448, 0xffffffff, 0x00000100,
 486         0x55e4, 0xffffffff, 0x00000100,
 487         0x160c, 0xffffffff, 0x00000100,
 488         0x5644, 0xffffffff, 0x00000100,
 489         0xc164, 0xffffffff, 0x00000100,
 490         0x8a18, 0xffffffff, 0x00000100,
 491         0x897c, 0xffffffff, 0x06000100,
 492         0x8b28, 0xffffffff, 0x00000100,
 493         0x9144, 0xffffffff, 0x00000100,
 494         0x9a60, 0xffffffff, 0x00000100,
 495         0x9868, 0xffffffff, 0x00000100,
 496         0x8d58, 0xffffffff, 0x00000100,
 497         0x9510, 0xffffffff, 0x00000100,
 498         0x949c, 0xffffffff, 0x00000100,
 499         0x9654, 0xffffffff, 0x00000100,
 500         0x9030, 0xffffffff, 0x00000100,
 501         0x9034, 0xffffffff, 0x00000100,
 502         0x9038, 0xffffffff, 0x00000100,
 503         0x903c, 0xffffffff, 0x00000100,
 504         0x9040, 0xffffffff, 0x00000100,
 505         0xa200, 0xffffffff, 0x00000100,
 506         0xa204, 0xffffffff, 0x00000100,
 507         0xa208, 0xffffffff, 0x00000100,
 508         0xa20c, 0xffffffff, 0x00000100,
 509         0x971c, 0xffffffff, 0x00000100,
 510         0x977c, 0xffffffff, 0x00000100,
 511         0x3f80, 0xffffffff, 0x00000100,
 512         0xa210, 0xffffffff, 0x00000100,
 513         0xa214, 0xffffffff, 0x00000100,
 514         0x4d8, 0xffffffff, 0x00000100,
 515         0x9784, 0xffffffff, 0x00000100,
 516         0x9698, 0xffffffff, 0x00000100,
 517         0x4d4, 0xffffffff, 0x00000200,
 518         0x30cc, 0xffffffff, 0x00000100,
 519         0xd0c0, 0xffffffff, 0xff000100,
 520         0x802c, 0xffffffff, 0x40000000,
 521         0x915c, 0xffffffff, 0x00010000,
 522         0x9178, 0xffffffff, 0x00050000,
 523         0x917c, 0xffffffff, 0x00030002,
 524         0x918c, 0xffffffff, 0x00010004,
 525         0x9190, 0xffffffff, 0x00070006,
 526         0x9194, 0xffffffff, 0x00050000,
 527         0x9198, 0xffffffff, 0x00030002,
 528         0x91a8, 0xffffffff, 0x00010004,
 529         0x91ac, 0xffffffff, 0x00070006,
 530         0x91e8, 0xffffffff, 0x00000000,
 531         0x9294, 0xffffffff, 0x00000000,
 532         0x929c, 0xffffffff, 0x00000001,
 533         0x802c, 0xffffffff, 0xc0000000
 534 };
 535
 536 static const u32 juniper_mgcg_init[] =
 537 {
 538         0x802c, 0xffffffff, 0xc0000000,
 539         0x5448, 0xffffffff, 0x00000100,
 540         0x55e4, 0xffffffff, 0x00000100,
 541         0x160c, 0xffffffff, 0x00000100,
 542         0x5644, 0xffffffff, 0x00000100,
 543         0xc164, 0xffffffff, 0x00000100,
 544         0x8a18, 0xffffffff, 0x00000100,
 545         0x897c, 0xffffffff, 0x06000100,
 546         0x8b28, 0xffffffff, 0x00000100,
 547         0x9144, 0xffffffff, 0x00000100,
 548         0x9a60, 0xffffffff, 0x00000100,
 549         0x9868, 0xffffffff, 0x00000100,
 550         0x8d58, 0xffffffff, 0x00000100,
 551         0x9510, 0xffffffff, 0x00000100,
 552         0x949c, 0xffffffff, 0x00000100,
 553         0x9654, 0xffffffff, 0x00000100,
 554         0x9030, 0xffffffff, 0x00000100,
 555         0x9034, 0xffffffff, 0x00000100,
 556         0x9038, 0xffffffff, 0x00000100,
 557         0x903c, 0xffffffff, 0x00000100,
 558         0x9040, 0xffffffff, 0x00000100,
 559         0xa200, 0xffffffff, 0x00000100,
 560         0xa204, 0xffffffff, 0x00000100,
 561         0xa208, 0xffffffff, 0x00000100,
 562         0xa20c, 0xffffffff, 0x00000100,
 563         0x971c, 0xffffffff, 0x00000100,
 564         0xd0c0, 0xffffffff, 0xff000100,
 565         0x802c, 0xffffffff, 0x40000000,
 566         0x915c, 0xffffffff, 0x00010000,
 567         0x9160, 0xffffffff, 0x00030002,
 568         0x9178, 0xffffffff, 0x00070000,
 569         0x917c, 0xffffffff, 0x00030002,
 570         0x9180, 0xffffffff, 0x00050004,
 571         0x918c, 0xffffffff, 0x00010006,
 572         0x9190, 0xffffffff, 0x00090008,
 573         0x9194, 0xffffffff, 0x00070000,
 574         0x9198, 0xffffffff, 0x00030002,
 575         0x919c, 0xffffffff, 0x00050004,
 576         0x91a8, 0xffffffff, 0x00010006,
 577         0x91ac, 0xffffffff, 0x00090008,
 578         0x91b0, 0xffffffff, 0x00070000,
 579         0x91b4, 0xffffffff, 0x00030002,
 580         0x91b8, 0xffffffff, 0x00050004,
 581         0x91c4, 0xffffffff, 0x00010006,
 582         0x91c8, 0xffffffff, 0x00090008,
 583         0x91cc, 0xffffffff, 0x00070000,
 584         0x91d0, 0xffffffff, 0x00030002,
 585         0x91d4, 0xffffffff, 0x00050004,
 586         0x91e0, 0xffffffff, 0x00010006,
 587         0x91e4, 0xffffffff, 0x00090008,
 588         0x91e8, 0xffffffff, 0x00000000,
 589         0x91ec, 0xffffffff, 0x00070000,
 590         0x91f0, 0xffffffff, 0x00030002,
 591         0x91f4, 0xffffffff, 0x00050004,
 592         0x9200, 0xffffffff, 0x00010006,
 593         0x9204, 0xffffffff, 0x00090008,
 594         0x9208, 0xffffffff, 0x00070000,
 595         0x920c, 0xffffffff, 0x00030002,
 596         0x9210, 0xffffffff, 0x00050004,
 597         0x921c, 0xffffffff, 0x00010006,
 598         0x9220, 0xffffffff, 0x00090008,
 599         0x9224, 0xffffffff, 0x00070000,
 600         0x9228, 0xffffffff, 0x00030002,
 601         0x922c, 0xffffffff, 0x00050004,
 602         0x9238, 0xffffffff, 0x00010006,
 603         0x923c, 0xffffffff, 0x00090008,
 604         0x9240, 0xffffffff, 0x00070000,
 605         0x9244, 0xffffffff, 0x00030002,
 606         0x9248, 0xffffffff, 0x00050004,
 607         0x9254, 0xffffffff, 0x00010006,
 608         0x9258, 0xffffffff, 0x00090008,
 609         0x925c, 0xffffffff, 0x00070000,
 610         0x9260, 0xffffffff, 0x00030002,
 611         0x9264, 0xffffffff, 0x00050004,
 612         0x9270, 0xffffffff, 0x00010006,
 613         0x9274, 0xffffffff, 0x00090008,
 614         0x9278, 0xffffffff, 0x00070000,
 615         0x927c, 0xffffffff, 0x00030002,
 616         0x9280, 0xffffffff, 0x00050004,
 617         0x928c, 0xffffffff, 0x00010006,
 618         0x9290, 0xffffffff, 0x00090008,
 619         0x9294, 0xffffffff, 0x00000000,
 620         0x929c, 0xffffffff, 0x00000001,
 621         0x802c, 0xffffffff, 0xc0000000,
 622         0x977c, 0xffffffff, 0x00000100,
 623         0x3f80, 0xffffffff, 0x00000100,
 624         0xa210, 0xffffffff, 0x00000100,
 625         0xa214, 0xffffffff, 0x00000100,
 626         0x4d8, 0xffffffff, 0x00000100,
 627         0x9784, 0xffffffff, 0x00000100,
 628         0x9698, 0xffffffff, 0x00000100,
 629         0x4d4, 0xffffffff, 0x00000200,
 630         0x30cc, 0xffffffff, 0x00000100,
 631         0x802c, 0xffffffff, 0xc0000000
 632 };
 633
 634 static const u32 supersumo_golden_registers[] =
 635 {
 636         0x5eb4, 0xffffffff, 0x00000002,
 637         0x5cc, 0xffffffff, 0x00000001,
 638         0x7030, 0xffffffff, 0x00000011,
 639         0x7c30, 0xffffffff, 0x00000011,
 640         0x6104, 0x01000300, 0x00000000,
 641         0x5bc0, 0x00300000, 0x00000000,
 642         0x8c04, 0xffffffff, 0x40600060,
 643         0x8c08, 0xffffffff, 0x001c001c,
 644         0x8c20, 0xffffffff, 0x00800080,
 645         0x8c24, 0xffffffff, 0x00800080,
 646         0x8c18, 0xffffffff, 0x20202078,
 647         0x8c1c, 0xffffffff, 0x00001010,
 648         0x918c, 0xffffffff, 0x00010006,
 649         0x91a8, 0xffffffff, 0x00010006,
 650         0x91c4, 0xffffffff, 0x00010006,
 651         0x91e0, 0xffffffff, 0x00010006,
 652         0x9200, 0xffffffff, 0x00010006,
 653         0x9150, 0xffffffff, 0x6e944040,
 654         0x917c, 0xffffffff, 0x00030002,
 655         0x9180, 0xffffffff, 0x00050004,
 656         0x9198, 0xffffffff, 0x00030002,
 657         0x919c, 0xffffffff, 0x00050004,
 658         0x91b4, 0xffffffff, 0x00030002,
 659         0x91b8, 0xffffffff, 0x00050004,
 660         0x91d0, 0xffffffff, 0x00030002,
 661         0x91d4, 0xffffffff, 0x00050004,
 662         0x91f0, 0xffffffff, 0x00030002,
 663         0x91f4, 0xffffffff, 0x00050004,
 664         0x915c, 0xffffffff, 0x00010000,
 665         0x9160, 0xffffffff, 0x00030002,
 666         0x3f90, 0xffff0000, 0xff000000,
 667         0x9178, 0xffffffff, 0x00070000,
 668         0x9194, 0xffffffff, 0x00070000,
 669         0x91b0, 0xffffffff, 0x00070000,
 670         0x91cc, 0xffffffff, 0x00070000,
 671         0x91ec, 0xffffffff, 0x00070000,
 672         0x9148, 0xffff0000, 0xff000000,
 673         0x9190, 0xffffffff, 0x00090008,
 674         0x91ac, 0xffffffff, 0x00090008,
 675         0x91c8, 0xffffffff, 0x00090008,
 676         0x91e4, 0xffffffff, 0x00090008,
 677         0x9204, 0xffffffff, 0x00090008,
 678         0x3f94, 0xffff0000, 0xff000000,
 679         0x914c, 0xffff0000, 0xff000000,
 680         0x929c, 0xffffffff, 0x00000001,
 681         0x8a18, 0xffffffff, 0x00000100,
 682         0x8b28, 0xffffffff, 0x00000100,
 683         0x9144, 0xffffffff, 0x00000100,
 684         0x5644, 0xffffffff, 0x00000100,
 685         0x9b7c, 0xffffffff, 0x00000000,
 686         0x8030, 0xffffffff, 0x0000100a,
 687         0x8a14, 0xffffffff, 0x00000007,
 688         0x8b24, 0xffffffff, 0x00ff0fff,
 689         0x8b10, 0xffffffff, 0x00000000,
 690         0x28a4c, 0x06000000, 0x06000000,
 691         0x4d8, 0xffffffff, 0x00000100,
 692         0x913c, 0xffff000f, 0x0100000a,
 693         0x960c, 0xffffffff, 0x54763210,
 694         0x88c4, 0xffffffff, 0x000000c2,
 695         0x88d4, 0xffffffff, 0x00000010,
 696         0x8974, 0xffffffff, 0x00000000,
 697         0xc78, 0x00000080, 0x00000080,
 698         0x5e78, 0xffffffff, 0x001000f0,
 699         0xd02c, 0xffffffff, 0x08421000,
 700         0xa008, 0xffffffff, 0x00010000,
 701         0x8d00, 0xffffffff, 0x100e4848,
 702         0x8d04, 0xffffffff, 0x00164745,
 703         0x8c00, 0xffffffff, 0xe4000003,
 704         0x8cf0, 0x1fffffff, 0x08e00620,
 705         0x28350, 0xffffffff, 0x00000000,
 706         0x9508, 0xffffffff, 0x00000002
 707 };
 708
 709 static const u32 sumo_golden_registers[] =
 710 {
 711         0x900c, 0x00ffffff, 0x0017071f,
 712         0x8c18, 0xffffffff, 0x10101060,
 713         0x8c1c, 0xffffffff, 0x00001010,
 714         0x8c30, 0x0000000f, 0x00000005,
 715         0x9688, 0x0000000f, 0x00000007
 716 };
 717
 718 static const u32 wrestler_golden_registers[] =
 719 {
 720         0x5eb4, 0xffffffff, 0x00000002,
 721         0x5cc, 0xffffffff, 0x00000001,
 722         0x7030, 0xffffffff, 0x00000011,
 723         0x7c30, 0xffffffff, 0x00000011,
 724         0x6104, 0x01000300, 0x00000000,
 725         0x5bc0, 0x00300000, 0x00000000,
 726         0x918c, 0xffffffff, 0x00010006,
 727         0x91a8, 0xffffffff, 0x00010006,
 728         0x9150, 0xffffffff, 0x6e944040,
 729         0x917c, 0xffffffff, 0x00030002,
 730         0x9198, 0xffffffff, 0x00030002,
 731         0x915c, 0xffffffff, 0x00010000,
 732         0x3f90, 0xffff0000, 0xff000000,
 733         0x9178, 0xffffffff, 0x00070000,
 734         0x9194, 0xffffffff, 0x00070000,
 735         0x9148, 0xffff0000, 0xff000000,
 736         0x9190, 0xffffffff, 0x00090008,
 737         0x91ac, 0xffffffff, 0x00090008,
 738         0x3f94, 0xffff0000, 0xff000000,
 739         0x914c, 0xffff0000, 0xff000000,
 740         0x929c, 0xffffffff, 0x00000001,
 741         0x8a18, 0xffffffff, 0x00000100,
 742         0x8b28, 0xffffffff, 0x00000100,
 743         0x9144, 0xffffffff, 0x00000100,
 744         0x9b7c, 0xffffffff, 0x00000000,
 745         0x8030, 0xffffffff, 0x0000100a,
 746         0x8a14, 0xffffffff, 0x00000001,
 747         0x8b24, 0xffffffff, 0x00ff0fff,
 748         0x8b10, 0xffffffff, 0x00000000,
 749         0x28a4c, 0x06000000, 0x06000000,
 750         0x4d8, 0xffffffff, 0x00000100,
 751         0x913c, 0xffff000f, 0x0100000a,
 752         0x960c, 0xffffffff, 0x54763210,
 753         0x88c4, 0xffffffff, 0x000000c2,
 754         0x88d4, 0xffffffff, 0x00000010,
 755         0x8974, 0xffffffff, 0x00000000,
 756         0xc78, 0x00000080, 0x00000080,
 757         0x5e78, 0xffffffff, 0x001000f0,
 758         0xd02c, 0xffffffff, 0x08421000,
 759         0xa008, 0xffffffff, 0x00010000,
 760         0x8d00, 0xffffffff, 0x100e4848,
 761         0x8d04, 0xffffffff, 0x00164745,
 762         0x8c00, 0xffffffff, 0xe4000003,
 763         0x8cf0, 0x1fffffff, 0x08e00410,
 764         0x28350, 0xffffffff, 0x00000000,
 765         0x9508, 0xffffffff, 0x00000002,
 766         0x900c, 0xffffffff, 0x0017071f,
 767         0x8c18, 0xffffffff, 0x10101060,
 768         0x8c1c, 0xffffffff, 0x00001010
 769 };
 770
 771 static const u32 barts_golden_registers[] =
 772 {
 773         0x5eb4, 0xffffffff, 0x00000002,
 774         0x5e78, 0x8f311ff1, 0x001000f0,
 775         0x3f90, 0xffff0000, 0xff000000,
 776         0x9148, 0xffff0000, 0xff000000,
 777         0x3f94, 0xffff0000, 0xff000000,
 778         0x914c, 0xffff0000, 0xff000000,
 779         0xc78, 0x00000080, 0x00000080,
 780         0xbd4, 0x70073777, 0x00010001,
 781         0xd02c, 0xbfffff1f, 0x08421000,
 782         0xd0b8, 0x03773777, 0x02011003,
 783         0x5bc0, 0x00200000, 0x50100000,
 784         0x98f8, 0x33773777, 0x02011003,
 785         0x98fc, 0xffffffff, 0x76543210,
 786         0x7030, 0x31000311, 0x00000011,
 787         0x2f48, 0x00000007, 0x02011003,
 788         0x6b28, 0x00000010, 0x00000012,
 789         0x7728, 0x00000010, 0x00000012,
 790         0x10328, 0x00000010, 0x00000012,
 791         0x10f28, 0x00000010, 0x00000012,
 792         0x11b28, 0x00000010, 0x00000012,
 793         0x12728, 0x00000010, 0x00000012,
 794         0x240c, 0x000007ff, 0x00000380,
 795         0x8a14, 0xf000001f, 0x00000007,
 796         0x8b24, 0x3fff3fff, 0x00ff0fff,
 797         0x8b10, 0x0000ff0f, 0x00000000,
 798         0x28a4c, 0x07ffffff, 0x06000000,
 799         0x10c, 0x00000001, 0x00010003,
 800         0xa02c, 0xffffffff, 0x0000009b,
 801         0x913c, 0x0000000f, 0x0100000a,
 802         0x8d00, 0xffff7f7f, 0x100e4848,
 803         0x8d04, 0x00ffffff, 0x00164745,
 804         0x8c00, 0xfffc0003, 0xe4000003,
 805         0x8c04, 0xf8ff00ff, 0x40600060,
 806         0x8c08, 0x00ff00ff, 0x001c001c,
 807         0x8cf0, 0x1fff1fff, 0x08e00620,
 808         0x8c20, 0x0fff0fff, 0x00800080,
 809         0x8c24, 0x0fff0fff, 0x00800080,
 810         0x8c18, 0xffffffff, 0x20202078,
 811         0x8c1c, 0x0000ffff, 0x00001010,
 812         0x28350, 0x00000f01, 0x00000000,
 813         0x9508, 0x3700001f, 0x00000002,
 814         0x960c, 0xffffffff, 0x54763210,
 815         0x88c4, 0x001f3ae3, 0x000000c2,
 816         0x88d4, 0x0000001f, 0x00000010,
 817         0x8974, 0xffffffff, 0x00000000
 818 };
 819
 820 static const u32 turks_golden_registers[] =
 821 {
 822         0x5eb4, 0xffffffff, 0x00000002,
 823         0x5e78, 0x8f311ff1, 0x001000f0,
 824         0x8c8, 0x00003000, 0x00001070,
 825         0x8cc, 0x000fffff, 0x00040035,
 826         0x3f90, 0xffff0000, 0xfff00000,
 827         0x9148, 0xffff0000, 0xfff00000,
 828         0x3f94, 0xffff0000, 0xfff00000,
 829         0x914c, 0xffff0000, 0xfff00000,
 830         0xc78, 0x00000080, 0x00000080,
 831         0xbd4, 0x00073007, 0x00010002,
 832         0xd02c, 0xbfffff1f, 0x08421000,
 833         0xd0b8, 0x03773777, 0x02010002,
 834         0x5bc0, 0x00200000, 0x50100000,
 835         0x98f8, 0x33773777, 0x00010002,
 836         0x98fc, 0xffffffff, 0x33221100,
 837         0x7030, 0x31000311, 0x00000011,
 838         0x2f48, 0x33773777, 0x00010002,
 839         0x6b28, 0x00000010, 0x00000012,
 840         0x7728, 0x00000010, 0x00000012,
 841         0x10328, 0x00000010, 0x00000012,
 842         0x10f28, 0x00000010, 0x00000012,
 843         0x11b28, 0x00000010, 0x00000012,
 844         0x12728, 0x00000010, 0x00000012,
 845         0x240c, 0x000007ff, 0x00000380,
 846         0x8a14, 0xf000001f, 0x00000007,
 847         0x8b24, 0x3fff3fff, 0x00ff0fff,
 848         0x8b10, 0x0000ff0f, 0x00000000,
 849         0x28a4c, 0x07ffffff, 0x06000000,
 850         0x10c, 0x00000001, 0x00010003,
 851         0xa02c, 0xffffffff, 0x0000009b,
 852         0x913c, 0x0000000f, 0x0100000a,
 853         0x8d00, 0xffff7f7f, 0x100e4848,
 854         0x8d04, 0x00ffffff, 0x00164745,
 855         0x8c00, 0xfffc0003, 0xe4000003,
 856         0x8c04, 0xf8ff00ff, 0x40600060,
 857         0x8c08, 0x00ff00ff, 0x001c001c,
 858         0x8cf0, 0x1fff1fff, 0x08e00410,
 859         0x8c20, 0x0fff0fff, 0x00800080,
 860         0x8c24, 0x0fff0fff, 0x00800080,
 861         0x8c18, 0xffffffff, 0x20202078,
 862         0x8c1c, 0x0000ffff, 0x00001010,
 863         0x28350, 0x00000f01, 0x00000000,
 864         0x9508, 0x3700001f, 0x00000002,
 865         0x960c, 0xffffffff, 0x54763210,
 866         0x88c4, 0x001f3ae3, 0x000000c2,
 867         0x88d4, 0x0000001f, 0x00000010,
 868         0x8974, 0xffffffff, 0x00000000
 869 };
 870
 871 static const u32 caicos_golden_registers[] =
 872 {
 873         0x5eb4, 0xffffffff, 0x00000002,
 874         0x5e78, 0x8f311ff1, 0x001000f0,
 875         0x8c8, 0x00003420, 0x00001450,
 876         0x8cc, 0x000fffff, 0x00040035,
 877         0x3f90, 0xffff0000, 0xfffc0000,
 878         0x9148, 0xffff0000, 0xfffc0000,
 879         0x3f94, 0xffff0000, 0xfffc0000,
 880         0x914c, 0xffff0000, 0xfffc0000,
 881         0xc78, 0x00000080, 0x00000080,
 882         0xbd4, 0x00073007, 0x00010001,
 883         0xd02c, 0xbfffff1f, 0x08421000,
 884         0xd0b8, 0x03773777, 0x02010001,
 885         0x5bc0, 0x00200000, 0x50100000,
 886         0x98f8, 0x33773777, 0x02010001,
 887         0x98fc, 0xffffffff, 0x33221100,
 888         0x7030, 0x31000311, 0x00000011,
 889         0x2f48, 0x33773777, 0x02010001,
 890         0x6b28, 0x00000010, 0x00000012,
 891         0x7728, 0x00000010, 0x00000012,
 892         0x10328, 0x00000010, 0x00000012,
 893         0x10f28, 0x00000010, 0x00000012,
 894         0x11b28, 0x00000010, 0x00000012,
 895         0x12728, 0x00000010, 0x00000012,
 896         0x240c, 0x000007ff, 0x00000380,
 897         0x8a14, 0xf000001f, 0x00000001,
 898         0x8b24, 0x3fff3fff, 0x00ff0fff,
 899         0x8b10, 0x0000ff0f, 0x00000000,
 900         0x28a4c, 0x07ffffff, 0x06000000,
 901         0x10c, 0x00000001, 0x00010003,
 902         0xa02c, 0xffffffff, 0x0000009b,
 903         0x913c, 0x0000000f, 0x0100000a,
 904         0x8d00, 0xffff7f7f, 0x100e4848,
 905         0x8d04, 0x00ffffff, 0x00164745,
 906         0x8c00, 0xfffc0003, 0xe4000003,
 907         0x8c04, 0xf8ff00ff, 0x40600060,
 908         0x8c08, 0x00ff00ff, 0x001c001c,
 909         0x8cf0, 0x1fff1fff, 0x08e00410,
 910         0x8c20, 0x0fff0fff, 0x00800080,
 911         0x8c24, 0x0fff0fff, 0x00800080,
 912         0x8c18, 0xffffffff, 0x20202078,
 913         0x8c1c, 0x0000ffff, 0x00001010,
 914         0x28350, 0x00000f01, 0x00000000,
 915         0x9508, 0x3700001f, 0x00000002,
 916         0x960c, 0xffffffff, 0x54763210,
 917         0x88c4, 0x001f3ae3, 0x000000c2,
 918         0x88d4, 0x0000001f, 0x00000010,
 919         0x8974, 0xffffffff, 0x00000000
 920 };
 921
 922 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 923 {
 924         switch (rdev->family) {
 925         case CHIP_CYPRESS:
 926         case CHIP_HEMLOCK:
 927                 radeon_program_register_sequence(rdev,
 928                                                  evergreen_golden_registers,
 929                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 930                 radeon_program_register_sequence(rdev,
 931                                                  evergreen_golden_registers2,
 932                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 933                 radeon_program_register_sequence(rdev,
 934                                                  cypress_mgcg_init,
 935                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
 936                 break;
 937         case CHIP_JUNIPER:
 938                 radeon_program_register_sequence(rdev,
 939                                                  evergreen_golden_registers,
 940                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 941                 radeon_program_register_sequence(rdev,
 942                                                  evergreen_golden_registers2,
 943                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 944                 radeon_program_register_sequence(rdev,
 945                                                  juniper_mgcg_init,
 946                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
 947                 break;
 948         case CHIP_REDWOOD:
 949                 radeon_program_register_sequence(rdev,
 950                                                  evergreen_golden_registers,
 951                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 952                 radeon_program_register_sequence(rdev,
 953                                                  evergreen_golden_registers2,
 954                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 955                 radeon_program_register_sequence(rdev,
 956                                                  redwood_mgcg_init,
 957                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
 958                 break;
 959         case CHIP_CEDAR:
 960                 radeon_program_register_sequence(rdev,
 961                                                  cedar_golden_registers,
 962                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
 963                 radeon_program_register_sequence(rdev,
 964                                                  evergreen_golden_registers2,
 965                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 966                 radeon_program_register_sequence(rdev,
 967                                                  cedar_mgcg_init,
 968                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
 969                 break;
 970         case CHIP_PALM:
 971                 radeon_program_register_sequence(rdev,
 972                                                  wrestler_golden_registers,
 973                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
 974                 break;
 975         case CHIP_SUMO:
 976                 radeon_program_register_sequence(rdev,
 977                                                  supersumo_golden_registers,
 978                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 979                 break;
 980         case CHIP_SUMO2:
 981                 radeon_program_register_sequence(rdev,
 982                                                  supersumo_golden_registers,
 983                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 984                 radeon_program_register_sequence(rdev,
 985                                                  sumo_golden_registers,
 986                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
 987                 break;
 988         case CHIP_BARTS:
 989                 radeon_program_register_sequence(rdev,
 990                                                  barts_golden_registers,
 991                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
 992                 break;
 993         case CHIP_TURKS:
 994                 radeon_program_register_sequence(rdev,
 995                                                  turks_golden_registers,
 996                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
 997                 break;
 998         case CHIP_CAICOS:
 999                 radeon_program_register_sequence(rdev,
1000                                                  caicos_golden_registers,
1001                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1002                 break;
1003         default:
1004                 break;
1005         }
1006 }
1007
1008 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009                              unsigned *bankh, unsigned *mtaspect,
1010                              unsigned *tile_split)
1011 {
1012         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016         switch (*bankw) {
1017         default:
1018         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022         }
1023         switch (*bankh) {
1024         default:
1025         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029         }
1030         switch (*mtaspect) {
1031         default:
1032         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036         }
1037 }
1038
1039 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040                               u32 cntl_reg, u32 status_reg)
1041 {
1042         int r, i;
1043         struct atom_clock_dividers dividers;
1044
1045         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046                                            clock, false, &dividers);
1047         if (r)
1048                 return r;
1049
1050         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052         for (i = 0; i < 100; i++) {
1053                 if (RREG32(status_reg) & DCLK_STATUS)
1054                         break;
1055                 mdelay(10);
1056         }
1057         if (i == 100)
1058                 return -ETIMEDOUT;
1059
1060         return 0;
1061 }
1062
1063 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064 {
1065         int r = 0;
1066         u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069         if (r)
1070                 goto done;
1071         cg_scratch &= 0xffff0000;
1072         cg_scratch |= vclk / 100; /* Mhz */
1073
1074         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075         if (r)
1076                 goto done;
1077         cg_scratch &= 0x0000ffff;
1078         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080 done:
1081         WREG32(CG_SCRATCH1, cg_scratch);
1082
1083         return r;
1084 }
1085
1086 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087 {
1088         /* start off with something large */
1089         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1090         int r;
1091
1092         /* bypass vclk and dclk with bclk */
1093         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097         /* put PLL in bypass mode */
1098         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100         if (!vclk || !dclk) {
1101                 /* keep the Bypass mode, put PLL to sleep */
1102                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103                 return 0;
1104         }
1105
1106         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107                                           16384, 0x03FFFFFF, 0, 128, 5,
1108                                           &fb_div, &vclk_div, &dclk_div);
1109         if (r)
1110                 return r;
1111
1112         /* set VCO_MODE to 1 */
1113         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115         /* toggle UPLL_SLEEP to 1 then back to 0 */
1116         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119         /* deassert UPLL_RESET */
1120         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122         mdelay(1);
1123
1124         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1125         if (r)
1126                 return r;
1127
1128         /* assert UPLL_RESET again */
1129         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131         /* disable spread spectrum. */
1132         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134         /* set feedback divider */
1135         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1136
1137         /* set ref divider to 0 */
1138         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
1140         if (fb_div < 307200)
1141                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142         else
1143                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145         /* set PDIV_A and PDIV_B */
1146         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1147                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1148                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150         /* give the PLL some time to settle */
1151         mdelay(15);
1152
1153         /* deassert PLL_RESET */
1154         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156         mdelay(15);
1157
1158         /* switch from bypass mode to normal mode */
1159         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
1161         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1162         if (r)
1163                 return r;
1164
1165         /* switch VCLK and DCLK selection */
1166         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170         mdelay(100);
1171
1172         return 0;
1173 }
1174
1175 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176 {
1177         int readrq;
1178         u16 v;
1179
1180         readrq = pcie_get_readrq(rdev->pdev);
1181         v = ffs(readrq) - 8;
1182         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183          * to avoid hangs or perfomance issues
1184          */
1185         if ((v == 0) || (v == 6) || (v == 7))
1186                 pcie_set_readrq(rdev->pdev, 512);
1187 }
1188
1189 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1190 {
1191         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1192                 return true;
1193         else
1194                 return false;
1195 }
1196
1197 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1198 {
1199         u32 pos1, pos2;
1200
1201         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1203
1204         if (pos1 != pos2)
1205                 return true;
1206         else
1207                 return false;
1208 }
1209
1210 /**
1211  * dce4_wait_for_vblank - vblank wait asic callback.
1212  *
1213  * @rdev: radeon_device pointer
1214  * @crtc: crtc to wait for vblank on
1215  *
1216  * Wait for vblank on the requested crtc (evergreen+).
1217  */
1218 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1219 {
1220         unsigned i = 0;
1221
1222         if (crtc >= rdev->num_crtc)
1223                 return;
1224
1225         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1226                 return;
1227
1228         /* depending on when we hit vblank, we may be close to active; if so,
1229          * wait for another frame.
1230          */
1231         while (dce4_is_in_vblank(rdev, crtc)) {
1232                 if (i++ % 100 == 0) {
1233                         if (!dce4_is_counter_moving(rdev, crtc))
1234                                 break;
1235                 }
1236         }
1237
1238         while (!dce4_is_in_vblank(rdev, crtc)) {
1239                 if (i++ % 100 == 0) {
1240                         if (!dce4_is_counter_moving(rdev, crtc))
1241                                 break;
1242                 }
1243         }
1244 }
1245
1246 /**
1247  * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1248  *
1249  * @rdev: radeon_device pointer
1250  * @crtc: crtc to prepare for pageflip on
1251  *
1252  * Pre-pageflip callback (evergreen+).
1253  * Enables the pageflip irq (vblank irq).
1254  */
1255 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1256 {
1257         /* enable the pflip int */
1258         radeon_irq_kms_pflip_irq_get(rdev, crtc);
1259 }
1260
1261 /**
1262  * evergreen_post_page_flip - pos-pageflip callback.
1263  *
1264  * @rdev: radeon_device pointer
1265  * @crtc: crtc to cleanup pageflip on
1266  *
1267  * Post-pageflip callback (evergreen+).
1268  * Disables the pageflip irq (vblank irq).
1269  */
1270 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1271 {
1272         /* disable the pflip int */
1273         radeon_irq_kms_pflip_irq_put(rdev, crtc);
1274 }
1275
1276 /**
1277  * evergreen_page_flip - pageflip callback.
1278  *
1279  * @rdev: radeon_device pointer
1280  * @crtc_id: crtc to cleanup pageflip on
1281  * @crtc_base: new address of the crtc (GPU MC address)
1282  *
1283  * Does the actual pageflip (evergreen+).
1284  * During vblank we take the crtc lock and wait for the update_pending
1285  * bit to go high, when it does, we release the lock, and allow the
1286  * double buffered update to take place.
1287  * Returns the current update pending status.
1288  */
1289 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1290 {
1291         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1292         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1293         int i;
1294
1295         /* Lock the graphics update lock */
1296         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1297         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1298
1299         /* update the scanout addresses */
1300         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1301                upper_32_bits(crtc_base));
1302         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1303                (u32)crtc_base);
1304
1305         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1306                upper_32_bits(crtc_base));
1307         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1308                (u32)crtc_base);
1309
1310         /* Wait for update_pending to go high. */
1311         for (i = 0; i < rdev->usec_timeout; i++) {
1312                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1313                         break;
1314                 udelay(1);
1315         }
1316         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1317
1318         /* Unlock the lock, so double-buffering can take place inside vblank */
1319         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1320         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1321
1322         /* Return current update_pending status: */
1323         return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1324 }
1325
1326 /* get temperature in millidegrees */
1327 int evergreen_get_temp(struct radeon_device *rdev)
1328 {
1329         u32 temp, toffset;
1330         int actual_temp = 0;
1331
1332         if (rdev->family == CHIP_JUNIPER) {
1333                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1334                         TOFFSET_SHIFT;
1335                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1336                         TS0_ADC_DOUT_SHIFT;
1337
1338                 if (toffset & 0x100)
1339                         actual_temp = temp / 2 - (0x200 - toffset);
1340                 else
1341                         actual_temp = temp / 2 + toffset;
1342
1343                 actual_temp = actual_temp * 1000;
1344
1345         } else {
1346                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1347                         ASIC_T_SHIFT;
1348
1349                 if (temp & 0x400)
1350                         actual_temp = -256;
1351                 else if (temp & 0x200)
1352                         actual_temp = 255;
1353                 else if (temp & 0x100) {
1354                         actual_temp = temp & 0x1ff;
1355                         actual_temp |= ~0x1ff;
1356                 } else
1357                         actual_temp = temp & 0xff;
1358
1359                 actual_temp = (actual_temp * 1000) / 2;
1360         }
1361
1362         return actual_temp;
1363 }
1364
1365 int sumo_get_temp(struct radeon_device *rdev)
1366 {
1367         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1368         int actual_temp = temp - 49;
1369
1370         return actual_temp * 1000;
1371 }
1372
1373 /**
1374  * sumo_pm_init_profile - Initialize power profiles callback.
1375  *
1376  * @rdev: radeon_device pointer
1377  *
1378  * Initialize the power states used in profile mode
1379  * (sumo, trinity, SI).
1380  * Used for profile mode only.
1381  */
1382 void sumo_pm_init_profile(struct radeon_device *rdev)
1383 {
1384         int idx;
1385
1386         /* default */
1387         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1388         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1389         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1390         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1391
1392         /* low,mid sh/mh */
1393         if (rdev->flags & RADEON_IS_MOBILITY)
1394                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1395         else
1396                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1397
1398         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1399         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1400         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1401         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1402
1403         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1404         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1405         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1406         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1407
1408         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1409         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1410         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1411         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1412
1413         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1414         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1415         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1416         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1417
1418         /* high sh/mh */
1419         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1420         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1421         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1422         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1423         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1424                 rdev->pm.power_state[idx].num_clock_modes - 1;
1425
1426         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1427         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1428         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1429         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1430                 rdev->pm.power_state[idx].num_clock_modes - 1;
1431 }
1432
1433 /**
1434  * btc_pm_init_profile - Initialize power profiles callback.
1435  *
1436  * @rdev: radeon_device pointer
1437  *
1438  * Initialize the power states used in profile mode
1439  * (BTC, cayman).
1440  * Used for profile mode only.
1441  */
1442 void btc_pm_init_profile(struct radeon_device *rdev)
1443 {
1444         int idx;
1445
1446         /* default */
1447         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1448         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1449         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1450         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1451         /* starting with BTC, there is one state that is used for both
1452          * MH and SH.  Difference is that we always use the high clock index for
1453          * mclk.
1454          */
1455         if (rdev->flags & RADEON_IS_MOBILITY)
1456                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1457         else
1458                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1459         /* low sh */
1460         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1461         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1462         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1463         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1464         /* mid sh */
1465         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1466         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1467         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1468         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1469         /* high sh */
1470         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1471         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1472         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1473         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1474         /* low mh */
1475         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1476         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1477         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1478         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1479         /* mid mh */
1480         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1481         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1482         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1483         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1484         /* high mh */
1485         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1486         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1487         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1488         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1489 }
1490
1491 /**
1492  * evergreen_pm_misc - set additional pm hw parameters callback.
1493  *
1494  * @rdev: radeon_device pointer
1495  *
1496  * Set non-clock parameters associated with a power state
1497  * (voltage, etc.) (evergreen+).
1498  */
1499 void evergreen_pm_misc(struct radeon_device *rdev)
1500 {
1501         int req_ps_idx = rdev->pm.requested_power_state_index;
1502         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1503         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1504         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1505
1506         if (voltage->type == VOLTAGE_SW) {
1507                 /* 0xff0x are flags rather then an actual voltage */
1508                 if ((voltage->voltage & 0xff00) == 0xff00)
1509                         return;
1510                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1511                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1512                         rdev->pm.current_vddc = voltage->voltage;
1513                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1514                 }
1515
1516                 /* starting with BTC, there is one state that is used for both
1517                  * MH and SH.  Difference is that we always use the high clock index for
1518                  * mclk and vddci.
1519                  */
1520                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1521                     (rdev->family >= CHIP_BARTS) &&
1522                     rdev->pm.active_crtc_count &&
1523                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1524                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1525                         voltage = &rdev->pm.power_state[req_ps_idx].
1526                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1527
1528                 /* 0xff0x are flags rather then an actual voltage */
1529                 if ((voltage->vddci & 0xff00) == 0xff00)
1530                         return;
1531                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1532                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1533                         rdev->pm.current_vddci = voltage->vddci;
1534                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1535                 }
1536         }
1537 }
1538
1539 /**
1540  * evergreen_pm_prepare - pre-power state change callback.
1541  *
1542  * @rdev: radeon_device pointer
1543  *
1544  * Prepare for a power state change (evergreen+).
1545  */
1546 void evergreen_pm_prepare(struct radeon_device *rdev)
1547 {
1548         struct drm_device *ddev = rdev->ddev;
1549         struct drm_crtc *crtc;
1550         struct radeon_crtc *radeon_crtc;
1551         u32 tmp;
1552
1553         /* disable any active CRTCs */
1554         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1555                 radeon_crtc = to_radeon_crtc(crtc);
1556                 if (radeon_crtc->enabled) {
1557                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1558                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1559                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1560                 }
1561         }
1562 }
1563
1564 /**
1565  * evergreen_pm_finish - post-power state change callback.
1566  *
1567  * @rdev: radeon_device pointer
1568  *
1569  * Clean up after a power state change (evergreen+).
1570  */
1571 void evergreen_pm_finish(struct radeon_device *rdev)
1572 {
1573         struct drm_device *ddev = rdev->ddev;
1574         struct drm_crtc *crtc;
1575         struct radeon_crtc *radeon_crtc;
1576         u32 tmp;
1577
1578         /* enable any active CRTCs */
1579         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1580                 radeon_crtc = to_radeon_crtc(crtc);
1581                 if (radeon_crtc->enabled) {
1582                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1583                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1584                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1585                 }
1586         }
1587 }
1588
1589 /**
1590  * evergreen_hpd_sense - hpd sense callback.
1591  *
1592  * @rdev: radeon_device pointer
1593  * @hpd: hpd (hotplug detect) pin
1594  *
1595  * Checks if a digital monitor is connected (evergreen+).
1596  * Returns true if connected, false if not connected.
1597  */
1598 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1599 {
1600         bool connected = false;
1601
1602         switch (hpd) {
1603         case RADEON_HPD_1:
1604                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1605                         connected = true;
1606                 break;
1607         case RADEON_HPD_2:
1608                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1609                         connected = true;
1610                 break;
1611         case RADEON_HPD_3:
1612                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1613                         connected = true;
1614                 break;
1615         case RADEON_HPD_4:
1616                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1617                         connected = true;
1618                 break;
1619         case RADEON_HPD_5:
1620                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1621                         connected = true;
1622                 break;
1623         case RADEON_HPD_6:
1624                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1625                         connected = true;
1626                         break;
1627         default:
1628                 break;
1629         }
1630
1631         return connected;
1632 }
1633
1634 /**
1635  * evergreen_hpd_set_polarity - hpd set polarity callback.
1636  *
1637  * @rdev: radeon_device pointer
1638  * @hpd: hpd (hotplug detect) pin
1639  *
1640  * Set the polarity of the hpd pin (evergreen+).
1641  */
1642 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1643                                 enum radeon_hpd_id hpd)
1644 {
1645         u32 tmp;
1646         bool connected = evergreen_hpd_sense(rdev, hpd);
1647
1648         switch (hpd) {
1649         case RADEON_HPD_1:
1650                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1651                 if (connected)
1652                         tmp &= ~DC_HPDx_INT_POLARITY;
1653                 else
1654                         tmp |= DC_HPDx_INT_POLARITY;
1655                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1656                 break;
1657         case RADEON_HPD_2:
1658                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1659                 if (connected)
1660                         tmp &= ~DC_HPDx_INT_POLARITY;
1661                 else
1662                         tmp |= DC_HPDx_INT_POLARITY;
1663                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1664                 break;
1665         case RADEON_HPD_3:
1666                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1667                 if (connected)
1668                         tmp &= ~DC_HPDx_INT_POLARITY;
1669                 else
1670                         tmp |= DC_HPDx_INT_POLARITY;
1671                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1672                 break;
1673         case RADEON_HPD_4:
1674                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1675                 if (connected)
1676                         tmp &= ~DC_HPDx_INT_POLARITY;
1677                 else
1678                         tmp |= DC_HPDx_INT_POLARITY;
1679                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1680                 break;
1681         case RADEON_HPD_5:
1682                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1683                 if (connected)
1684                         tmp &= ~DC_HPDx_INT_POLARITY;
1685                 else
1686                         tmp |= DC_HPDx_INT_POLARITY;
1687                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1688                         break;
1689         case RADEON_HPD_6:
1690                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1691                 if (connected)
1692                         tmp &= ~DC_HPDx_INT_POLARITY;
1693                 else
1694                         tmp |= DC_HPDx_INT_POLARITY;
1695                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1696                 break;
1697         default:
1698                 break;
1699         }
1700 }
1701
1702 /**
1703  * evergreen_hpd_init - hpd setup callback.
1704  *
1705  * @rdev: radeon_device pointer
1706  *
1707  * Setup the hpd pins used by the card (evergreen+).
1708  * Enable the pin, set the polarity, and enable the hpd interrupts.
1709  */
1710 void evergreen_hpd_init(struct radeon_device *rdev)
1711 {
1712         struct drm_device *dev = rdev->ddev;
1713         struct drm_connector *connector;
1714         unsigned enabled = 0;
1715         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1716                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1717
1718         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1719                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1720
1721                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1722                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1723                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1724                          * aux dp channel on imac and help (but not completely fix)
1725                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1726                          * also avoid interrupt storms during dpms.
1727                          */
1728                         continue;
1729                 }
1730                 switch (radeon_connector->hpd.hpd) {
1731                 case RADEON_HPD_1:
1732                         WREG32(DC_HPD1_CONTROL, tmp);
1733                         break;
1734                 case RADEON_HPD_2:
1735                         WREG32(DC_HPD2_CONTROL, tmp);
1736                         break;
1737                 case RADEON_HPD_3:
1738                         WREG32(DC_HPD3_CONTROL, tmp);
1739                         break;
1740                 case RADEON_HPD_4:
1741                         WREG32(DC_HPD4_CONTROL, tmp);
1742                         break;
1743                 case RADEON_HPD_5:
1744                         WREG32(DC_HPD5_CONTROL, tmp);
1745                         break;
1746                 case RADEON_HPD_6:
1747                         WREG32(DC_HPD6_CONTROL, tmp);
1748                         break;
1749                 default:
1750                         break;
1751                 }
1752                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1753                 enabled |= 1 << radeon_connector->hpd.hpd;
1754         }
1755         radeon_irq_kms_enable_hpd(rdev, enabled);
1756 }
1757
1758 /**
1759  * evergreen_hpd_fini - hpd tear down callback.
1760  *
1761  * @rdev: radeon_device pointer
1762  *
1763  * Tear down the hpd pins used by the card (evergreen+).
1764  * Disable the hpd interrupts.
1765  */
1766 void evergreen_hpd_fini(struct radeon_device *rdev)
1767 {
1768         struct drm_device *dev = rdev->ddev;
1769         struct drm_connector *connector;
1770         unsigned disabled = 0;
1771
1772         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1774                 switch (radeon_connector->hpd.hpd) {
1775                 case RADEON_HPD_1:
1776                         WREG32(DC_HPD1_CONTROL, 0);
1777                         break;
1778                 case RADEON_HPD_2:
1779                         WREG32(DC_HPD2_CONTROL, 0);
1780                         break;
1781                 case RADEON_HPD_3:
1782                         WREG32(DC_HPD3_CONTROL, 0);
1783                         break;
1784                 case RADEON_HPD_4:
1785                         WREG32(DC_HPD4_CONTROL, 0);
1786                         break;
1787                 case RADEON_HPD_5:
1788                         WREG32(DC_HPD5_CONTROL, 0);
1789                         break;
1790                 case RADEON_HPD_6:
1791                         WREG32(DC_HPD6_CONTROL, 0);
1792                         break;
1793                 default:
1794                         break;
1795                 }
1796                 disabled |= 1 << radeon_connector->hpd.hpd;
1797         }
1798         radeon_irq_kms_disable_hpd(rdev, disabled);
1799 }
1800
1801 /* watermark setup */
1802
1803 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1804                                         struct radeon_crtc *radeon_crtc,
1805                                         struct drm_display_mode *mode,
1806                                         struct drm_display_mode *other_mode)
1807 {
1808         u32 tmp, buffer_alloc, i;
1809         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1810         /*
1811          * Line Buffer Setup
1812          * There are 3 line buffers, each one shared by 2 display controllers.
1813          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1814          * the display controllers.  The paritioning is done via one of four
1815          * preset allocations specified in bits 2:0:
1816          * first display controller
1817          *  0 - first half of lb (3840 * 2)
1818          *  1 - first 3/4 of lb (5760 * 2)
1819          *  2 - whole lb (7680 * 2), other crtc must be disabled
1820          *  3 - first 1/4 of lb (1920 * 2)
1821          * second display controller
1822          *  4 - second half of lb (3840 * 2)
1823          *  5 - second 3/4 of lb (5760 * 2)
1824          *  6 - whole lb (7680 * 2), other crtc must be disabled
1825          *  7 - last 1/4 of lb (1920 * 2)
1826          */
1827         /* this can get tricky if we have two large displays on a paired group
1828          * of crtcs.  Ideally for multiple large displays we'd assign them to
1829          * non-linked crtcs for maximum line buffer allocation.
1830          */
1831         if (radeon_crtc->base.enabled && mode) {
1832                 if (other_mode) {
1833                         tmp = 0; /* 1/2 */
1834                         buffer_alloc = 1;
1835                 } else {
1836                         tmp = 2; /* whole */
1837                         buffer_alloc = 2;
1838                 }
1839         } else {
1840                 tmp = 0;
1841                 buffer_alloc = 0;
1842         }
1843
1844         /* second controller of the pair uses second half of the lb */
1845         if (radeon_crtc->crtc_id % 2)
1846                 tmp += 4;
1847         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1848
1849         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1850                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1851                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1852                 for (i = 0; i < rdev->usec_timeout; i++) {
1853                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1854                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1855                                 break;
1856                         udelay(1);
1857                 }
1858         }
1859
1860         if (radeon_crtc->base.enabled && mode) {
1861                 switch (tmp) {
1862                 case 0:
1863                 case 4:
1864                 default:
1865                         if (ASIC_IS_DCE5(rdev))
1866                                 return 4096 * 2;
1867                         else
1868                                 return 3840 * 2;
1869                 case 1:
1870                 case 5:
1871                         if (ASIC_IS_DCE5(rdev))
1872                                 return 6144 * 2;
1873                         else
1874                                 return 5760 * 2;
1875                 case 2:
1876                 case 6:
1877                         if (ASIC_IS_DCE5(rdev))
1878                                 return 8192 * 2;
1879                         else
1880                                 return 7680 * 2;
1881                 case 3:
1882                 case 7:
1883                         if (ASIC_IS_DCE5(rdev))
1884                                 return 2048 * 2;
1885                         else
1886                                 return 1920 * 2;
1887                 }
1888         }
1889
1890         /* controller not enabled, so no lb used */
1891         return 0;
1892 }
1893
1894 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1895 {
1896         u32 tmp = RREG32(MC_SHARED_CHMAP);
1897
1898         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1899         case 0:
1900         default:
1901                 return 1;
1902         case 1:
1903                 return 2;
1904         case 2:
1905                 return 4;
1906         case 3:
1907                 return 8;
1908         }
1909 }
1910
1911 struct evergreen_wm_params {
1912         u32 dram_channels; /* number of dram channels */
1913         u32 yclk;          /* bandwidth per dram data pin in kHz */
1914         u32 sclk;          /* engine clock in kHz */
1915         u32 disp_clk;      /* display clock in kHz */
1916         u32 src_width;     /* viewport width */
1917         u32 active_time;   /* active display time in ns */
1918         u32 blank_time;    /* blank time in ns */
1919         bool interlaced;    /* mode is interlaced */
1920         fixed20_12 vsc;    /* vertical scale ratio */
1921         u32 num_heads;     /* number of active crtcs */
1922         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1923         u32 lb_size;       /* line buffer allocated to pipe */
1924         u32 vtaps;         /* vertical scaler taps */
1925 };
1926
1927 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1928 {
1929         /* Calculate DRAM Bandwidth and the part allocated to display. */
1930         fixed20_12 dram_efficiency; /* 0.7 */
1931         fixed20_12 yclk, dram_channels, bandwidth;
1932         fixed20_12 a;
1933
1934         a.full = dfixed_const(1000);
1935         yclk.full = dfixed_const(wm->yclk);
1936         yclk.full = dfixed_div(yclk, a);
1937         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1938         a.full = dfixed_const(10);
1939         dram_efficiency.full = dfixed_const(7);
1940         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1941         bandwidth.full = dfixed_mul(dram_channels, yclk);
1942         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1943
1944         return dfixed_trunc(bandwidth);
1945 }
1946
1947 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1948 {
1949         /* Calculate DRAM Bandwidth and the part allocated to display. */
1950         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1951         fixed20_12 yclk, dram_channels, bandwidth;
1952         fixed20_12 a;
1953
1954         a.full = dfixed_const(1000);
1955         yclk.full = dfixed_const(wm->yclk);
1956         yclk.full = dfixed_div(yclk, a);
1957         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1958         a.full = dfixed_const(10);
1959         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1960         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1961         bandwidth.full = dfixed_mul(dram_channels, yclk);
1962         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1963
1964         return dfixed_trunc(bandwidth);
1965 }
1966
1967 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1968 {
1969         /* Calculate the display Data return Bandwidth */
1970         fixed20_12 return_efficiency; /* 0.8 */
1971         fixed20_12 sclk, bandwidth;
1972         fixed20_12 a;
1973
1974         a.full = dfixed_const(1000);
1975         sclk.full = dfixed_const(wm->sclk);
1976         sclk.full = dfixed_div(sclk, a);
1977         a.full = dfixed_const(10);
1978         return_efficiency.full = dfixed_const(8);
1979         return_efficiency.full = dfixed_div(return_efficiency, a);
1980         a.full = dfixed_const(32);
1981         bandwidth.full = dfixed_mul(a, sclk);
1982         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1983
1984         return dfixed_trunc(bandwidth);
1985 }
1986
1987 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1988 {
1989         /* Calculate the DMIF Request Bandwidth */
1990         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1991         fixed20_12 disp_clk, bandwidth;
1992         fixed20_12 a;
1993
1994         a.full = dfixed_const(1000);
1995         disp_clk.full = dfixed_const(wm->disp_clk);
1996         disp_clk.full = dfixed_div(disp_clk, a);
1997         a.full = dfixed_const(10);
1998         disp_clk_request_efficiency.full = dfixed_const(8);
1999         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2000         a.full = dfixed_const(32);
2001         bandwidth.full = dfixed_mul(a, disp_clk);
2002         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2003
2004         return dfixed_trunc(bandwidth);
2005 }
2006
2007 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2008 {
2009         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2010         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2011         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2012         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2013
2014         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2015 }
2016
2017 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2018 {
2019         /* Calculate the display mode Average Bandwidth
2020          * DisplayMode should contain the source and destination dimensions,
2021          * timing, etc.
2022          */
2023         fixed20_12 bpp;
2024         fixed20_12 line_time;
2025         fixed20_12 src_width;
2026         fixed20_12 bandwidth;
2027         fixed20_12 a;
2028
2029         a.full = dfixed_const(1000);
2030         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2031         line_time.full = dfixed_div(line_time, a);
2032         bpp.full = dfixed_const(wm->bytes_per_pixel);
2033         src_width.full = dfixed_const(wm->src_width);
2034         bandwidth.full = dfixed_mul(src_width, bpp);
2035         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2036         bandwidth.full = dfixed_div(bandwidth, line_time);
2037
2038         return dfixed_trunc(bandwidth);
2039 }
2040
2041 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2042 {
2043         /* First calcualte the latency in ns */
2044         u32 mc_latency = 2000; /* 2000 ns. */
2045         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2046         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2047         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2048         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2049         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2050                 (wm->num_heads * cursor_line_pair_return_time);
2051         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2052         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2053         fixed20_12 a, b, c;
2054
2055         if (wm->num_heads == 0)
2056                 return 0;
2057
2058         a.full = dfixed_const(2);
2059         b.full = dfixed_const(1);
2060         if ((wm->vsc.full > a.full) ||
2061             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2062             (wm->vtaps >= 5) ||
2063             ((wm->vsc.full >= a.full) && wm->interlaced))
2064                 max_src_lines_per_dst_line = 4;
2065         else
2066                 max_src_lines_per_dst_line = 2;
2067
2068         a.full = dfixed_const(available_bandwidth);
2069         b.full = dfixed_const(wm->num_heads);
2070         a.full = dfixed_div(a, b);
2071
2072         b.full = dfixed_const(1000);
2073         c.full = dfixed_const(wm->disp_clk);
2074         b.full = dfixed_div(c, b);
2075         c.full = dfixed_const(wm->bytes_per_pixel);
2076         b.full = dfixed_mul(b, c);
2077
2078         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2079
2080         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2081         b.full = dfixed_const(1000);
2082         c.full = dfixed_const(lb_fill_bw);
2083         b.full = dfixed_div(c, b);
2084         a.full = dfixed_div(a, b);
2085         line_fill_time = dfixed_trunc(a);
2086
2087         if (line_fill_time < wm->active_time)
2088                 return latency;
2089         else
2090                 return latency + (line_fill_time - wm->active_time);
2091
2092 }
2093
2094 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2095 {
2096         if (evergreen_average_bandwidth(wm) <=
2097             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2098                 return true;
2099         else
2100                 return false;
2101 };
2102
2103 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2104 {
2105         if (evergreen_average_bandwidth(wm) <=
2106             (evergreen_available_bandwidth(wm) / wm->num_heads))
2107                 return true;
2108         else
2109                 return false;
2110 };
2111
2112 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2113 {
2114         u32 lb_partitions = wm->lb_size / wm->src_width;
2115         u32 line_time = wm->active_time + wm->blank_time;
2116         u32 latency_tolerant_lines;
2117         u32 latency_hiding;
2118         fixed20_12 a;
2119
2120         a.full = dfixed_const(1);
2121         if (wm->vsc.full > a.full)
2122                 latency_tolerant_lines = 1;
2123         else {
2124                 if (lb_partitions <= (wm->vtaps + 1))
2125                         latency_tolerant_lines = 1;
2126                 else
2127                         latency_tolerant_lines = 2;
2128         }
2129
2130         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2131
2132         if (evergreen_latency_watermark(wm) <= latency_hiding)
2133                 return true;
2134         else
2135                 return false;
2136 }
2137
2138 static void evergreen_program_watermarks(struct radeon_device *rdev,
2139                                          struct radeon_crtc *radeon_crtc,
2140                                          u32 lb_size, u32 num_heads)
2141 {
2142         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2143         struct evergreen_wm_params wm_low, wm_high;
2144         u32 dram_channels;
2145         u32 pixel_period;
2146         u32 line_time = 0;
2147         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2148         u32 priority_a_mark = 0, priority_b_mark = 0;
2149         u32 priority_a_cnt = PRIORITY_OFF;
2150         u32 priority_b_cnt = PRIORITY_OFF;
2151         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2152         u32 tmp, arb_control3;
2153         fixed20_12 a, b, c;
2154
2155         if (radeon_crtc->base.enabled && num_heads && mode) {
2156                 pixel_period = 1000000 / (u32)mode->clock;
2157                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2158                 priority_a_cnt = 0;
2159                 priority_b_cnt = 0;
2160                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2161
2162                 /* watermark for high clocks */
2163                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2164                         wm_high.yclk =
2165                                 radeon_dpm_get_mclk(rdev, false) * 10;
2166                         wm_high.sclk =
2167                                 radeon_dpm_get_sclk(rdev, false) * 10;
2168                 } else {
2169                         wm_high.yclk = rdev->pm.current_mclk * 10;
2170                         wm_high.sclk = rdev->pm.current_sclk * 10;
2171                 }
2172
2173                 wm_high.disp_clk = mode->clock;
2174                 wm_high.src_width = mode->crtc_hdisplay;
2175                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2176                 wm_high.blank_time = line_time - wm_high.active_time;
2177                 wm_high.interlaced = false;
2178                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2179                         wm_high.interlaced = true;
2180                 wm_high.vsc = radeon_crtc->vsc;
2181                 wm_high.vtaps = 1;
2182                 if (radeon_crtc->rmx_type != RMX_OFF)
2183                         wm_high.vtaps = 2;
2184                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2185                 wm_high.lb_size = lb_size;
2186                 wm_high.dram_channels = dram_channels;
2187                 wm_high.num_heads = num_heads;
2188
2189                 /* watermark for low clocks */
2190                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2191                         wm_low.yclk =
2192                                 radeon_dpm_get_mclk(rdev, true) * 10;
2193                         wm_low.sclk =
2194                                 radeon_dpm_get_sclk(rdev, true) * 10;
2195                 } else {
2196                         wm_low.yclk = rdev->pm.current_mclk * 10;
2197                         wm_low.sclk = rdev->pm.current_sclk * 10;
2198                 }
2199
2200                 wm_low.disp_clk = mode->clock;
2201                 wm_low.src_width = mode->crtc_hdisplay;
2202                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2203                 wm_low.blank_time = line_time - wm_low.active_time;
2204                 wm_low.interlaced = false;
2205                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2206                         wm_low.interlaced = true;
2207                 wm_low.vsc = radeon_crtc->vsc;
2208                 wm_low.vtaps = 1;
2209                 if (radeon_crtc->rmx_type != RMX_OFF)
2210                         wm_low.vtaps = 2;
2211                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2212                 wm_low.lb_size = lb_size;
2213                 wm_low.dram_channels = dram_channels;
2214                 wm_low.num_heads = num_heads;
2215
2216                 /* set for high clocks */
2217                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2218                 /* set for low clocks */
2219                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2220
2221                 /* possibly force display priority to high */
2222                 /* should really do this at mode validation time... */
2223                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2224                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2225                     !evergreen_check_latency_hiding(&wm_high) ||
2226                     (rdev->disp_priority == 2)) {
2227                         DRM_DEBUG_KMS("force priority a to high\n");
2228                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2229                 }
2230                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2231                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2232                     !evergreen_check_latency_hiding(&wm_low) ||
2233                     (rdev->disp_priority == 2)) {
2234                         DRM_DEBUG_KMS("force priority b to high\n");
2235                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2236                 }
2237
2238                 a.full = dfixed_const(1000);
2239                 b.full = dfixed_const(mode->clock);
2240                 b.full = dfixed_div(b, a);
2241                 c.full = dfixed_const(latency_watermark_a);
2242                 c.full = dfixed_mul(c, b);
2243                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2244                 c.full = dfixed_div(c, a);
2245                 a.full = dfixed_const(16);
2246                 c.full = dfixed_div(c, a);
2247                 priority_a_mark = dfixed_trunc(c);
2248                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2249
2250                 a.full = dfixed_const(1000);
2251                 b.full = dfixed_const(mode->clock);
2252                 b.full = dfixed_div(b, a);
2253                 c.full = dfixed_const(latency_watermark_b);
2254                 c.full = dfixed_mul(c, b);
2255                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2256                 c.full = dfixed_div(c, a);
2257                 a.full = dfixed_const(16);
2258                 c.full = dfixed_div(c, a);
2259                 priority_b_mark = dfixed_trunc(c);
2260                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2261         }
2262
2263         /* select wm A */
2264         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2265         tmp = arb_control3;
2266         tmp &= ~LATENCY_WATERMARK_MASK(3);
2267         tmp |= LATENCY_WATERMARK_MASK(1);
2268         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2269         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2270                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2271                 LATENCY_HIGH_WATERMARK(line_time)));
2272         /* select wm B */
2273         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2274         tmp &= ~LATENCY_WATERMARK_MASK(3);
2275         tmp |= LATENCY_WATERMARK_MASK(2);
2276         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2277         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2278                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2279                 LATENCY_HIGH_WATERMARK(line_time)));
2280         /* restore original selection */
2281         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2282
2283         /* write the priority marks */
2284         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2285         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2286
2287         /* save values for DPM */
2288         radeon_crtc->line_time = line_time;
2289         radeon_crtc->wm_high = latency_watermark_a;
2290         radeon_crtc->wm_low = latency_watermark_b;
2291 }
2292
2293 /**
2294  * evergreen_bandwidth_update - update display watermarks callback.
2295  *
2296  * @rdev: radeon_device pointer
2297  *
2298  * Update the display watermarks based on the requested mode(s)
2299  * (evergreen+).
2300  */
2301 void evergreen_bandwidth_update(struct radeon_device *rdev)
2302 {
2303         struct drm_display_mode *mode0 = NULL;
2304         struct drm_display_mode *mode1 = NULL;
2305         u32 num_heads = 0, lb_size;
2306         int i;
2307
2308         radeon_update_display_priority(rdev);
2309
2310         for (i = 0; i < rdev->num_crtc; i++) {
2311                 if (rdev->mode_info.crtcs[i]->base.enabled)
2312                         num_heads++;
2313         }
2314         for (i = 0; i < rdev->num_crtc; i += 2) {
2315                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2316                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2317                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2318                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2319                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2320                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2321         }
2322 }
2323
2324 /**
2325  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2326  *
2327  * @rdev: radeon_device pointer
2328  *
2329  * Wait for the MC (memory controller) to be idle.
2330  * (evergreen+).
2331  * Returns 0 if the MC is idle, -1 if not.
2332  */
2333 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2334 {
2335         unsigned i;
2336         u32 tmp;
2337
2338         for (i = 0; i < rdev->usec_timeout; i++) {
2339                 /* read MC_STATUS */
2340                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2341                 if (!tmp)
2342                         return 0;
2343                 udelay(1);
2344         }
2345         return -1;
2346 }
2347
2348 /*
2349  * GART
2350  */
2351 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2352 {
2353         unsigned i;
2354         u32 tmp;
2355
2356         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2357
2358         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2359         for (i = 0; i < rdev->usec_timeout; i++) {
2360                 /* read MC_STATUS */
2361                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2362                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2363                 if (tmp == 2) {
2364                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2365                         return;
2366                 }
2367                 if (tmp) {
2368                         return;
2369                 }
2370                 udelay(1);
2371         }
2372 }
2373
2374 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2375 {
2376         u32 tmp;
2377         int r;
2378
2379         if (rdev->gart.robj == NULL) {
2380                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2381                 return -EINVAL;
2382         }
2383         r = radeon_gart_table_vram_pin(rdev);
2384         if (r)
2385                 return r;
2386         radeon_gart_restore(rdev);
2387         /* Setup L2 cache */
2388         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2389                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2390                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2391         WREG32(VM_L2_CNTL2, 0);
2392         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2393         /* Setup TLB control */
2394         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2395                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2396                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2397                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2398         if (rdev->flags & RADEON_IS_IGP) {
2399                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2400                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2401                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2402         } else {
2403                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2404                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2405                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2406                 if ((rdev->family == CHIP_JUNIPER) ||
2407                     (rdev->family == CHIP_CYPRESS) ||
2408                     (rdev->family == CHIP_HEMLOCK) ||
2409                     (rdev->family == CHIP_BARTS))
2410                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2411         }
2412         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2413         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2414         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2415         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2416         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2417         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2418         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2419         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2420                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2421         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2422                         (u32)(rdev->dummy_page.addr >> 12));
2423         WREG32(VM_CONTEXT1_CNTL, 0);
2424
2425         evergreen_pcie_gart_tlb_flush(rdev);
2426         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2427                  (unsigned)(rdev->mc.gtt_size >> 20),
2428                  (unsigned long long)rdev->gart.table_addr);
2429         rdev->gart.ready = true;
2430         return 0;
2431 }
2432
2433 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2434 {
2435         u32 tmp;
2436
2437         /* Disable all tables */
2438         WREG32(VM_CONTEXT0_CNTL, 0);
2439         WREG32(VM_CONTEXT1_CNTL, 0);
2440
2441         /* Setup L2 cache */
2442         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2443                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2444         WREG32(VM_L2_CNTL2, 0);
2445         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2446         /* Setup TLB control */
2447         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2448         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2449         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2450         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2451         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2452         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2453         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2454         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2455         radeon_gart_table_vram_unpin(rdev);
2456 }
2457
2458 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2459 {
2460         evergreen_pcie_gart_disable(rdev);
2461         radeon_gart_table_vram_free(rdev);
2462         radeon_gart_fini(rdev);
2463 }
2464
2465
2466 static void evergreen_agp_enable(struct radeon_device *rdev)
2467 {
2468         u32 tmp;
2469
2470         /* Setup L2 cache */
2471         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2472                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2473                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2474         WREG32(VM_L2_CNTL2, 0);
2475         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2476         /* Setup TLB control */
2477         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2478                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2479                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2480                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2481         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2482         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2483         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2484         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2485         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2486         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2487         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2488         WREG32(VM_CONTEXT0_CNTL, 0);
2489         WREG32(VM_CONTEXT1_CNTL, 0);
2490 }
2491
2492 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2493 {
2494         u32 crtc_enabled, tmp, frame_count, blackout;
2495         int i, j;
2496
2497         if (!ASIC_IS_NODCE(rdev)) {
2498                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2499                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2500
2501                 /* disable VGA render */
2502                 WREG32(VGA_RENDER_CONTROL, 0);
2503         }
2504         /* blank the display controllers */
2505         for (i = 0; i < rdev->num_crtc; i++) {
2506                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2507                 if (crtc_enabled) {
2508                         save->crtc_enabled[i] = true;
2509                         if (ASIC_IS_DCE6(rdev)) {
2510                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2511                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2512                                         radeon_wait_for_vblank(rdev, i);
2513                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2514                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2515                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2516                                 }
2517                         } else {
2518                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2519                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2520                                         radeon_wait_for_vblank(rdev, i);
2521                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2522                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2523                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2524                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2525                                 }
2526                         }
2527                         /* wait for the next frame */
2528                         frame_count = radeon_get_vblank_counter(rdev, i);
2529                         for (j = 0; j < rdev->usec_timeout; j++) {
2530                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2531                                         break;
2532                                 udelay(1);
2533                         }
2534
2535                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2536                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2537                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2538                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2539                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2540                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2541                         save->crtc_enabled[i] = false;
2542                         /* ***** */
2543                 } else {
2544                         save->crtc_enabled[i] = false;
2545                 }
2546         }
2547
2548         radeon_mc_wait_for_idle(rdev);
2549
2550         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2551         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2552                 /* Block CPU access */
2553                 WREG32(BIF_FB_EN, 0);
2554                 /* blackout the MC */
2555                 blackout &= ~BLACKOUT_MODE_MASK;
2556                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2557         }
2558         /* wait for the MC to settle */
2559         udelay(100);
2560
2561         /* lock double buffered regs */
2562         for (i = 0; i < rdev->num_crtc; i++) {
2563                 if (save->crtc_enabled[i]) {
2564                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2565                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2566                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2567                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2568                         }
2569                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2570                         if (!(tmp & 1)) {
2571                                 tmp |= 1;
2572                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2573                         }
2574                 }
2575         }
2576 }
2577
2578 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2579 {
2580         u32 tmp, frame_count;
2581         int i, j;
2582
2583         /* update crtc base addresses */
2584         for (i = 0; i < rdev->num_crtc; i++) {
2585                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2586                        upper_32_bits(rdev->mc.vram_start));
2587                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2588                        upper_32_bits(rdev->mc.vram_start));
2589                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2590                        (u32)rdev->mc.vram_start);
2591                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2592                        (u32)rdev->mc.vram_start);
2593         }
2594
2595         if (!ASIC_IS_NODCE(rdev)) {
2596                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2597                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2598         }
2599
2600         /* unlock regs and wait for update */
2601         for (i = 0; i < rdev->num_crtc; i++) {
2602                 if (save->crtc_enabled[i]) {
2603                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2604                         if ((tmp & 0x3) != 0) {
2605                                 tmp &= ~0x3;
2606                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2607                         }
2608                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2609                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2610                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2611                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2612                         }
2613                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2614                         if (tmp & 1) {
2615                                 tmp &= ~1;
2616                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2617                         }
2618                         for (j = 0; j < rdev->usec_timeout; j++) {
2619                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2620                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2621                                         break;
2622                                 udelay(1);
2623                         }
2624                 }
2625         }
2626
2627         /* unblackout the MC */
2628         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2629         tmp &= ~BLACKOUT_MODE_MASK;
2630         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2631         /* allow CPU access */
2632         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2633
2634         for (i = 0; i < rdev->num_crtc; i++) {
2635                 if (save->crtc_enabled[i]) {
2636                         if (ASIC_IS_DCE6(rdev)) {
2637                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2638                                 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2639                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2640                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2641                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2642                         } else {
2643                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2644                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2645                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2646                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2647                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2648                         }
2649                         /* wait for the next frame */
2650                         frame_count = radeon_get_vblank_counter(rdev, i);
2651                         for (j = 0; j < rdev->usec_timeout; j++) {
2652                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2653                                         break;
2654                                 udelay(1);
2655                         }
2656                 }
2657         }
2658         if (!ASIC_IS_NODCE(rdev)) {
2659                 /* Unlock vga access */
2660                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2661                 mdelay(1);
2662                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2663         }
2664 }
2665
2666 void evergreen_mc_program(struct radeon_device *rdev)
2667 {
2668         struct evergreen_mc_save save;
2669         u32 tmp;
2670         int i, j;
2671
2672         /* Initialize HDP */
2673         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2674                 WREG32((0x2c14 + j), 0x00000000);
2675                 WREG32((0x2c18 + j), 0x00000000);
2676                 WREG32((0x2c1c + j), 0x00000000);
2677                 WREG32((0x2c20 + j), 0x00000000);
2678                 WREG32((0x2c24 + j), 0x00000000);
2679         }
2680         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2681
2682         evergreen_mc_stop(rdev, &save);
2683         if (evergreen_mc_wait_for_idle(rdev)) {
2684                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2685         }
2686         /* Lockout access through VGA aperture*/
2687         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2688         /* Update configuration */
2689         if (rdev->flags & RADEON_IS_AGP) {
2690                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2691                         /* VRAM before AGP */
2692                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2693                                 rdev->mc.vram_start >> 12);
2694                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2695                                 rdev->mc.gtt_end >> 12);
2696                 } else {
2697                         /* VRAM after AGP */
2698                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2699                                 rdev->mc.gtt_start >> 12);
2700                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2701                                 rdev->mc.vram_end >> 12);
2702                 }
2703         } else {
2704                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2705                         rdev->mc.vram_start >> 12);
2706                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2707                         rdev->mc.vram_end >> 12);
2708         }
2709         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2710         /* llano/ontario only */
2711         if ((rdev->family == CHIP_PALM) ||
2712             (rdev->family == CHIP_SUMO) ||
2713             (rdev->family == CHIP_SUMO2)) {
2714                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2715                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2716                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2717                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2718         }
2719         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2720         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2721         WREG32(MC_VM_FB_LOCATION, tmp);
2722         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2723         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2724         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2725         if (rdev->flags & RADEON_IS_AGP) {
2726                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2727                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2728                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2729         } else {
2730                 WREG32(MC_VM_AGP_BASE, 0);
2731                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2732                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2733         }
2734         if (evergreen_mc_wait_for_idle(rdev)) {
2735                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2736         }
2737         evergreen_mc_resume(rdev, &save);
2738         /* we need to own VRAM, so turn off the VGA renderer here
2739          * to stop it overwriting our objects */
2740         rv515_vga_render_disable(rdev);
2741 }
2742
2743 /*
2744  * CP.
2745  */
2746 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2747 {
2748         struct radeon_ring *ring = &rdev->ring[ib->ring];
2749         u32 next_rptr;
2750
2751         /* set to DX10/11 mode */
2752         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2753         radeon_ring_write(ring, 1);
2754
2755         if (ring->rptr_save_reg) {
2756                 next_rptr = ring->wptr + 3 + 4;
2757                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2758                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2759                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2760                 radeon_ring_write(ring, next_rptr);
2761         } else if (rdev->wb.enabled) {
2762                 next_rptr = ring->wptr + 5 + 4;
2763                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2764                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2765                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2766                 radeon_ring_write(ring, next_rptr);
2767                 radeon_ring_write(ring, 0);
2768         }
2769
2770         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2771         radeon_ring_write(ring,
2772 #ifdef __BIG_ENDIAN
2773                           (2 << 0) |
2774 #endif
2775                           (ib->gpu_addr & 0xFFFFFFFC));
2776         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2777         radeon_ring_write(ring, ib->length_dw);
2778 }
2779
2780
2781 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2782 {
2783         const __be32 *fw_data;
2784         int i;
2785
2786         if (!rdev->me_fw || !rdev->pfp_fw)
2787                 return -EINVAL;
2788
2789         r700_cp_stop(rdev);
2790         WREG32(CP_RB_CNTL,
2791 #ifdef __BIG_ENDIAN
2792                BUF_SWAP_32BIT |
2793 #endif
2794                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2795
2796         fw_data = (const __be32 *)rdev->pfp_fw->data;
2797         WREG32(CP_PFP_UCODE_ADDR, 0);
2798         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2799                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2800         WREG32(CP_PFP_UCODE_ADDR, 0);
2801
2802         fw_data = (const __be32 *)rdev->me_fw->data;
2803         WREG32(CP_ME_RAM_WADDR, 0);
2804         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2805                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2806
2807         WREG32(CP_PFP_UCODE_ADDR, 0);
2808         WREG32(CP_ME_RAM_WADDR, 0);
2809         WREG32(CP_ME_RAM_RADDR, 0);
2810         return 0;
2811 }
2812
2813 static int evergreen_cp_start(struct radeon_device *rdev)
2814 {
2815         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2816         int r, i;
2817         uint32_t cp_me;
2818
2819         r = radeon_ring_lock(rdev, ring, 7);
2820         if (r) {
2821                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2822                 return r;
2823         }
2824         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2825         radeon_ring_write(ring, 0x1);
2826         radeon_ring_write(ring, 0x0);
2827         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2828         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2829         radeon_ring_write(ring, 0);
2830         radeon_ring_write(ring, 0);
2831         radeon_ring_unlock_commit(rdev, ring);
2832
2833         cp_me = 0xff;
2834         WREG32(CP_ME_CNTL, cp_me);
2835
2836         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2837         if (r) {
2838                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2839                 return r;
2840         }
2841
2842         /* setup clear context state */
2843         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2844         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2845
2846         for (i = 0; i < evergreen_default_size; i++)
2847                 radeon_ring_write(ring, evergreen_default_state[i]);
2848
2849         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2850         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2851
2852         /* set clear context state */
2853         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2854         radeon_ring_write(ring, 0);
2855
2856         /* SQ_VTX_BASE_VTX_LOC */
2857         radeon_ring_write(ring, 0xc0026f00);
2858         radeon_ring_write(ring, 0x00000000);
2859         radeon_ring_write(ring, 0x00000000);
2860         radeon_ring_write(ring, 0x00000000);
2861
2862         /* Clear consts */
2863         radeon_ring_write(ring, 0xc0036f00);
2864         radeon_ring_write(ring, 0x00000bc4);
2865         radeon_ring_write(ring, 0xffffffff);
2866         radeon_ring_write(ring, 0xffffffff);
2867         radeon_ring_write(ring, 0xffffffff);
2868
2869         radeon_ring_write(ring, 0xc0026900);
2870         radeon_ring_write(ring, 0x00000316);
2871         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2872         radeon_ring_write(ring, 0x00000010); /*  */
2873
2874         radeon_ring_unlock_commit(rdev, ring);
2875
2876         return 0;
2877 }
2878
2879 static int evergreen_cp_resume(struct radeon_device *rdev)
2880 {
2881         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2882         u32 tmp;
2883         u32 rb_bufsz;
2884         int r;
2885
2886         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2887         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2888                                  SOFT_RESET_PA |
2889                                  SOFT_RESET_SH |
2890                                  SOFT_RESET_VGT |
2891                                  SOFT_RESET_SPI |
2892                                  SOFT_RESET_SX));
2893         RREG32(GRBM_SOFT_RESET);
2894         mdelay(15);
2895         WREG32(GRBM_SOFT_RESET, 0);
2896         RREG32(GRBM_SOFT_RESET);
2897
2898         /* Set ring buffer size */
2899         rb_bufsz = order_base_2(ring->ring_size / 8);
2900         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2901 #ifdef __BIG_ENDIAN
2902         tmp |= BUF_SWAP_32BIT;
2903 #endif
2904         WREG32(CP_RB_CNTL, tmp);
2905         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2906         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2907
2908         /* Set the write pointer delay */
2909         WREG32(CP_RB_WPTR_DELAY, 0);
2910
2911         /* Initialize the ring buffer's read and write pointers */
2912         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2913         WREG32(CP_RB_RPTR_WR, 0);
2914         ring->wptr = 0;
2915         WREG32(CP_RB_WPTR, ring->wptr);
2916
2917         /* set the wb address whether it's enabled or not */
2918         WREG32(CP_RB_RPTR_ADDR,
2919                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2920         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2921         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2922
2923         if (rdev->wb.enabled)
2924                 WREG32(SCRATCH_UMSK, 0xff);
2925         else {
2926                 tmp |= RB_NO_UPDATE;
2927                 WREG32(SCRATCH_UMSK, 0);
2928         }
2929
2930         mdelay(1);
2931         WREG32(CP_RB_CNTL, tmp);
2932
2933         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2934         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2935
2936         ring->rptr = RREG32(CP_RB_RPTR);
2937
2938         evergreen_cp_start(rdev);
2939         ring->ready = true;
2940         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2941         if (r) {
2942                 ring->ready = false;
2943                 return r;
2944         }
2945         return 0;
2946 }
2947
2948 /*
2949  * Core functions
2950  */
2951 static void evergreen_gpu_init(struct radeon_device *rdev)
2952 {
2953         u32 gb_addr_config;
2954         u32 mc_shared_chmap, mc_arb_ramcfg;
2955         u32 sx_debug_1;
2956         u32 smx_dc_ctl0;
2957         u32 sq_config;
2958         u32 sq_lds_resource_mgmt;
2959         u32 sq_gpr_resource_mgmt_1;
2960         u32 sq_gpr_resource_mgmt_2;
2961         u32 sq_gpr_resource_mgmt_3;
2962         u32 sq_thread_resource_mgmt;
2963         u32 sq_thread_resource_mgmt_2;
2964         u32 sq_stack_resource_mgmt_1;
2965         u32 sq_stack_resource_mgmt_2;
2966         u32 sq_stack_resource_mgmt_3;
2967         u32 vgt_cache_invalidation;
2968         u32 hdp_host_path_cntl, tmp;
2969         u32 disabled_rb_mask;
2970         int i, j, num_shader_engines, ps_thread_count;
2971
2972         switch (rdev->family) {
2973         case CHIP_CYPRESS:
2974         case CHIP_HEMLOCK:
2975                 rdev->config.evergreen.num_ses = 2;
2976                 rdev->config.evergreen.max_pipes = 4;
2977                 rdev->config.evergreen.max_tile_pipes = 8;
2978                 rdev->config.evergreen.max_simds = 10;
2979                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2980                 rdev->config.evergreen.max_gprs = 256;
2981                 rdev->config.evergreen.max_threads = 248;
2982                 rdev->config.evergreen.max_gs_threads = 32;
2983                 rdev->config.evergreen.max_stack_entries = 512;
2984                 rdev->config.evergreen.sx_num_of_sets = 4;
2985                 rdev->config.evergreen.sx_max_export_size = 256;
2986                 rdev->config.evergreen.sx_max_export_pos_size = 64;
2987                 rdev->config.evergreen.sx_max_export_smx_size = 192;
2988                 rdev->config.evergreen.max_hw_contexts = 8;
2989                 rdev->config.evergreen.sq_num_cf_insts = 2;
2990
2991                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2992                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2993                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2994                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2995                 break;
2996         case CHIP_JUNIPER:
2997                 rdev->config.evergreen.num_ses = 1;
2998                 rdev->config.evergreen.max_pipes = 4;
2999                 rdev->config.evergreen.max_tile_pipes = 4;
3000                 rdev->config.evergreen.max_simds = 10;
3001                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3002                 rdev->config.evergreen.max_gprs = 256;
3003                 rdev->config.evergreen.max_threads = 248;
3004                 rdev->config.evergreen.max_gs_threads = 32;
3005                 rdev->config.evergreen.max_stack_entries = 512;
3006                 rdev->config.evergreen.sx_num_of_sets = 4;
3007                 rdev->config.evergreen.sx_max_export_size = 256;
3008                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3009                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3010                 rdev->config.evergreen.max_hw_contexts = 8;
3011                 rdev->config.evergreen.sq_num_cf_insts = 2;
3012
3013                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3014                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3015                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3016                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3017                 break;
3018         case CHIP_REDWOOD:
3019                 rdev->config.evergreen.num_ses = 1;
3020                 rdev->config.evergreen.max_pipes = 4;
3021                 rdev->config.evergreen.max_tile_pipes = 4;
3022                 rdev->config.evergreen.max_simds = 5;
3023                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3024                 rdev->config.evergreen.max_gprs = 256;
3025                 rdev->config.evergreen.max_threads = 248;
3026                 rdev->config.evergreen.max_gs_threads = 32;
3027                 rdev->config.evergreen.max_stack_entries = 256;
3028                 rdev->config.evergreen.sx_num_of_sets = 4;
3029                 rdev->config.evergreen.sx_max_export_size = 256;
3030                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3031                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3032                 rdev->config.evergreen.max_hw_contexts = 8;
3033                 rdev->config.evergreen.sq_num_cf_insts = 2;
3034
3035                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3036                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3037                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3038                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3039                 break;
3040         case CHIP_CEDAR:
3041         default:
3042                 rdev->config.evergreen.num_ses = 1;
3043                 rdev->config.evergreen.max_pipes = 2;
3044                 rdev->config.evergreen.max_tile_pipes = 2;
3045                 rdev->config.evergreen.max_simds = 2;
3046                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3047                 rdev->config.evergreen.max_gprs = 256;
3048                 rdev->config.evergreen.max_threads = 192;
3049                 rdev->config.evergreen.max_gs_threads = 16;
3050                 rdev->config.evergreen.max_stack_entries = 256;
3051                 rdev->config.evergreen.sx_num_of_sets = 4;
3052                 rdev->config.evergreen.sx_max_export_size = 128;
3053                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3054                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3055                 rdev->config.evergreen.max_hw_contexts = 4;
3056                 rdev->config.evergreen.sq_num_cf_insts = 1;
3057
3058                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3059                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3060                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3061                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3062                 break;
3063         case CHIP_PALM:
3064                 rdev->config.evergreen.num_ses = 1;
3065                 rdev->config.evergreen.max_pipes = 2;
3066                 rdev->config.evergreen.max_tile_pipes = 2;
3067                 rdev->config.evergreen.max_simds = 2;
3068                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3069                 rdev->config.evergreen.max_gprs = 256;
3070                 rdev->config.evergreen.max_threads = 192;
3071                 rdev->config.evergreen.max_gs_threads = 16;
3072                 rdev->config.evergreen.max_stack_entries = 256;
3073                 rdev->config.evergreen.sx_num_of_sets = 4;
3074                 rdev->config.evergreen.sx_max_export_size = 128;
3075                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3076                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3077                 rdev->config.evergreen.max_hw_contexts = 4;
3078                 rdev->config.evergreen.sq_num_cf_insts = 1;
3079
3080                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3081                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3082                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3083                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3084                 break;
3085         case CHIP_SUMO:
3086                 rdev->config.evergreen.num_ses = 1;
3087                 rdev->config.evergreen.max_pipes = 4;
3088                 rdev->config.evergreen.max_tile_pipes = 4;
3089                 if (rdev->pdev->device == 0x9648)
3090                         rdev->config.evergreen.max_simds = 3;
3091                 else if ((rdev->pdev->device == 0x9647) ||
3092                          (rdev->pdev->device == 0x964a))
3093                         rdev->config.evergreen.max_simds = 4;
3094                 else
3095                         rdev->config.evergreen.max_simds = 5;
3096                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3097                 rdev->config.evergreen.max_gprs = 256;
3098                 rdev->config.evergreen.max_threads = 248;
3099                 rdev->config.evergreen.max_gs_threads = 32;
3100                 rdev->config.evergreen.max_stack_entries = 256;
3101                 rdev->config.evergreen.sx_num_of_sets = 4;
3102                 rdev->config.evergreen.sx_max_export_size = 256;
3103                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3104                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3105                 rdev->config.evergreen.max_hw_contexts = 8;
3106                 rdev->config.evergreen.sq_num_cf_insts = 2;
3107
3108                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3109                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3110                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3111                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3112                 break;
3113         case CHIP_SUMO2:
3114                 rdev->config.evergreen.num_ses = 1;
3115                 rdev->config.evergreen.max_pipes = 4;
3116                 rdev->config.evergreen.max_tile_pipes = 4;
3117                 rdev->config.evergreen.max_simds = 2;
3118                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3119                 rdev->config.evergreen.max_gprs = 256;
3120                 rdev->config.evergreen.max_threads = 248;
3121                 rdev->config.evergreen.max_gs_threads = 32;
3122                 rdev->config.evergreen.max_stack_entries = 512;
3123                 rdev->config.evergreen.sx_num_of_sets = 4;
3124                 rdev->config.evergreen.sx_max_export_size = 256;
3125                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3126                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3127                 rdev->config.evergreen.max_hw_contexts = 4;
3128                 rdev->config.evergreen.sq_num_cf_insts = 2;
3129
3130                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3131                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3132                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3133                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3134                 break;
3135         case CHIP_BARTS:
3136                 rdev->config.evergreen.num_ses = 2;
3137                 rdev->config.evergreen.max_pipes = 4;
3138                 rdev->config.evergreen.max_tile_pipes = 8;
3139                 rdev->config.evergreen.max_simds = 7;
3140                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3141                 rdev->config.evergreen.max_gprs = 256;
3142                 rdev->config.evergreen.max_threads = 248;
3143                 rdev->config.evergreen.max_gs_threads = 32;
3144                 rdev->config.evergreen.max_stack_entries = 512;
3145                 rdev->config.evergreen.sx_num_of_sets = 4;
3146                 rdev->config.evergreen.sx_max_export_size = 256;
3147                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3148                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3149                 rdev->config.evergreen.max_hw_contexts = 8;
3150                 rdev->config.evergreen.sq_num_cf_insts = 2;
3151
3152                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3153                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3154                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3155                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3156                 break;
3157         case CHIP_TURKS:
3158                 rdev->config.evergreen.num_ses = 1;
3159                 rdev->config.evergreen.max_pipes = 4;
3160                 rdev->config.evergreen.max_tile_pipes = 4;
3161                 rdev->config.evergreen.max_simds = 6;
3162                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3163                 rdev->config.evergreen.max_gprs = 256;
3164                 rdev->config.evergreen.max_threads = 248;
3165                 rdev->config.evergreen.max_gs_threads = 32;
3166                 rdev->config.evergreen.max_stack_entries = 256;
3167                 rdev->config.evergreen.sx_num_of_sets = 4;
3168                 rdev->config.evergreen.sx_max_export_size = 256;
3169                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3170                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3171                 rdev->config.evergreen.max_hw_contexts = 8;
3172                 rdev->config.evergreen.sq_num_cf_insts = 2;
3173
3174                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3175                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3177                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3178                 break;
3179         case CHIP_CAICOS:
3180                 rdev->config.evergreen.num_ses = 1;
3181                 rdev->config.evergreen.max_pipes = 2;
3182                 rdev->config.evergreen.max_tile_pipes = 2;
3183                 rdev->config.evergreen.max_simds = 2;
3184                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3185                 rdev->config.evergreen.max_gprs = 256;
3186                 rdev->config.evergreen.max_threads = 192;
3187                 rdev->config.evergreen.max_gs_threads = 16;
3188                 rdev->config.evergreen.max_stack_entries = 256;
3189                 rdev->config.evergreen.sx_num_of_sets = 4;
3190                 rdev->config.evergreen.sx_max_export_size = 128;
3191                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3192                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3193                 rdev->config.evergreen.max_hw_contexts = 4;
3194                 rdev->config.evergreen.sq_num_cf_insts = 1;
3195
3196                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3197                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3198                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3199                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3200                 break;
3201         }
3202
3203         /* Initialize HDP */
3204         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3205                 WREG32((0x2c14 + j), 0x00000000);
3206                 WREG32((0x2c18 + j), 0x00000000);
3207                 WREG32((0x2c1c + j), 0x00000000);
3208                 WREG32((0x2c20 + j), 0x00000000);
3209                 WREG32((0x2c24 + j), 0x00000000);
3210         }
3211
3212         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3213
3214         evergreen_fix_pci_max_read_req_size(rdev);
3215
3216         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3217         if ((rdev->family == CHIP_PALM) ||
3218             (rdev->family == CHIP_SUMO) ||
3219             (rdev->family == CHIP_SUMO2))
3220                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3221         else
3222                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3223
3224         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3225          * not have bank info, so create a custom tiling dword.
3226          * bits 3:0   num_pipes
3227          * bits 7:4   num_banks
3228          * bits 11:8  group_size
3229          * bits 15:12 row_size
3230          */
3231         rdev->config.evergreen.tile_config = 0;
3232         switch (rdev->config.evergreen.max_tile_pipes) {
3233         case 1:
3234         default:
3235                 rdev->config.evergreen.tile_config |= (0 << 0);
3236                 break;
3237         case 2:
3238                 rdev->config.evergreen.tile_config |= (1 << 0);
3239                 break;
3240         case 4:
3241                 rdev->config.evergreen.tile_config |= (2 << 0);
3242                 break;
3243         case 8:
3244                 rdev->config.evergreen.tile_config |= (3 << 0);
3245                 break;
3246         }
3247         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3248         if (rdev->flags & RADEON_IS_IGP)
3249                 rdev->config.evergreen.tile_config |= 1 << 4;
3250         else {
3251                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3252                 case 0: /* four banks */
3253                         rdev->config.evergreen.tile_config |= 0 << 4;
3254                         break;
3255                 case 1: /* eight banks */
3256                         rdev->config.evergreen.tile_config |= 1 << 4;
3257                         break;
3258                 case 2: /* sixteen banks */
3259                 default:
3260                         rdev->config.evergreen.tile_config |= 2 << 4;
3261                         break;
3262                 }
3263         }
3264         rdev->config.evergreen.tile_config |= 0 << 8;
3265         rdev->config.evergreen.tile_config |=
3266                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3267
3268         num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3269
3270         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3271                 u32 efuse_straps_4;
3272                 u32 efuse_straps_3;
3273
3274                 efuse_straps_4 = RREG32_RCU(0x204);
3275                 efuse_straps_3 = RREG32_RCU(0x203);
3276                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3277                       ((efuse_straps_3 & 0xf0000000) >> 28));
3278         } else {
3279                 tmp = 0;
3280                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3281                         u32 rb_disable_bitmap;
3282
3283                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3284                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3285                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3286                         tmp <<= 4;
3287                         tmp |= rb_disable_bitmap;
3288                 }
3289         }
3290         /* enabled rb are just the one not disabled :) */
3291         disabled_rb_mask = tmp;
3292         tmp = 0;
3293         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3294                 tmp |= (1 << i);
3295         /* if all the backends are disabled, fix it up here */
3296         if ((disabled_rb_mask & tmp) == tmp) {
3297                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3298                         disabled_rb_mask &= ~(1 << i);
3299         }
3300
3301         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3302         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3303
3304         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3305         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3306         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3307         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3308         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3309         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3310         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3311
3312         if ((rdev->config.evergreen.max_backends == 1) &&
3313             (rdev->flags & RADEON_IS_IGP)) {
3314                 if ((disabled_rb_mask & 3) == 1) {
3315                         /* RB0 disabled, RB1 enabled */
3316                         tmp = 0x11111111;
3317                 } else {
3318                         /* RB1 disabled, RB0 enabled */
3319                         tmp = 0x00000000;
3320                 }
3321         } else {
3322                 tmp = gb_addr_config & NUM_PIPES_MASK;
3323                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3324                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3325         }
3326         WREG32(GB_BACKEND_MAP, tmp);
3327
3328         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3329         WREG32(CGTS_TCC_DISABLE, 0);
3330         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3331         WREG32(CGTS_USER_TCC_DISABLE, 0);
3332
3333         /* set HW defaults for 3D engine */
3334         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3335                                      ROQ_IB2_START(0x2b)));
3336
3337         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3338
3339         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3340                              SYNC_GRADIENT |
3341                              SYNC_WALKER |
3342                              SYNC_ALIGNER));
3343
3344         sx_debug_1 = RREG32(SX_DEBUG_1);
3345         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3346         WREG32(SX_DEBUG_1, sx_debug_1);
3347
3348
3349         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3350         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3351         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3352         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3353
3354         if (rdev->family <= CHIP_SUMO2)
3355                 WREG32(SMX_SAR_CTL0, 0x00010000);
3356
3357         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3358                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3359                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3360
3361         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3362                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3363                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3364
3365         WREG32(VGT_NUM_INSTANCES, 1);
3366         WREG32(SPI_CONFIG_CNTL, 0);
3367         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3368         WREG32(CP_PERFMON_CNTL, 0);
3369
3370         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3371                                   FETCH_FIFO_HIWATER(0x4) |
3372                                   DONE_FIFO_HIWATER(0xe0) |
3373                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3374
3375         sq_config = RREG32(SQ_CONFIG);
3376         sq_config &= ~(PS_PRIO(3) |
3377                        VS_PRIO(3) |
3378                        GS_PRIO(3) |
3379                        ES_PRIO(3));
3380         sq_config |= (VC_ENABLE |
3381                       EXPORT_SRC_C |
3382                       PS_PRIO(0) |
3383                       VS_PRIO(1) |
3384                       GS_PRIO(2) |
3385                       ES_PRIO(3));
3386
3387         switch (rdev->family) {
3388         case CHIP_CEDAR:
3389         case CHIP_PALM:
3390         case CHIP_SUMO:
3391         case CHIP_SUMO2:
3392         case CHIP_CAICOS:
3393                 /* no vertex cache */
3394                 sq_config &= ~VC_ENABLE;
3395                 break;
3396         default:
3397                 break;
3398         }
3399
3400         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3401
3402         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3403         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3404         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3405         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3406         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3407         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3408         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3409
3410         switch (rdev->family) {
3411         case CHIP_CEDAR:
3412         case CHIP_PALM:
3413         case CHIP_SUMO:
3414         case CHIP_SUMO2:
3415                 ps_thread_count = 96;
3416                 break;
3417         default:
3418                 ps_thread_count = 128;
3419                 break;
3420         }
3421
3422         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3423         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3424         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3425         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3426         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3427         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3428
3429         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3430         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3431         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3432         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3433         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3434         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3435
3436         WREG32(SQ_CONFIG, sq_config);
3437         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3438         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3439         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3440         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3441         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3442         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3443         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3444         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3445         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3446         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3447
3448         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3449                                           FORCE_EOV_MAX_REZ_CNT(255)));
3450
3451         switch (rdev->family) {
3452         case CHIP_CEDAR:
3453         case CHIP_PALM:
3454         case CHIP_SUMO:
3455         case CHIP_SUMO2:
3456         case CHIP_CAICOS:
3457                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3458                 break;
3459         default:
3460                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3461                 break;
3462         }
3463         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3464         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3465
3466         WREG32(VGT_GS_VERTEX_REUSE, 16);
3467         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3468         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3469
3470         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3471         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3472
3473         WREG32(CB_PERF_CTR0_SEL_0, 0);
3474         WREG32(CB_PERF_CTR0_SEL_1, 0);
3475         WREG32(CB_PERF_CTR1_SEL_0, 0);
3476         WREG32(CB_PERF_CTR1_SEL_1, 0);
3477         WREG32(CB_PERF_CTR2_SEL_0, 0);
3478         WREG32(CB_PERF_CTR2_SEL_1, 0);
3479         WREG32(CB_PERF_CTR3_SEL_0, 0);
3480         WREG32(CB_PERF_CTR3_SEL_1, 0);
3481
3482         /* clear render buffer base addresses */
3483         WREG32(CB_COLOR0_BASE, 0);
3484         WREG32(CB_COLOR1_BASE, 0);
3485         WREG32(CB_COLOR2_BASE, 0);
3486         WREG32(CB_COLOR3_BASE, 0);
3487         WREG32(CB_COLOR4_BASE, 0);
3488         WREG32(CB_COLOR5_BASE, 0);
3489         WREG32(CB_COLOR6_BASE, 0);
3490         WREG32(CB_COLOR7_BASE, 0);
3491         WREG32(CB_COLOR8_BASE, 0);
3492         WREG32(CB_COLOR9_BASE, 0);
3493         WREG32(CB_COLOR10_BASE, 0);
3494         WREG32(CB_COLOR11_BASE, 0);
3495
3496         /* set the shader const cache sizes to 0 */
3497         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3498                 WREG32(i, 0);
3499         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3500                 WREG32(i, 0);
3501
3502         tmp = RREG32(HDP_MISC_CNTL);
3503         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3504         WREG32(HDP_MISC_CNTL, tmp);
3505
3506         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3507         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3508
3509         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3510
3511         udelay(50);
3512
3513 }
3514
3515 int evergreen_mc_init(struct radeon_device *rdev)
3516 {
3517         u32 tmp;
3518         int chansize, numchan;
3519
3520         /* Get VRAM informations */
3521         rdev->mc.vram_is_ddr = true;
3522         if ((rdev->family == CHIP_PALM) ||
3523             (rdev->family == CHIP_SUMO) ||
3524             (rdev->family == CHIP_SUMO2))
3525                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3526         else
3527                 tmp = RREG32(MC_ARB_RAMCFG);
3528         if (tmp & CHANSIZE_OVERRIDE) {
3529                 chansize = 16;
3530         } else if (tmp & CHANSIZE_MASK) {
3531                 chansize = 64;
3532         } else {
3533                 chansize = 32;
3534         }
3535         tmp = RREG32(MC_SHARED_CHMAP);
3536         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3537         case 0:
3538         default:
3539                 numchan = 1;
3540                 break;
3541         case 1:
3542                 numchan = 2;
3543                 break;
3544         case 2:
3545                 numchan = 4;
3546                 break;
3547         case 3:
3548                 numchan = 8;
3549                 break;
3550         }
3551         rdev->mc.vram_width = numchan * chansize;
3552         /* Could aper size report 0 ? */
3553         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3554         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3555         /* Setup GPU memory space */
3556         if ((rdev->family == CHIP_PALM) ||
3557             (rdev->family == CHIP_SUMO) ||
3558             (rdev->family == CHIP_SUMO2)) {
3559                 /* size in bytes on fusion */
3560                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3561                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3562         } else {
3563                 /* size in MB on evergreen/cayman/tn */
3564                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3565                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3566         }
3567         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3568         r700_vram_gtt_location(rdev, &rdev->mc);
3569         radeon_update_bandwidth_info(rdev);
3570
3571         return 0;
3572 }
3573
3574 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3575 {
3576         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3577                 RREG32(GRBM_STATUS));
3578         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3579                 RREG32(GRBM_STATUS_SE0));
3580         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3581                 RREG32(GRBM_STATUS_SE1));
3582         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3583                 RREG32(SRBM_STATUS));
3584         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3585                 RREG32(SRBM_STATUS2));
3586         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3587                 RREG32(CP_STALLED_STAT1));
3588         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3589                 RREG32(CP_STALLED_STAT2));
3590         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3591                 RREG32(CP_BUSY_STAT));
3592         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3593                 RREG32(CP_STAT));
3594         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3595                 RREG32(DMA_STATUS_REG));
3596         if (rdev->family >= CHIP_CAYMAN) {
3597                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3598                          RREG32(DMA_STATUS_REG + 0x800));
3599         }
3600 }
3601
3602 bool evergreen_is_display_hung(struct radeon_device *rdev)
3603 {
3604         u32 crtc_hung = 0;
3605         u32 crtc_status[6];
3606         u32 i, j, tmp;
3607
3608         for (i = 0; i < rdev->num_crtc; i++) {
3609                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3610                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3611                         crtc_hung |= (1 << i);
3612                 }
3613         }
3614
3615         for (j = 0; j < 10; j++) {
3616                 for (i = 0; i < rdev->num_crtc; i++) {
3617                         if (crtc_hung & (1 << i)) {
3618                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3619                                 if (tmp != crtc_status[i])
3620                                         crtc_hung &= ~(1 << i);
3621                         }
3622                 }
3623                 if (crtc_hung == 0)
3624                         return false;
3625                 udelay(100);
3626         }
3627
3628         return true;
3629 }
3630
3631 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3632 {
3633         u32 reset_mask = 0;
3634         u32 tmp;
3635
3636         /* GRBM_STATUS */
3637         tmp = RREG32(GRBM_STATUS);
3638         if (tmp & (PA_BUSY | SC_BUSY |
3639                    SH_BUSY | SX_BUSY |
3640                    TA_BUSY | VGT_BUSY |
3641                    DB_BUSY | CB_BUSY |
3642                    SPI_BUSY | VGT_BUSY_NO_DMA))
3643                 reset_mask |= RADEON_RESET_GFX;
3644
3645         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3646                    CP_BUSY | CP_COHERENCY_BUSY))
3647                 reset_mask |= RADEON_RESET_CP;
3648
3649         if (tmp & GRBM_EE_BUSY)
3650                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3651
3652         /* DMA_STATUS_REG */
3653         tmp = RREG32(DMA_STATUS_REG);
3654         if (!(tmp & DMA_IDLE))
3655                 reset_mask |= RADEON_RESET_DMA;
3656
3657         /* SRBM_STATUS2 */
3658         tmp = RREG32(SRBM_STATUS2);
3659         if (tmp & DMA_BUSY)
3660                 reset_mask |= RADEON_RESET_DMA;
3661
3662         /* SRBM_STATUS */
3663         tmp = RREG32(SRBM_STATUS);
3664         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3665                 reset_mask |= RADEON_RESET_RLC;
3666
3667         if (tmp & IH_BUSY)
3668                 reset_mask |= RADEON_RESET_IH;
3669
3670         if (tmp & SEM_BUSY)
3671                 reset_mask |= RADEON_RESET_SEM;
3672
3673         if (tmp & GRBM_RQ_PENDING)
3674                 reset_mask |= RADEON_RESET_GRBM;
3675
3676         if (tmp & VMC_BUSY)
3677                 reset_mask |= RADEON_RESET_VMC;
3678
3679         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3680                    MCC_BUSY | MCD_BUSY))
3681                 reset_mask |= RADEON_RESET_MC;
3682
3683         if (evergreen_is_display_hung(rdev))
3684                 reset_mask |= RADEON_RESET_DISPLAY;
3685
3686         /* VM_L2_STATUS */
3687         tmp = RREG32(VM_L2_STATUS);
3688         if (tmp & L2_BUSY)
3689                 reset_mask |= RADEON_RESET_VMC;
3690
3691         /* Skip MC reset as it's mostly likely not hung, just busy */
3692         if (reset_mask & RADEON_RESET_MC) {
3693                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3694                 reset_mask &= ~RADEON_RESET_MC;
3695         }
3696
3697         return reset_mask;
3698 }
3699
3700 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3701 {
3702         struct evergreen_mc_save save;
3703         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3704         u32 tmp;
3705
3706         if (reset_mask == 0)
3707                 return;
3708
3709         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3710
3711         evergreen_print_gpu_status_regs(rdev);
3712
3713         /* Disable CP parsing/prefetching */
3714         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3715
3716         if (reset_mask & RADEON_RESET_DMA) {
3717                 /* Disable DMA */
3718                 tmp = RREG32(DMA_RB_CNTL);
3719                 tmp &= ~DMA_RB_ENABLE;
3720                 WREG32(DMA_RB_CNTL, tmp);
3721         }
3722
3723         udelay(50);
3724
3725         evergreen_mc_stop(rdev, &save);
3726         if (evergreen_mc_wait_for_idle(rdev)) {
3727                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3728         }
3729
3730         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3731                 grbm_soft_reset |= SOFT_RESET_DB |
3732                         SOFT_RESET_CB |
3733                         SOFT_RESET_PA |
3734                         SOFT_RESET_SC |
3735                         SOFT_RESET_SPI |
3736                         SOFT_RESET_SX |
3737                         SOFT_RESET_SH |
3738                         SOFT_RESET_TC |
3739                         SOFT_RESET_TA |
3740                         SOFT_RESET_VC |
3741                         SOFT_RESET_VGT;
3742         }
3743
3744         if (reset_mask & RADEON_RESET_CP) {
3745                 grbm_soft_reset |= SOFT_RESET_CP |
3746                         SOFT_RESET_VGT;
3747
3748                 srbm_soft_reset |= SOFT_RESET_GRBM;
3749         }
3750
3751         if (reset_mask & RADEON_RESET_DMA)
3752                 srbm_soft_reset |= SOFT_RESET_DMA;
3753
3754         if (reset_mask & RADEON_RESET_DISPLAY)
3755                 srbm_soft_reset |= SOFT_RESET_DC;
3756
3757         if (reset_mask & RADEON_RESET_RLC)
3758                 srbm_soft_reset |= SOFT_RESET_RLC;
3759
3760         if (reset_mask & RADEON_RESET_SEM)
3761                 srbm_soft_reset |= SOFT_RESET_SEM;
3762
3763         if (reset_mask & RADEON_RESET_IH)
3764                 srbm_soft_reset |= SOFT_RESET_IH;
3765
3766         if (reset_mask & RADEON_RESET_GRBM)
3767                 srbm_soft_reset |= SOFT_RESET_GRBM;
3768
3769         if (reset_mask & RADEON_RESET_VMC)
3770                 srbm_soft_reset |= SOFT_RESET_VMC;
3771
3772         if (!(rdev->flags & RADEON_IS_IGP)) {
3773                 if (reset_mask & RADEON_RESET_MC)
3774                         srbm_soft_reset |= SOFT_RESET_MC;
3775         }
3776
3777         if (grbm_soft_reset) {
3778                 tmp = RREG32(GRBM_SOFT_RESET);
3779                 tmp |= grbm_soft_reset;
3780                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3781                 WREG32(GRBM_SOFT_RESET, tmp);
3782                 tmp = RREG32(GRBM_SOFT_RESET);
3783
3784                 udelay(50);
3785
3786                 tmp &= ~grbm_soft_reset;
3787                 WREG32(GRBM_SOFT_RESET, tmp);
3788                 tmp = RREG32(GRBM_SOFT_RESET);
3789         }
3790
3791         if (srbm_soft_reset) {
3792                 tmp = RREG32(SRBM_SOFT_RESET);
3793                 tmp |= srbm_soft_reset;
3794                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3795                 WREG32(SRBM_SOFT_RESET, tmp);
3796                 tmp = RREG32(SRBM_SOFT_RESET);
3797
3798                 udelay(50);
3799
3800                 tmp &= ~srbm_soft_reset;
3801                 WREG32(SRBM_SOFT_RESET, tmp);
3802                 tmp = RREG32(SRBM_SOFT_RESET);
3803         }
3804
3805         /* Wait a little for things to settle down */
3806         udelay(50);
3807
3808         evergreen_mc_resume(rdev, &save);
3809         udelay(50);
3810
3811         evergreen_print_gpu_status_regs(rdev);
3812 }
3813
3814 int evergreen_asic_reset(struct radeon_device *rdev)
3815 {
3816         u32 reset_mask;
3817
3818         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3819
3820         if (reset_mask)
3821                 r600_set_bios_scratch_engine_hung(rdev, true);
3822
3823         evergreen_gpu_soft_reset(rdev, reset_mask);
3824
3825         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3826
3827         if (!reset_mask)
3828                 r600_set_bios_scratch_engine_hung(rdev, false);
3829
3830         return 0;
3831 }
3832
3833 /**
3834  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3835  *
3836  * @rdev: radeon_device pointer
3837  * @ring: radeon_ring structure holding ring information
3838  *
3839  * Check if the GFX engine is locked up.
3840  * Returns true if the engine appears to be locked up, false if not.
3841  */
3842 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3843 {
3844         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3845
3846         if (!(reset_mask & (RADEON_RESET_GFX |
3847                             RADEON_RESET_COMPUTE |
3848                             RADEON_RESET_CP))) {
3849                 radeon_ring_lockup_update(ring);
3850                 return false;
3851         }
3852         /* force CP activities */
3853         radeon_ring_force_activity(rdev, ring);
3854         return radeon_ring_test_lockup(rdev, ring);
3855 }
3856
3857 /*
3858  * RLC
3859  */
3860 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
3861 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
3862
3863 void sumo_rlc_fini(struct radeon_device *rdev)
3864 {
3865         int r;
3866
3867         /* save restore block */
3868         if (rdev->rlc.save_restore_obj) {
3869                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3870                 if (unlikely(r != 0))
3871                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3872                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3873                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3874
3875                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3876                 rdev->rlc.save_restore_obj = NULL;
3877         }
3878
3879         /* clear state block */
3880         if (rdev->rlc.clear_state_obj) {
3881                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3882                 if (unlikely(r != 0))
3883                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3884                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3885                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3886
3887                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3888                 rdev->rlc.clear_state_obj = NULL;
3889         }
3890
3891         /* clear state block */
3892         if (rdev->rlc.cp_table_obj) {
3893                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3894                 if (unlikely(r != 0))
3895                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3896                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3897                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3898
3899                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3900                 rdev->rlc.cp_table_obj = NULL;
3901         }
3902 }
3903
3904 #define CP_ME_TABLE_SIZE    96
3905
3906 int sumo_rlc_init(struct radeon_device *rdev)
3907 {
3908         const u32 *src_ptr;
3909         volatile u32 *dst_ptr;
3910         u32 dws, data, i, j, k, reg_num;
3911         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
3912         u64 reg_list_mc_addr;
3913         const struct cs_section_def *cs_data;
3914         int r;
3915
3916         src_ptr = rdev->rlc.reg_list;
3917         dws = rdev->rlc.reg_list_size;
3918         if (rdev->family >= CHIP_BONAIRE) {
3919                 dws += (5 * 16) + 48 + 48 + 64;
3920         }
3921         cs_data = rdev->rlc.cs_data;
3922
3923         if (src_ptr) {
3924                 /* save restore block */
3925                 if (rdev->rlc.save_restore_obj == NULL) {
3926                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3927                                              RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3928                         if (r) {
3929                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3930                                 return r;
3931                         }
3932                 }
3933
3934                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3935                 if (unlikely(r != 0)) {
3936                         sumo_rlc_fini(rdev);
3937                         return r;
3938                 }
3939                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3940                                   &rdev->rlc.save_restore_gpu_addr);
3941                 if (r) {
3942                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3943                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3944                         sumo_rlc_fini(rdev);
3945                         return r;
3946                 }
3947
3948                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3949                 if (r) {
3950                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3951                         sumo_rlc_fini(rdev);
3952                         return r;
3953                 }
3954                 /* write the sr buffer */
3955                 dst_ptr = rdev->rlc.sr_ptr;
3956                 if (rdev->family >= CHIP_TAHITI) {
3957                         /* SI */
3958                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
3959                                 dst_ptr[i] = src_ptr[i];
3960                 } else {
3961                         /* ON/LN/TN */
3962                         /* format:
3963                          * dw0: (reg2 << 16) | reg1
3964                          * dw1: reg1 save space
3965                          * dw2: reg2 save space
3966                          */
3967                         for (i = 0; i < dws; i++) {
3968                                 data = src_ptr[i] >> 2;
3969                                 i++;
3970                                 if (i < dws)
3971                                         data |= (src_ptr[i] >> 2) << 16;
3972                                 j = (((i - 1) * 3) / 2);
3973                                 dst_ptr[j] = data;
3974                         }
3975                         j = ((i * 3) / 2);
3976                         dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3977                 }
3978                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3979                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3980         }
3981
3982         if (cs_data) {
3983                 /* clear state block */
3984                 if (rdev->family >= CHIP_BONAIRE) {
3985                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
3986                 } else if (rdev->family >= CHIP_TAHITI) {
3987                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
3988                         dws = rdev->rlc.clear_state_size + (256 / 4);
3989                 } else {
3990                         reg_list_num = 0;
3991                         dws = 0;
3992                         for (i = 0; cs_data[i].section != NULL; i++) {
3993                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3994                                         reg_list_num++;
3995                                         dws += cs_data[i].section[j].reg_count;
3996                                 }
3997                         }
3998                         reg_list_blk_index = (3 * reg_list_num + 2);
3999                         dws += reg_list_blk_index;
4000                         rdev->rlc.clear_state_size = dws;
4001                 }
4002
4003                 if (rdev->rlc.clear_state_obj == NULL) {
4004                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4005                                              RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4006                         if (r) {
4007                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4008                                 sumo_rlc_fini(rdev);
4009                                 return r;
4010                         }
4011                 }
4012                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4013                 if (unlikely(r != 0)) {
4014                         sumo_rlc_fini(rdev);
4015                         return r;
4016                 }
4017                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4018                                   &rdev->rlc.clear_state_gpu_addr);
4019                 if (r) {
4020                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4021                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4022                         sumo_rlc_fini(rdev);
4023                         return r;
4024                 }
4025
4026                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4027                 if (r) {
4028                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4029                         sumo_rlc_fini(rdev);
4030                         return r;
4031                 }
4032                 /* set up the cs buffer */
4033                 dst_ptr = rdev->rlc.cs_ptr;
4034                 if (rdev->family >= CHIP_BONAIRE) {
4035                         cik_get_csb_buffer(rdev, dst_ptr);
4036                 } else if (rdev->family >= CHIP_TAHITI) {
4037                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4038                         dst_ptr[0] = upper_32_bits(reg_list_mc_addr);
4039                         dst_ptr[1] = lower_32_bits(reg_list_mc_addr);
4040                         dst_ptr[2] = rdev->rlc.clear_state_size;
4041                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4042                 } else {
4043                         reg_list_hdr_blk_index = 0;
4044                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4045                         data = upper_32_bits(reg_list_mc_addr);
4046                         dst_ptr[reg_list_hdr_blk_index] = data;
4047                         reg_list_hdr_blk_index++;
4048                         for (i = 0; cs_data[i].section != NULL; i++) {
4049                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4050                                         reg_num = cs_data[i].section[j].reg_count;
4051                                         data = reg_list_mc_addr & 0xffffffff;
4052                                         dst_ptr[reg_list_hdr_blk_index] = data;
4053                                         reg_list_hdr_blk_index++;
4054
4055                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4056                                         dst_ptr[reg_list_hdr_blk_index] = data;
4057                                         reg_list_hdr_blk_index++;
4058
4059                                         data = 0x08000000 | (reg_num * 4);
4060                                         dst_ptr[reg_list_hdr_blk_index] = data;
4061                                         reg_list_hdr_blk_index++;
4062
4063                                         for (k = 0; k < reg_num; k++) {
4064                                                 data = cs_data[i].section[j].extent[k];
4065                                                 dst_ptr[reg_list_blk_index + k] = data;
4066                                         }
4067                                         reg_list_mc_addr += reg_num * 4;
4068                                         reg_list_blk_index += reg_num;
4069                                 }
4070                         }
4071                         dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4072                 }
4073                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4074                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4075         }
4076
4077         if (rdev->rlc.cp_table_size) {
4078                 if (rdev->rlc.cp_table_obj == NULL) {
4079                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4080                                              RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4081                         if (r) {
4082                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4083                                 sumo_rlc_fini(rdev);
4084                                 return r;
4085                         }
4086                 }
4087
4088                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4089                 if (unlikely(r != 0)) {
4090                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4091                         sumo_rlc_fini(rdev);
4092                         return r;
4093                 }
4094                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4095                                   &rdev->rlc.cp_table_gpu_addr);
4096                 if (r) {
4097                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4098                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4099                         sumo_rlc_fini(rdev);
4100                         return r;
4101                 }
4102                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4103                 if (r) {
4104                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4105                         sumo_rlc_fini(rdev);
4106                         return r;
4107                 }
4108
4109                 cik_init_cp_pg_table(rdev);
4110
4111                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4112                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4113
4114         }
4115
4116         return 0;
4117 }
4118
4119 static void evergreen_rlc_start(struct radeon_device *rdev)
4120 {
4121         u32 mask = RLC_ENABLE;
4122
4123         if (rdev->flags & RADEON_IS_IGP) {
4124                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4125         }
4126
4127         WREG32(RLC_CNTL, mask);
4128 }
4129
4130 int evergreen_rlc_resume(struct radeon_device *rdev)
4131 {
4132         u32 i;
4133         const __be32 *fw_data;
4134
4135         if (!rdev->rlc_fw)
4136                 return -EINVAL;
4137
4138         r600_rlc_stop(rdev);
4139
4140         WREG32(RLC_HB_CNTL, 0);
4141
4142         if (rdev->flags & RADEON_IS_IGP) {
4143                 if (rdev->family == CHIP_ARUBA) {
4144                         u32 always_on_bitmap =
4145                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4146                         /* find out the number of active simds */
4147                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4148                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4149                         tmp = hweight32(~tmp);
4150                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4151                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4152                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4153                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4154                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4155                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4156                         }
4157                 } else {
4158                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4159                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4160                 }
4161                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4162                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4163         } else {
4164                 WREG32(RLC_HB_BASE, 0);
4165                 WREG32(RLC_HB_RPTR, 0);
4166                 WREG32(RLC_HB_WPTR, 0);
4167                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4168                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4169         }
4170         WREG32(RLC_MC_CNTL, 0);
4171         WREG32(RLC_UCODE_CNTL, 0);
4172
4173         fw_data = (const __be32 *)rdev->rlc_fw->data;
4174         if (rdev->family >= CHIP_ARUBA) {
4175                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4176                         WREG32(RLC_UCODE_ADDR, i);
4177                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4178                 }
4179         } else if (rdev->family >= CHIP_CAYMAN) {
4180                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4181                         WREG32(RLC_UCODE_ADDR, i);
4182                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4183                 }
4184         } else {
4185                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4186                         WREG32(RLC_UCODE_ADDR, i);
4187                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4188                 }
4189         }
4190         WREG32(RLC_UCODE_ADDR, 0);
4191
4192         evergreen_rlc_start(rdev);
4193
4194         return 0;
4195 }
4196
4197 /* Interrupts */
4198
4199 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4200 {
4201         if (crtc >= rdev->num_crtc)
4202                 return 0;
4203         else
4204                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4205 }
4206
4207 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4208 {
4209         u32 tmp;
4210
4211         if (rdev->family >= CHIP_CAYMAN) {
4212                 cayman_cp_int_cntl_setup(rdev, 0,
4213                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4214                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4215                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4216                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4217                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4218         } else
4219                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4220         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4221         WREG32(DMA_CNTL, tmp);
4222         WREG32(GRBM_INT_CNTL, 0);
4223         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4224         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4225         if (rdev->num_crtc >= 4) {
4226                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4227                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4228         }
4229         if (rdev->num_crtc >= 6) {
4230                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4231                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4232         }
4233
4234         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4235         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4236         if (rdev->num_crtc >= 4) {
4237                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4238                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4239         }
4240         if (rdev->num_crtc >= 6) {
4241                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4242                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4243         }
4244
4245         /* only one DAC on DCE6 */
4246         if (!ASIC_IS_DCE6(rdev))
4247                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4248         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4249
4250         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4251         WREG32(DC_HPD1_INT_CONTROL, tmp);
4252         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4253         WREG32(DC_HPD2_INT_CONTROL, tmp);
4254         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4255         WREG32(DC_HPD3_INT_CONTROL, tmp);
4256         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4257         WREG32(DC_HPD4_INT_CONTROL, tmp);
4258         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4259         WREG32(DC_HPD5_INT_CONTROL, tmp);
4260         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4261         WREG32(DC_HPD6_INT_CONTROL, tmp);
4262
4263 }
4264
4265 int evergreen_irq_set(struct radeon_device *rdev)
4266 {
4267         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4268         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4269         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4270         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4271         u32 grbm_int_cntl = 0;
4272         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4273         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4274         u32 dma_cntl, dma_cntl1 = 0;
4275         u32 thermal_int = 0;
4276
4277         if (!rdev->irq.installed) {
4278                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4279                 return -EINVAL;
4280         }
4281         /* don't enable anything if the ih is disabled */
4282         if (!rdev->ih.enabled) {
4283                 r600_disable_interrupts(rdev);
4284                 /* force the active interrupt state to all disabled */
4285                 evergreen_disable_interrupt_state(rdev);
4286                 return 0;
4287         }
4288
4289         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4290         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4291         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4292         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4293         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4294         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4295         if (rdev->family == CHIP_ARUBA)
4296                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4297                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4298         else
4299                 thermal_int = RREG32(CG_THERMAL_INT) &
4300                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4301
4302         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4303         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4304         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4305         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4306         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4307         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4308
4309         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4310
4311         if (rdev->family >= CHIP_CAYMAN) {
4312                 /* enable CP interrupts on all rings */
4313                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4314                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4315                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4316                 }
4317                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4318                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4319                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4320                 }
4321                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4322                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4323                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4324                 }
4325         } else {
4326                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4327                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4328                         cp_int_cntl |= RB_INT_ENABLE;
4329                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4330                 }
4331         }
4332
4333         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4334                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4335                 dma_cntl |= TRAP_ENABLE;
4336         }
4337
4338         if (rdev->family >= CHIP_CAYMAN) {
4339                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4340                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4341                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4342                         dma_cntl1 |= TRAP_ENABLE;
4343                 }
4344         }
4345
4346         if (rdev->irq.dpm_thermal) {
4347                 DRM_DEBUG("dpm thermal\n");
4348                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4349         }
4350
4351         if (rdev->irq.crtc_vblank_int[0] ||
4352             atomic_read(&rdev->irq.pflip[0])) {
4353                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4354                 crtc1 |= VBLANK_INT_MASK;
4355         }
4356         if (rdev->irq.crtc_vblank_int[1] ||
4357             atomic_read(&rdev->irq.pflip[1])) {
4358                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4359                 crtc2 |= VBLANK_INT_MASK;
4360         }
4361         if (rdev->irq.crtc_vblank_int[2] ||
4362             atomic_read(&rdev->irq.pflip[2])) {
4363                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4364                 crtc3 |= VBLANK_INT_MASK;
4365         }
4366         if (rdev->irq.crtc_vblank_int[3] ||
4367             atomic_read(&rdev->irq.pflip[3])) {
4368                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4369                 crtc4 |= VBLANK_INT_MASK;
4370         }
4371         if (rdev->irq.crtc_vblank_int[4] ||
4372             atomic_read(&rdev->irq.pflip[4])) {
4373                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4374                 crtc5 |= VBLANK_INT_MASK;
4375         }
4376         if (rdev->irq.crtc_vblank_int[5] ||
4377             atomic_read(&rdev->irq.pflip[5])) {
4378                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4379                 crtc6 |= VBLANK_INT_MASK;
4380         }
4381         if (rdev->irq.hpd[0]) {
4382                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4383                 hpd1 |= DC_HPDx_INT_EN;
4384         }
4385         if (rdev->irq.hpd[1]) {
4386                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4387                 hpd2 |= DC_HPDx_INT_EN;
4388         }
4389         if (rdev->irq.hpd[2]) {
4390                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4391                 hpd3 |= DC_HPDx_INT_EN;
4392         }
4393         if (rdev->irq.hpd[3]) {
4394                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4395                 hpd4 |= DC_HPDx_INT_EN;
4396         }
4397         if (rdev->irq.hpd[4]) {
4398                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4399                 hpd5 |= DC_HPDx_INT_EN;
4400         }
4401         if (rdev->irq.hpd[5]) {
4402                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4403                 hpd6 |= DC_HPDx_INT_EN;
4404         }
4405         if (rdev->irq.afmt[0]) {
4406                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4407                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4408         }
4409         if (rdev->irq.afmt[1]) {
4410                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4411                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4412         }
4413         if (rdev->irq.afmt[2]) {
4414                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4415                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4416         }
4417         if (rdev->irq.afmt[3]) {
4418                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4419                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4420         }
4421         if (rdev->irq.afmt[4]) {
4422                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4423                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4424         }
4425         if (rdev->irq.afmt[5]) {
4426                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4427                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4428         }
4429
4430         if (rdev->family >= CHIP_CAYMAN) {
4431                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4432                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4433                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4434         } else
4435                 WREG32(CP_INT_CNTL, cp_int_cntl);
4436
4437         WREG32(DMA_CNTL, dma_cntl);
4438
4439         if (rdev->family >= CHIP_CAYMAN)
4440                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4441
4442         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4443
4444         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4445         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4446         if (rdev->num_crtc >= 4) {
4447                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4448                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4449         }
4450         if (rdev->num_crtc >= 6) {
4451                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4452                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4453         }
4454
4455         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4456         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4457         if (rdev->num_crtc >= 4) {
4458                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4459                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4460         }
4461         if (rdev->num_crtc >= 6) {
4462                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4463                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4464         }
4465
4466         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4467         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4468         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4469         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4470         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4471         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4472         if (rdev->family == CHIP_ARUBA)
4473                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4474         else
4475                 WREG32(CG_THERMAL_INT, thermal_int);
4476
4477         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4478         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4479         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4480         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4481         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4482         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4483
4484         return 0;
4485 }
4486
4487 static void evergreen_irq_ack(struct radeon_device *rdev)
4488 {
4489         u32 tmp;
4490
4491         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4492         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4493         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4494         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4495         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4496         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4497         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4498         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4499         if (rdev->num_crtc >= 4) {
4500                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4501                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4502         }
4503         if (rdev->num_crtc >= 6) {
4504                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4505                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4506         }
4507
4508         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4509         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4510         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4511         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4512         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4513         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4514
4515         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4516                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4517         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4518                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4519         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4520                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4521         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4522                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4523         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4524                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4525         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4526                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4527
4528         if (rdev->num_crtc >= 4) {
4529                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4530                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4531                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4532                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4533                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4534                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4535                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4536                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4537                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4538                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4539                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4540                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4541         }
4542
4543         if (rdev->num_crtc >= 6) {
4544                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4545                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4546                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4547                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4548                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4549                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4550                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4551                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4552                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4553                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4554                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4555                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4556         }
4557
4558         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4559                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4560                 tmp |= DC_HPDx_INT_ACK;
4561                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4562         }
4563         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4564                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4565                 tmp |= DC_HPDx_INT_ACK;
4566                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4567         }
4568         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4569                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4570                 tmp |= DC_HPDx_INT_ACK;
4571                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4572         }
4573         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4574                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4575                 tmp |= DC_HPDx_INT_ACK;
4576                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4577         }
4578         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4579                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4580                 tmp |= DC_HPDx_INT_ACK;
4581                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4582         }
4583         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4584                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4585                 tmp |= DC_HPDx_INT_ACK;
4586                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4587         }
4588         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4589                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4590                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4591                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4592         }
4593         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4594                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4595                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4596                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4597         }
4598         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4599                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4600                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4601                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4602         }
4603         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4604                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4605                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4606                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4607         }
4608         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4609                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4610                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4611                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4612         }
4613         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4614                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4615                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4616                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4617         }
4618 }
4619
4620 static void evergreen_irq_disable(struct radeon_device *rdev)
4621 {
4622         r600_disable_interrupts(rdev);
4623         /* Wait and acknowledge irq */
4624         mdelay(1);
4625         evergreen_irq_ack(rdev);
4626         evergreen_disable_interrupt_state(rdev);
4627 }
4628
4629 void evergreen_irq_suspend(struct radeon_device *rdev)
4630 {
4631         evergreen_irq_disable(rdev);
4632         r600_rlc_stop(rdev);
4633 }
4634
4635 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4636 {
4637         u32 wptr, tmp;
4638
4639         if (rdev->wb.enabled)
4640                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4641         else
4642                 wptr = RREG32(IH_RB_WPTR);
4643
4644         if (wptr & RB_OVERFLOW) {
4645                 /* When a ring buffer overflow happen start parsing interrupt
4646                  * from the last not overwritten vector (wptr + 16). Hopefully
4647                  * this should allow us to catchup.
4648                  */
4649                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4650                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4651                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4652                 tmp = RREG32(IH_RB_CNTL);
4653                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4654                 WREG32(IH_RB_CNTL, tmp);
4655         }
4656         return (wptr & rdev->ih.ptr_mask);
4657 }
4658
4659 int evergreen_irq_process(struct radeon_device *rdev)
4660 {
4661         u32 wptr;
4662         u32 rptr;
4663         u32 src_id, src_data;
4664         u32 ring_index;
4665         bool queue_hotplug = false;
4666         bool queue_hdmi = false;
4667         bool queue_thermal = false;
4668         u32 status, addr;
4669
4670         if (!rdev->ih.enabled || rdev->shutdown)
4671                 return IRQ_NONE;
4672
4673         wptr = evergreen_get_ih_wptr(rdev);
4674
4675 restart_ih:
4676         /* is somebody else already processing irqs? */
4677         if (atomic_xchg(&rdev->ih.lock, 1))
4678                 return IRQ_NONE;
4679
4680         rptr = rdev->ih.rptr;
4681         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4682
4683         /* Order reading of wptr vs. reading of IH ring data */
4684         rmb();
4685
4686         /* display interrupts */
4687         evergreen_irq_ack(rdev);
4688
4689         while (rptr != wptr) {
4690                 /* wptr/rptr are in bytes! */
4691                 ring_index = rptr / 4;
4692                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4693                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4694
4695                 switch (src_id) {
4696                 case 1: /* D1 vblank/vline */
4697                         switch (src_data) {
4698                         case 0: /* D1 vblank */
4699                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4700                                         if (rdev->irq.crtc_vblank_int[0]) {
4701                                                 drm_handle_vblank(rdev->ddev, 0);
4702                                                 rdev->pm.vblank_sync = true;
4703                                                 wake_up(&rdev->irq.vblank_queue);
4704                                         }
4705                                         if (atomic_read(&rdev->irq.pflip[0]))
4706                                                 radeon_crtc_handle_flip(rdev, 0);
4707                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4708                                         DRM_DEBUG("IH: D1 vblank\n");
4709                                 }
4710                                 break;
4711                         case 1: /* D1 vline */
4712                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4713                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4714                                         DRM_DEBUG("IH: D1 vline\n");
4715                                 }
4716                                 break;
4717                         default:
4718                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4719                                 break;
4720                         }
4721                         break;
4722                 case 2: /* D2 vblank/vline */
4723                         switch (src_data) {
4724                         case 0: /* D2 vblank */
4725                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4726                                         if (rdev->irq.crtc_vblank_int[1]) {
4727                                                 drm_handle_vblank(rdev->ddev, 1);
4728                                                 rdev->pm.vblank_sync = true;
4729                                                 wake_up(&rdev->irq.vblank_queue);
4730                                         }
4731                                         if (atomic_read(&rdev->irq.pflip[1]))
4732                                                 radeon_crtc_handle_flip(rdev, 1);
4733                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4734                                         DRM_DEBUG("IH: D2 vblank\n");
4735                                 }
4736                                 break;
4737                         case 1: /* D2 vline */
4738                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4739                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4740                                         DRM_DEBUG("IH: D2 vline\n");
4741                                 }
4742                                 break;
4743                         default:
4744                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4745                                 break;
4746                         }
4747                         break;
4748                 case 3: /* D3 vblank/vline */
4749                         switch (src_data) {
4750                         case 0: /* D3 vblank */
4751                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4752                                         if (rdev->irq.crtc_vblank_int[2]) {
4753                                                 drm_handle_vblank(rdev->ddev, 2);
4754                                                 rdev->pm.vblank_sync = true;
4755                                                 wake_up(&rdev->irq.vblank_queue);
4756                                         }
4757                                         if (atomic_read(&rdev->irq.pflip[2]))
4758                                                 radeon_crtc_handle_flip(rdev, 2);
4759                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4760                                         DRM_DEBUG("IH: D3 vblank\n");
4761                                 }
4762                                 break;
4763                         case 1: /* D3 vline */
4764                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4765                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4766                                         DRM_DEBUG("IH: D3 vline\n");
4767                                 }
4768                                 break;
4769                         default:
4770                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4771                                 break;
4772                         }
4773                         break;
4774                 case 4: /* D4 vblank/vline */
4775                         switch (src_data) {
4776                         case 0: /* D4 vblank */
4777                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4778                                         if (rdev->irq.crtc_vblank_int[3]) {
4779                                                 drm_handle_vblank(rdev->ddev, 3);
4780                                                 rdev->pm.vblank_sync = true;
4781                                                 wake_up(&rdev->irq.vblank_queue);
4782                                         }
4783                                         if (atomic_read(&rdev->irq.pflip[3]))
4784                                                 radeon_crtc_handle_flip(rdev, 3);
4785                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4786                                         DRM_DEBUG("IH: D4 vblank\n");
4787                                 }
4788                                 break;
4789                         case 1: /* D4 vline */
4790                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4791                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4792                                         DRM_DEBUG("IH: D4 vline\n");
4793                                 }
4794                                 break;
4795                         default:
4796                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4797                                 break;
4798                         }
4799                         break;
4800                 case 5: /* D5 vblank/vline */
4801                         switch (src_data) {
4802                         case 0: /* D5 vblank */
4803                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4804                                         if (rdev->irq.crtc_vblank_int[4]) {
4805                                                 drm_handle_vblank(rdev->ddev, 4);
4806                                                 rdev->pm.vblank_sync = true;
4807                                                 wake_up(&rdev->irq.vblank_queue);
4808                                         }
4809                                         if (atomic_read(&rdev->irq.pflip[4]))
4810                                                 radeon_crtc_handle_flip(rdev, 4);
4811                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4812                                         DRM_DEBUG("IH: D5 vblank\n");
4813                                 }
4814                                 break;
4815                         case 1: /* D5 vline */
4816                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4817                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4818                                         DRM_DEBUG("IH: D5 vline\n");
4819                                 }
4820                                 break;
4821                         default:
4822                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4823                                 break;
4824                         }
4825                         break;
4826                 case 6: /* D6 vblank/vline */
4827                         switch (src_data) {
4828                         case 0: /* D6 vblank */
4829                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4830                                         if (rdev->irq.crtc_vblank_int[5]) {
4831                                                 drm_handle_vblank(rdev->ddev, 5);
4832                                                 rdev->pm.vblank_sync = true;
4833                                                 wake_up(&rdev->irq.vblank_queue);
4834                                         }
4835                                         if (atomic_read(&rdev->irq.pflip[5]))
4836                                                 radeon_crtc_handle_flip(rdev, 5);
4837                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4838                                         DRM_DEBUG("IH: D6 vblank\n");
4839                                 }
4840                                 break;
4841                         case 1: /* D6 vline */
4842                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4843                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4844                                         DRM_DEBUG("IH: D6 vline\n");
4845                                 }
4846                                 break;
4847                         default:
4848                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4849                                 break;
4850                         }
4851                         break;
4852                 case 42: /* HPD hotplug */
4853                         switch (src_data) {
4854                         case 0:
4855                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4856                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4857                                         queue_hotplug = true;
4858                                         DRM_DEBUG("IH: HPD1\n");
4859                                 }
4860                                 break;
4861                         case 1:
4862                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4863                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4864                                         queue_hotplug = true;
4865                                         DRM_DEBUG("IH: HPD2\n");
4866                                 }
4867                                 break;
4868                         case 2:
4869                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4870                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4871                                         queue_hotplug = true;
4872                                         DRM_DEBUG("IH: HPD3\n");
4873                                 }
4874                                 break;
4875                         case 3:
4876                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4877                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4878                                         queue_hotplug = true;
4879                                         DRM_DEBUG("IH: HPD4\n");
4880                                 }
4881                                 break;
4882                         case 4:
4883                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4884                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4885                                         queue_hotplug = true;
4886                                         DRM_DEBUG("IH: HPD5\n");
4887                                 }
4888                                 break;
4889                         case 5:
4890                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4891                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4892                                         queue_hotplug = true;
4893                                         DRM_DEBUG("IH: HPD6\n");
4894                                 }
4895                                 break;
4896                         default:
4897                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4898                                 break;
4899                         }
4900                         break;
4901                 case 44: /* hdmi */
4902                         switch (src_data) {
4903                         case 0:
4904                                 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4905                                         rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4906                                         queue_hdmi = true;
4907                                         DRM_DEBUG("IH: HDMI0\n");
4908                                 }
4909                                 break;
4910                         case 1:
4911                                 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4912                                         rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4913                                         queue_hdmi = true;
4914                                         DRM_DEBUG("IH: HDMI1\n");
4915                                 }
4916                                 break;
4917                         case 2:
4918                                 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4919                                         rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4920                                         queue_hdmi = true;
4921                                         DRM_DEBUG("IH: HDMI2\n");
4922                                 }
4923                                 break;
4924                         case 3:
4925                                 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4926                                         rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4927                                         queue_hdmi = true;
4928                                         DRM_DEBUG("IH: HDMI3\n");
4929                                 }
4930                                 break;
4931                         case 4:
4932                                 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4933                                         rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4934                                         queue_hdmi = true;
4935                                         DRM_DEBUG("IH: HDMI4\n");
4936                                 }
4937                                 break;
4938                         case 5:
4939                                 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4940                                         rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4941                                         queue_hdmi = true;
4942                                         DRM_DEBUG("IH: HDMI5\n");
4943                                 }
4944                                 break;
4945                         default:
4946                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4947                                 break;
4948                         }
4949                 case 124: /* UVD */
4950                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4951                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4952                         break;
4953                 case 146:
4954                 case 147:
4955                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4956                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4957                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4958                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4959                                 addr);
4960                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4961                                 status);
4962                         cayman_vm_decode_fault(rdev, status, addr);
4963                         /* reset addr and status */
4964                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4965                         break;
4966                 case 176: /* CP_INT in ring buffer */
4967                 case 177: /* CP_INT in IB1 */
4968                 case 178: /* CP_INT in IB2 */
4969                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4970                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4971                         break;
4972                 case 181: /* CP EOP event */
4973                         DRM_DEBUG("IH: CP EOP\n");
4974                         if (rdev->family >= CHIP_CAYMAN) {
4975                                 switch (src_data) {
4976                                 case 0:
4977                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4978                                         break;
4979                                 case 1:
4980                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4981                                         break;
4982                                 case 2:
4983                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4984                                         break;
4985                                 }
4986                         } else
4987                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4988                         break;
4989                 case 224: /* DMA trap event */
4990                         DRM_DEBUG("IH: DMA trap\n");
4991                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4992                         break;
4993                 case 230: /* thermal low to high */
4994                         DRM_DEBUG("IH: thermal low to high\n");
4995                         rdev->pm.dpm.thermal.high_to_low = false;
4996                         queue_thermal = true;
4997                         break;
4998                 case 231: /* thermal high to low */
4999                         DRM_DEBUG("IH: thermal high to low\n");
5000                         rdev->pm.dpm.thermal.high_to_low = true;
5001                         queue_thermal = true;
5002                         break;
5003                 case 233: /* GUI IDLE */
5004                         DRM_DEBUG("IH: GUI idle\n");
5005                         break;
5006                 case 244: /* DMA trap event */
5007                         if (rdev->family >= CHIP_CAYMAN) {
5008                                 DRM_DEBUG("IH: DMA1 trap\n");
5009                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5010                         }
5011                         break;
5012                 default:
5013                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5014                         break;
5015                 }
5016
5017                 /* wptr/rptr are in bytes! */
5018                 rptr += 16;
5019                 rptr &= rdev->ih.ptr_mask;
5020         }
5021         if (queue_hotplug)
5022                 schedule_work(&rdev->hotplug_work);
5023         if (queue_hdmi)
5024                 schedule_work(&rdev->audio_work);
5025         if (queue_thermal && rdev->pm.dpm_enabled)
5026                 schedule_work(&rdev->pm.dpm.thermal.work);
5027         rdev->ih.rptr = rptr;
5028         WREG32(IH_RB_RPTR, rdev->ih.rptr);
5029         atomic_set(&rdev->ih.lock, 0);
5030
5031         /* make sure wptr hasn't changed while processing */
5032         wptr = evergreen_get_ih_wptr(rdev);
5033         if (wptr != rptr)
5034                 goto restart_ih;
5035
5036         return IRQ_HANDLED;
5037 }
5038
5039 static int evergreen_startup(struct radeon_device *rdev)
5040 {
5041         struct radeon_ring *ring;
5042         int r;
5043
5044         /* enable pcie gen2 link */
5045         evergreen_pcie_gen2_enable(rdev);
5046         /* enable aspm */
5047         evergreen_program_aspm(rdev);
5048
5049         /* scratch needs to be initialized before MC */
5050         r = r600_vram_scratch_init(rdev);
5051         if (r)
5052                 return r;
5053
5054         evergreen_mc_program(rdev);
5055
5056         if (ASIC_IS_DCE5(rdev)) {
5057                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5058                         r = ni_init_microcode(rdev);
5059                         if (r) {
5060                                 DRM_ERROR("Failed to load firmware!\n");
5061                                 return r;
5062                         }
5063                 }
5064                 r = ni_mc_load_microcode(rdev);
5065                 if (r) {
5066                         DRM_ERROR("Failed to load MC firmware!\n");
5067                         return r;
5068                 }
5069         } else {
5070                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5071                         r = r600_init_microcode(rdev);
5072                         if (r) {
5073                                 DRM_ERROR("Failed to load firmware!\n");
5074                                 return r;
5075                         }
5076                 }
5077         }
5078
5079         if (rdev->flags & RADEON_IS_AGP) {
5080                 evergreen_agp_enable(rdev);
5081         } else {
5082                 r = evergreen_pcie_gart_enable(rdev);
5083                 if (r)
5084                         return r;
5085         }
5086         evergreen_gpu_init(rdev);
5087
5088         /* allocate rlc buffers */
5089         if (rdev->flags & RADEON_IS_IGP) {
5090                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5091                 rdev->rlc.reg_list_size =
5092                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5093                 rdev->rlc.cs_data = evergreen_cs_data;
5094                 r = sumo_rlc_init(rdev);
5095                 if (r) {
5096                         DRM_ERROR("Failed to init rlc BOs!\n");
5097                         return r;
5098                 }
5099         }
5100
5101         /* allocate wb buffer */
5102         r = radeon_wb_init(rdev);
5103         if (r)
5104                 return r;
5105
5106         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5107         if (r) {
5108                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5109                 return r;
5110         }
5111
5112         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5113         if (r) {
5114                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5115                 return r;
5116         }
5117
5118         r = uvd_v2_2_resume(rdev);
5119         if (!r) {
5120                 r = radeon_fence_driver_start_ring(rdev,
5121                                                    R600_RING_TYPE_UVD_INDEX);
5122                 if (r)
5123                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5124         }
5125
5126         if (r)
5127                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5128
5129         /* Enable IRQ */
5130         if (!rdev->irq.installed) {
5131                 r = radeon_irq_kms_init(rdev);
5132                 if (r)
5133                         return r;
5134         }
5135
5136         r = r600_irq_init(rdev);
5137         if (r) {
5138                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5139                 radeon_irq_kms_fini(rdev);
5140                 return r;
5141         }
5142         evergreen_irq_set(rdev);
5143
5144         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5145         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5146                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5147                              RADEON_CP_PACKET2);
5148         if (r)
5149                 return r;
5150
5151         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5152         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5153                              DMA_RB_RPTR, DMA_RB_WPTR,
5154                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5155         if (r)
5156                 return r;
5157
5158         r = evergreen_cp_load_microcode(rdev);
5159         if (r)
5160                 return r;
5161         r = evergreen_cp_resume(rdev);
5162         if (r)
5163                 return r;
5164         r = r600_dma_resume(rdev);
5165         if (r)
5166                 return r;
5167
5168         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5169         if (ring->ring_size) {
5170                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5171                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5172                                      RADEON_CP_PACKET2);
5173                 if (!r)
5174                         r = uvd_v1_0_init(rdev);
5175
5176                 if (r)
5177                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5178         }
5179
5180         r = radeon_ib_pool_init(rdev);
5181         if (r) {
5182                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5183                 return r;
5184         }
5185
5186         r = r600_audio_init(rdev);
5187         if (r) {
5188                 DRM_ERROR("radeon: audio init failed\n");
5189                 return r;
5190         }
5191
5192         return 0;
5193 }
5194
5195 int evergreen_resume(struct radeon_device *rdev)
5196 {
5197         int r;
5198
5199         /* reset the asic, the gfx blocks are often in a bad state
5200          * after the driver is unloaded or after a resume
5201          */
5202         if (radeon_asic_reset(rdev))
5203                 dev_warn(rdev->dev, "GPU reset failed !\n");
5204         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5205          * posting will perform necessary task to bring back GPU into good
5206          * shape.
5207          */
5208         /* post card */
5209         atom_asic_init(rdev->mode_info.atom_context);
5210
5211         /* init golden registers */
5212         evergreen_init_golden_registers(rdev);
5213
5214         rdev->accel_working = true;
5215         r = evergreen_startup(rdev);
5216         if (r) {
5217                 DRM_ERROR("evergreen startup failed on resume\n");
5218                 rdev->accel_working = false;
5219                 return r;
5220         }
5221
5222         return r;
5223
5224 }
5225
5226 int evergreen_suspend(struct radeon_device *rdev)
5227 {
5228         r600_audio_fini(rdev);
5229         uvd_v1_0_fini(rdev);
5230         radeon_uvd_suspend(rdev);
5231         r700_cp_stop(rdev);
5232         r600_dma_stop(rdev);
5233         evergreen_irq_suspend(rdev);
5234         radeon_wb_disable(rdev);
5235         evergreen_pcie_gart_disable(rdev);
5236
5237         return 0;
5238 }
5239
5240 /* Plan is to move initialization in that function and use
5241  * helper function so that radeon_device_init pretty much
5242  * do nothing more than calling asic specific function. This
5243  * should also allow to remove a bunch of callback function
5244  * like vram_info.
5245  */
5246 int evergreen_init(struct radeon_device *rdev)
5247 {
5248         int r;
5249
5250         /* Read BIOS */
5251         if (!radeon_get_bios(rdev)) {
5252                 if (ASIC_IS_AVIVO(rdev))
5253                         return -EINVAL;
5254         }
5255         /* Must be an ATOMBIOS */
5256         if (!rdev->is_atom_bios) {
5257                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5258                 return -EINVAL;
5259         }
5260         r = radeon_atombios_init(rdev);
5261         if (r)
5262                 return r;
5263         /* reset the asic, the gfx blocks are often in a bad state
5264          * after the driver is unloaded or after a resume
5265          */
5266         if (radeon_asic_reset(rdev))
5267                 dev_warn(rdev->dev, "GPU reset failed !\n");
5268         /* Post card if necessary */
5269         if (!radeon_card_posted(rdev)) {
5270                 if (!rdev->bios) {
5271                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5272                         return -EINVAL;
5273                 }
5274                 DRM_INFO("GPU not posted. posting now...\n");
5275                 atom_asic_init(rdev->mode_info.atom_context);
5276         }
5277         /* init golden registers */
5278         evergreen_init_golden_registers(rdev);
5279         /* Initialize scratch registers */
5280         r600_scratch_init(rdev);
5281         /* Initialize surface registers */
5282         radeon_surface_init(rdev);
5283         /* Initialize clocks */
5284         radeon_get_clock_info(rdev->ddev);
5285         /* Fence driver */
5286         r = radeon_fence_driver_init(rdev);
5287         if (r)
5288                 return r;
5289         /* initialize AGP */
5290         if (rdev->flags & RADEON_IS_AGP) {
5291                 r = radeon_agp_init(rdev);
5292                 if (r)
5293                         radeon_agp_disable(rdev);
5294         }
5295         /* initialize memory controller */
5296         r = evergreen_mc_init(rdev);
5297         if (r)
5298                 return r;
5299         /* Memory manager */
5300         r = radeon_bo_init(rdev);
5301         if (r)
5302                 return r;
5303
5304         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5305         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5306
5307         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5308         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5309
5310         r = radeon_uvd_init(rdev);
5311         if (!r) {
5312                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5313                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5314                                4096);
5315         }
5316
5317         rdev->ih.ring_obj = NULL;
5318         r600_ih_ring_init(rdev, 64 * 1024);
5319
5320         r = r600_pcie_gart_init(rdev);
5321         if (r)
5322                 return r;
5323
5324         rdev->accel_working = true;
5325         r = evergreen_startup(rdev);
5326         if (r) {
5327                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5328                 r700_cp_fini(rdev);
5329                 r600_dma_fini(rdev);
5330                 r600_irq_fini(rdev);
5331                 if (rdev->flags & RADEON_IS_IGP)
5332                         sumo_rlc_fini(rdev);
5333                 radeon_wb_fini(rdev);
5334                 radeon_ib_pool_fini(rdev);
5335                 radeon_irq_kms_fini(rdev);
5336                 evergreen_pcie_gart_fini(rdev);
5337                 rdev->accel_working = false;
5338         }
5339
5340         /* Don't start up if the MC ucode is missing on BTC parts.
5341          * The default clocks and voltages before the MC ucode
5342          * is loaded are not suffient for advanced operations.
5343          */
5344         if (ASIC_IS_DCE5(rdev)) {
5345                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5346                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5347                         return -EINVAL;
5348                 }
5349         }
5350
5351         return 0;
5352 }
5353
5354 void evergreen_fini(struct radeon_device *rdev)
5355 {
5356         r600_audio_fini(rdev);
5357         r700_cp_fini(rdev);
5358         r600_dma_fini(rdev);
5359         r600_irq_fini(rdev);
5360         if (rdev->flags & RADEON_IS_IGP)
5361                 sumo_rlc_fini(rdev);
5362         radeon_wb_fini(rdev);
5363         radeon_ib_pool_fini(rdev);
5364         radeon_irq_kms_fini(rdev);
5365         evergreen_pcie_gart_fini(rdev);
5366         uvd_v1_0_fini(rdev);
5367         radeon_uvd_fini(rdev);
5368         r600_vram_scratch_fini(rdev);
5369         radeon_gem_fini(rdev);
5370         radeon_fence_driver_fini(rdev);
5371         radeon_agp_fini(rdev);
5372         radeon_bo_fini(rdev);
5373         radeon_atombios_fini(rdev);
5374         kfree(rdev->bios);
5375         rdev->bios = NULL;
5376 }
5377
5378 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5379 {
5380         u32 link_width_cntl, speed_cntl;
5381
5382         if (radeon_pcie_gen2 == 0)
5383                 return;
5384
5385         if (rdev->flags & RADEON_IS_IGP)
5386                 return;
5387
5388         if (!(rdev->flags & RADEON_IS_PCIE))
5389                 return;
5390
5391         /* x2 cards have a special sequence */
5392         if (ASIC_IS_X2(rdev))
5393                 return;
5394
5395         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5396                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5397                 return;
5398
5399         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5400         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5401                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5402                 return;
5403         }
5404
5405         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5406
5407         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5408             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5409
5410                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5411                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5412                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5413
5414                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5415                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5416                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5417
5418                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5419                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5420                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5421
5422                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5423                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5424                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5425
5426                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5427                 speed_cntl |= LC_GEN2_EN_STRAP;
5428                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5429
5430         } else {
5431                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5432                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5433                 if (1)
5434                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5435                 else
5436                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5437                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5438         }
5439 }
5440
5441 void evergreen_program_aspm(struct radeon_device *rdev)
5442 {
5443         u32 data, orig;
5444         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5445         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5446         /* fusion_platform = true
5447          * if the system is a fusion system
5448          * (APU or DGPU in a fusion system).
5449          * todo: check if the system is a fusion platform.
5450          */
5451         bool fusion_platform = false;
5452
5453         if (radeon_aspm == 0)
5454                 return;
5455
5456         if (!(rdev->flags & RADEON_IS_PCIE))
5457                 return;
5458
5459         switch (rdev->family) {
5460         case CHIP_CYPRESS:
5461         case CHIP_HEMLOCK:
5462         case CHIP_JUNIPER:
5463         case CHIP_REDWOOD:
5464         case CHIP_CEDAR:
5465         case CHIP_SUMO:
5466         case CHIP_SUMO2:
5467         case CHIP_PALM:
5468         case CHIP_ARUBA:
5469                 disable_l0s = true;
5470                 break;
5471         default:
5472                 disable_l0s = false;
5473                 break;
5474         }
5475
5476         if (rdev->flags & RADEON_IS_IGP)
5477                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5478
5479         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5480         if (fusion_platform)
5481                 data &= ~MULTI_PIF;
5482         else
5483                 data |= MULTI_PIF;
5484         if (data != orig)
5485                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5486
5487         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5488         if (fusion_platform)
5489                 data &= ~MULTI_PIF;
5490         else
5491                 data |= MULTI_PIF;
5492         if (data != orig)
5493                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5494
5495         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5496         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5497         if (!disable_l0s) {
5498                 if (rdev->family >= CHIP_BARTS)
5499                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5500                 else
5501                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5502         }
5503
5504         if (!disable_l1) {
5505                 if (rdev->family >= CHIP_BARTS)
5506                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5507                 else
5508                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5509
5510                 if (!disable_plloff_in_l1) {
5511                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5512                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5513                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5514                         if (data != orig)
5515                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5516
5517                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5518                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5519                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5520                         if (data != orig)
5521                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5522
5523                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5524                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5525                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5526                         if (data != orig)
5527                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5528
5529                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5530                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5531                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5532                         if (data != orig)
5533                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5534
5535                         if (rdev->family >= CHIP_BARTS) {
5536                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5537                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5538                                 data |= PLL_RAMP_UP_TIME_0(4);
5539                                 if (data != orig)
5540                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5541
5542                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5543                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5544                                 data |= PLL_RAMP_UP_TIME_1(4);
5545                                 if (data != orig)
5546                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5547
5548                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5549                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5550                                 data |= PLL_RAMP_UP_TIME_0(4);
5551                                 if (data != orig)
5552                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5553
5554                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5555                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5556                                 data |= PLL_RAMP_UP_TIME_1(4);
5557                                 if (data != orig)
5558                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5559                         }
5560
5561                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5562                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5563                         data |= LC_DYN_LANES_PWR_STATE(3);
5564                         if (data != orig)
5565                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5566
5567                         if (rdev->family >= CHIP_BARTS) {
5568                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5569                                 data &= ~LS2_EXIT_TIME_MASK;
5570                                 data |= LS2_EXIT_TIME(1);
5571                                 if (data != orig)
5572                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5573
5574                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5575                                 data &= ~LS2_EXIT_TIME_MASK;
5576                                 data |= LS2_EXIT_TIME(1);
5577                                 if (data != orig)
5578                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5579                         }
5580                 }
5581         }
5582
5583         /* evergreen parts only */
5584         if (rdev->family < CHIP_BARTS)
5585                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5586
5587         if (pcie_lc_cntl != pcie_lc_cntl_old)
5588                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5589 }