drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include "i915_drv.h"
  30 #include "intel_drv.h"
  31 #include "../../../platform/x86/intel_ips.h"
  32 #include <linux/module.h>
  33
  34 /**
  35  * RC6 is a special power stage which allows the GPU to enter an very
  36  * low-voltage mode when idle, using down to 0V while at this stage.  This
  37  * stage is entered automatically when the GPU is idle when RC6 support is
  38  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  39  *
  40  * There are different RC6 modes available in Intel GPU, which differentiate
  41  * among each other with the latency required to enter and leave RC6 and
  42  * voltage consumed by the GPU in different states.
  43  *
  44  * The combination of the following flags define which states GPU is allowed
  45  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  46  * RC6pp is deepest RC6. Their support by hardware varies according to the
  47  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  48  * which brings the most power savings; deeper states save more power, but
  49  * require higher latency to switch to and wake up.
  50  */
  51 #define INTEL_RC6_ENABLE                        (1<<0)
  52 #define INTEL_RC6p_ENABLE                       (1<<1)
  53 #define INTEL_RC6pp_ENABLE                      (1<<2)
  54
  55 static void skl_init_clock_gating(struct drm_device *dev)
  56 {
  57         struct drm_i915_private *dev_priv = dev->dev_private;
  58
  59         if (INTEL_REVID(dev) == SKL_REVID_A0) {
  60                 /*
  61                  * WaDisableSDEUnitClockGating:skl
  62                  * WaSetGAPSunitClckGateDisable:skl
  63                  */
  64                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
  65                            GEN8_GAPSUNIT_CLOCK_GATE_DISABLE |
  66                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  67         }
  68
  69         if (INTEL_REVID(dev) <= SKL_REVID_E0)
  70                 /* WaDisableLSQCROPERFforOCL:skl */
  71                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
  72                            GEN8_LQSC_RO_PERF_DIS);
  73 }
  74
  75 static void i915_pineview_get_mem_freq(struct drm_device *dev)
  76 {
  77         struct drm_i915_private *dev_priv = dev->dev_private;
  78         u32 tmp;
  79
  80         tmp = I915_READ(CLKCFG);
  81
  82         switch (tmp & CLKCFG_FSB_MASK) {
  83         case CLKCFG_FSB_533:
  84                 dev_priv->fsb_freq = 533; /* 133*4 */
  85                 break;
  86         case CLKCFG_FSB_800:
  87                 dev_priv->fsb_freq = 800; /* 200*4 */
  88                 break;
  89         case CLKCFG_FSB_667:
  90                 dev_priv->fsb_freq =  667; /* 167*4 */
  91                 break;
  92         case CLKCFG_FSB_400:
  93                 dev_priv->fsb_freq = 400; /* 100*4 */
  94                 break;
  95         }
  96
  97         switch (tmp & CLKCFG_MEM_MASK) {
  98         case CLKCFG_MEM_533:
  99                 dev_priv->mem_freq = 533;
 100                 break;
 101         case CLKCFG_MEM_667:
 102                 dev_priv->mem_freq = 667;
 103                 break;
 104         case CLKCFG_MEM_800:
 105                 dev_priv->mem_freq = 800;
 106                 break;
 107         }
 108
 109         /* detect pineview DDR3 setting */
 110         tmp = I915_READ(CSHRDDR3CTL);
 111         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 112 }
 113
 114 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 115 {
 116         struct drm_i915_private *dev_priv = dev->dev_private;
 117         u16 ddrpll, csipll;
 118
 119         ddrpll = I915_READ16(DDRMPLL1);
 120         csipll = I915_READ16(CSIPLL0);
 121
 122         switch (ddrpll & 0xff) {
 123         case 0xc:
 124                 dev_priv->mem_freq = 800;
 125                 break;
 126         case 0x10:
 127                 dev_priv->mem_freq = 1066;
 128                 break;
 129         case 0x14:
 130                 dev_priv->mem_freq = 1333;
 131                 break;
 132         case 0x18:
 133                 dev_priv->mem_freq = 1600;
 134                 break;
 135         default:
 136                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 137                                  ddrpll & 0xff);
 138                 dev_priv->mem_freq = 0;
 139                 break;
 140         }
 141
 142         dev_priv->ips.r_t = dev_priv->mem_freq;
 143
 144         switch (csipll & 0x3ff) {
 145         case 0x00c:
 146                 dev_priv->fsb_freq = 3200;
 147                 break;
 148         case 0x00e:
 149                 dev_priv->fsb_freq = 3733;
 150                 break;
 151         case 0x010:
 152                 dev_priv->fsb_freq = 4266;
 153                 break;
 154         case 0x012:
 155                 dev_priv->fsb_freq = 4800;
 156                 break;
 157         case 0x014:
 158                 dev_priv->fsb_freq = 5333;
 159                 break;
 160         case 0x016:
 161                 dev_priv->fsb_freq = 5866;
 162                 break;
 163         case 0x018:
 164                 dev_priv->fsb_freq = 6400;
 165                 break;
 166         default:
 167                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 168                                  csipll & 0x3ff);
 169                 dev_priv->fsb_freq = 0;
 170                 break;
 171         }
 172
 173         if (dev_priv->fsb_freq == 3200) {
 174                 dev_priv->ips.c_m = 0;
 175         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 176                 dev_priv->ips.c_m = 1;
 177         } else {
 178                 dev_priv->ips.c_m = 2;
 179         }
 180 }
 181
 182 static const struct cxsr_latency cxsr_latency_table[] = {
 183         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 184         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 185         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 186         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 187         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 188
 189         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 190         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 191         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 192         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 193         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 194
 195         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 196         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 197         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 198         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 199         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 200
 201         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 202         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 203         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 204         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 205         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 206
 207         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 208         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 209         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 210         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 211         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 212
 213         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 214         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 215         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 216         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 217         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 218 };
 219
 220 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
 221                                                          int is_ddr3,
 222                                                          int fsb,
 223                                                          int mem)
 224 {
 225         const struct cxsr_latency *latency;
 226         int i;
 227
 228         if (fsb == 0 || mem == 0)
 229                 return NULL;
 230
 231         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 232                 latency = &cxsr_latency_table[i];
 233                 if (is_desktop == latency->is_desktop &&
 234                     is_ddr3 == latency->is_ddr3 &&
 235                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 236                         return latency;
 237         }
 238
 239         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 240
 241         return NULL;
 242 }
 243
 244 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 245 {
 246         struct drm_device *dev = dev_priv->dev;
 247         u32 val;
 248
 249         if (IS_VALLEYVIEW(dev)) {
 250                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 251         } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
 252                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 253         } else if (IS_PINEVIEW(dev)) {
 254                 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
 255                 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
 256                 I915_WRITE(DSPFW3, val);
 257         } else if (IS_I945G(dev) || IS_I945GM(dev)) {
 258                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 259                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 260                 I915_WRITE(FW_BLC_SELF, val);
 261         } else if (IS_I915GM(dev)) {
 262                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 263                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 264                 I915_WRITE(INSTPM, val);
 265         } else {
 266                 return;
 267         }
 268
 269         DRM_DEBUG_KMS("memory self-refresh is %s\n",
 270                       enable ? "enabled" : "disabled");
 271 }
 272
 273 /*
 274  * Latency for FIFO fetches is dependent on several factors:
 275  *   - memory configuration (speed, channels)
 276  *   - chipset
 277  *   - current MCH state
 278  * It can be fairly high in some situations, so here we assume a fairly
 279  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 280  * set this value too high, the FIFO will fetch frequently to stay full)
 281  * and power consumption (set it too low to save power and we might see
 282  * FIFO underruns and display "flicker").
 283  *
 284  * A value of 5us seems to be a good balance; safe for very low end
 285  * platforms but not overly aggressive on lower latency configs.
 286  */
 287 static const int pessimal_latency_ns = 5000;
 288
 289 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 290 {
 291         struct drm_i915_private *dev_priv = dev->dev_private;
 292         uint32_t dsparb = I915_READ(DSPARB);
 293         int size;
 294
 295         size = dsparb & 0x7f;
 296         if (plane)
 297                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 298
 299         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 300                       plane ? "B" : "A", size);
 301
 302         return size;
 303 }
 304
 305 static int i830_get_fifo_size(struct drm_device *dev, int plane)
 306 {
 307         struct drm_i915_private *dev_priv = dev->dev_private;
 308         uint32_t dsparb = I915_READ(DSPARB);
 309         int size;
 310
 311         size = dsparb & 0x1ff;
 312         if (plane)
 313                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 314         size >>= 1; /* Convert to cachelines */
 315
 316         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 317                       plane ? "B" : "A", size);
 318
 319         return size;
 320 }
 321
 322 static int i845_get_fifo_size(struct drm_device *dev, int plane)
 323 {
 324         struct drm_i915_private *dev_priv = dev->dev_private;
 325         uint32_t dsparb = I915_READ(DSPARB);
 326         int size;
 327
 328         size = dsparb & 0x7f;
 329         size >>= 2; /* Convert to cachelines */
 330
 331         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 332                       plane ? "B" : "A",
 333                       size);
 334
 335         return size;
 336 }
 337
 338 /* Pineview has different values for various configs */
 339 static const struct intel_watermark_params pineview_display_wm = {
 340         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 341         .max_wm = PINEVIEW_MAX_WM,
 342         .default_wm = PINEVIEW_DFT_WM,
 343         .guard_size = PINEVIEW_GUARD_WM,
 344         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 345 };
 346 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 347         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 348         .max_wm = PINEVIEW_MAX_WM,
 349         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 350         .guard_size = PINEVIEW_GUARD_WM,
 351         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 352 };
 353 static const struct intel_watermark_params pineview_cursor_wm = {
 354         .fifo_size = PINEVIEW_CURSOR_FIFO,
 355         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 356         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 357         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 358         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 359 };
 360 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 361         .fifo_size = PINEVIEW_CURSOR_FIFO,
 362         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 363         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 364         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 365         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 366 };
 367 static const struct intel_watermark_params g4x_wm_info = {
 368         .fifo_size = G4X_FIFO_SIZE,
 369         .max_wm = G4X_MAX_WM,
 370         .default_wm = G4X_MAX_WM,
 371         .guard_size = 2,
 372         .cacheline_size = G4X_FIFO_LINE_SIZE,
 373 };
 374 static const struct intel_watermark_params g4x_cursor_wm_info = {
 375         .fifo_size = I965_CURSOR_FIFO,
 376         .max_wm = I965_CURSOR_MAX_WM,
 377         .default_wm = I965_CURSOR_DFT_WM,
 378         .guard_size = 2,
 379         .cacheline_size = G4X_FIFO_LINE_SIZE,
 380 };
 381 static const struct intel_watermark_params valleyview_wm_info = {
 382         .fifo_size = VALLEYVIEW_FIFO_SIZE,
 383         .max_wm = VALLEYVIEW_MAX_WM,
 384         .default_wm = VALLEYVIEW_MAX_WM,
 385         .guard_size = 2,
 386         .cacheline_size = G4X_FIFO_LINE_SIZE,
 387 };
 388 static const struct intel_watermark_params valleyview_cursor_wm_info = {
 389         .fifo_size = I965_CURSOR_FIFO,
 390         .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
 391         .default_wm = I965_CURSOR_DFT_WM,
 392         .guard_size = 2,
 393         .cacheline_size = G4X_FIFO_LINE_SIZE,
 394 };
 395 static const struct intel_watermark_params i965_cursor_wm_info = {
 396         .fifo_size = I965_CURSOR_FIFO,
 397         .max_wm = I965_CURSOR_MAX_WM,
 398         .default_wm = I965_CURSOR_DFT_WM,
 399         .guard_size = 2,
 400         .cacheline_size = I915_FIFO_LINE_SIZE,
 401 };
 402 static const struct intel_watermark_params i945_wm_info = {
 403         .fifo_size = I945_FIFO_SIZE,
 404         .max_wm = I915_MAX_WM,
 405         .default_wm = 1,
 406         .guard_size = 2,
 407         .cacheline_size = I915_FIFO_LINE_SIZE,
 408 };
 409 static const struct intel_watermark_params i915_wm_info = {
 410         .fifo_size = I915_FIFO_SIZE,
 411         .max_wm = I915_MAX_WM,
 412         .default_wm = 1,
 413         .guard_size = 2,
 414         .cacheline_size = I915_FIFO_LINE_SIZE,
 415 };
 416 static const struct intel_watermark_params i830_a_wm_info = {
 417         .fifo_size = I855GM_FIFO_SIZE,
 418         .max_wm = I915_MAX_WM,
 419         .default_wm = 1,
 420         .guard_size = 2,
 421         .cacheline_size = I830_FIFO_LINE_SIZE,
 422 };
 423 static const struct intel_watermark_params i830_bc_wm_info = {
 424         .fifo_size = I855GM_FIFO_SIZE,
 425         .max_wm = I915_MAX_WM/2,
 426         .default_wm = 1,
 427         .guard_size = 2,
 428         .cacheline_size = I830_FIFO_LINE_SIZE,
 429 };
 430 static const struct intel_watermark_params i845_wm_info = {
 431         .fifo_size = I830_FIFO_SIZE,
 432         .max_wm = I915_MAX_WM,
 433         .default_wm = 1,
 434         .guard_size = 2,
 435         .cacheline_size = I830_FIFO_LINE_SIZE,
 436 };
 437
 438 /**
 439  * intel_calculate_wm - calculate watermark level
 440  * @clock_in_khz: pixel clock
 441  * @wm: chip FIFO params
 442  * @pixel_size: display pixel size
 443  * @latency_ns: memory latency for the platform
 444  *
 445  * Calculate the watermark level (the level at which the display plane will
 446  * start fetching from memory again).  Each chip has a different display
 447  * FIFO size and allocation, so the caller needs to figure that out and pass
 448  * in the correct intel_watermark_params structure.
 449  *
 450  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 451  * on the pixel size.  When it reaches the watermark level, it'll start
 452  * fetching FIFO line sized based chunks from memory until the FIFO fills
 453  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 454  * will occur, and a display engine hang could result.
 455  */
 456 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 457                                         const struct intel_watermark_params *wm,
 458                                         int fifo_size,
 459                                         int pixel_size,
 460                                         unsigned long latency_ns)
 461 {
 462         long entries_required, wm_size;
 463
 464         /*
 465          * Note: we need to make sure we don't overflow for various clock &
 466          * latency values.
 467          * clocks go from a few thousand to several hundred thousand.
 468          * latency is usually a few thousand
 469          */
 470         entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
 471                 1000;
 472         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
 473
 474         DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
 475
 476         wm_size = fifo_size - (entries_required + wm->guard_size);
 477
 478         DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
 479
 480         /* Don't promote wm_size to unsigned... */
 481         if (wm_size > (long)wm->max_wm)
 482                 wm_size = wm->max_wm;
 483         if (wm_size <= 0)
 484                 wm_size = wm->default_wm;
 485
 486         /*
 487          * Bspec seems to indicate that the value shouldn't be lower than
 488          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 489          * Lets go for 8 which is the burst size since certain platforms
 490          * already use a hardcoded 8 (which is what the spec says should be
 491          * done).
 492          */
 493         if (wm_size <= 8)
 494                 wm_size = 8;
 495
 496         return wm_size;
 497 }
 498
 499 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
 500 {
 501         struct drm_crtc *crtc, *enabled = NULL;
 502
 503         for_each_crtc(dev, crtc) {
 504                 if (intel_crtc_active(crtc)) {
 505                         if (enabled)
 506                                 return NULL;
 507                         enabled = crtc;
 508                 }
 509         }
 510
 511         return enabled;
 512 }
 513
 514 static void pineview_update_wm(struct drm_crtc *unused_crtc)
 515 {
 516         struct drm_device *dev = unused_crtc->dev;
 517         struct drm_i915_private *dev_priv = dev->dev_private;
 518         struct drm_crtc *crtc;
 519         const struct cxsr_latency *latency;
 520         u32 reg;
 521         unsigned long wm;
 522
 523         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
 524                                          dev_priv->fsb_freq, dev_priv->mem_freq);
 525         if (!latency) {
 526                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 527                 intel_set_memory_cxsr(dev_priv, false);
 528                 return;
 529         }
 530
 531         crtc = single_enabled_crtc(dev);
 532         if (crtc) {
 533                 const struct drm_display_mode *adjusted_mode;
 534                 int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
 535                 int clock;
 536
 537                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 538                 clock = adjusted_mode->crtc_clock;
 539
 540                 /* Display SR */
 541                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 542                                         pineview_display_wm.fifo_size,
 543                                         pixel_size, latency->display_sr);
 544                 reg = I915_READ(DSPFW1);
 545                 reg &= ~DSPFW_SR_MASK;
 546                 reg |= wm << DSPFW_SR_SHIFT;
 547                 I915_WRITE(DSPFW1, reg);
 548                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 549
 550                 /* cursor SR */
 551                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 552                                         pineview_display_wm.fifo_size,
 553                                         pixel_size, latency->cursor_sr);
 554                 reg = I915_READ(DSPFW3);
 555                 reg &= ~DSPFW_CURSOR_SR_MASK;
 556                 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
 557                 I915_WRITE(DSPFW3, reg);
 558
 559                 /* Display HPLL off SR */
 560                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 561                                         pineview_display_hplloff_wm.fifo_size,
 562                                         pixel_size, latency->display_hpll_disable);
 563                 reg = I915_READ(DSPFW3);
 564                 reg &= ~DSPFW_HPLL_SR_MASK;
 565                 reg |= wm & DSPFW_HPLL_SR_MASK;
 566                 I915_WRITE(DSPFW3, reg);
 567
 568                 /* cursor HPLL off SR */
 569                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 570                                         pineview_display_hplloff_wm.fifo_size,
 571                                         pixel_size, latency->cursor_hpll_disable);
 572                 reg = I915_READ(DSPFW3);
 573                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 574                 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
 575                 I915_WRITE(DSPFW3, reg);
 576                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 577
 578                 intel_set_memory_cxsr(dev_priv, true);
 579         } else {
 580                 intel_set_memory_cxsr(dev_priv, false);
 581         }
 582 }
 583
 584 static bool g4x_compute_wm0(struct drm_device *dev,
 585                             int plane,
 586                             const struct intel_watermark_params *display,
 587                             int display_latency_ns,
 588                             const struct intel_watermark_params *cursor,
 589                             int cursor_latency_ns,
 590                             int *plane_wm,
 591                             int *cursor_wm)
 592 {
 593         struct drm_crtc *crtc;
 594         const struct drm_display_mode *adjusted_mode;
 595         int htotal, hdisplay, clock, pixel_size;
 596         int line_time_us, line_count;
 597         int entries, tlb_miss;
 598
 599         crtc = intel_get_crtc_for_plane(dev, plane);
 600         if (!intel_crtc_active(crtc)) {
 601                 *cursor_wm = cursor->guard_size;
 602                 *plane_wm = display->guard_size;
 603                 return false;
 604         }
 605
 606         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 607         clock = adjusted_mode->crtc_clock;
 608         htotal = adjusted_mode->crtc_htotal;
 609         hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
 610         pixel_size = crtc->primary->fb->bits_per_pixel / 8;
 611
 612         /* Use the small buffer method to calculate plane watermark */
 613         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
 614         tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
 615         if (tlb_miss > 0)
 616                 entries += tlb_miss;
 617         entries = DIV_ROUND_UP(entries, display->cacheline_size);
 618         *plane_wm = entries + display->guard_size;
 619         if (*plane_wm > (int)display->max_wm)
 620                 *plane_wm = display->max_wm;
 621
 622         /* Use the large buffer method to calculate cursor watermark */
 623         line_time_us = max(htotal * 1000 / clock, 1);
 624         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
 625         entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
 626         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
 627         if (tlb_miss > 0)
 628                 entries += tlb_miss;
 629         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 630         *cursor_wm = entries + cursor->guard_size;
 631         if (*cursor_wm > (int)cursor->max_wm)
 632                 *cursor_wm = (int)cursor->max_wm;
 633
 634         return true;
 635 }
 636
 637 /*
 638  * Check the wm result.
 639  *
 640  * If any calculated watermark values is larger than the maximum value that
 641  * can be programmed into the associated watermark register, that watermark
 642  * must be disabled.
 643  */
 644 static bool g4x_check_srwm(struct drm_device *dev,
 645                            int display_wm, int cursor_wm,
 646                            const struct intel_watermark_params *display,
 647                            const struct intel_watermark_params *cursor)
 648 {
 649         DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
 650                       display_wm, cursor_wm);
 651
 652         if (display_wm > display->max_wm) {
 653                 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
 654                               display_wm, display->max_wm);
 655                 return false;
 656         }
 657
 658         if (cursor_wm > cursor->max_wm) {
 659                 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
 660                               cursor_wm, cursor->max_wm);
 661                 return false;
 662         }
 663
 664         if (!(display_wm || cursor_wm)) {
 665                 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
 666                 return false;
 667         }
 668
 669         return true;
 670 }
 671
 672 static bool g4x_compute_srwm(struct drm_device *dev,
 673                              int plane,
 674                              int latency_ns,
 675                              const struct intel_watermark_params *display,
 676                              const struct intel_watermark_params *cursor,
 677                              int *display_wm, int *cursor_wm)
 678 {
 679         struct drm_crtc *crtc;
 680         const struct drm_display_mode *adjusted_mode;
 681         int hdisplay, htotal, pixel_size, clock;
 682         unsigned long line_time_us;
 683         int line_count, line_size;
 684         int small, large;
 685         int entries;
 686
 687         if (!latency_ns) {
 688                 *display_wm = *cursor_wm = 0;
 689                 return false;
 690         }
 691
 692         crtc = intel_get_crtc_for_plane(dev, plane);
 693         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 694         clock = adjusted_mode->crtc_clock;
 695         htotal = adjusted_mode->crtc_htotal;
 696         hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
 697         pixel_size = crtc->primary->fb->bits_per_pixel / 8;
 698
 699         line_time_us = max(htotal * 1000 / clock, 1);
 700         line_count = (latency_ns / line_time_us + 1000) / 1000;
 701         line_size = hdisplay * pixel_size;
 702
 703         /* Use the minimum of the small and large buffer method for primary */
 704         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
 705         large = line_count * line_size;
 706
 707         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
 708         *display_wm = entries + display->guard_size;
 709
 710         /* calculate the self-refresh watermark for display cursor */
 711         entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
 712         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 713         *cursor_wm = entries + cursor->guard_size;
 714
 715         return g4x_check_srwm(dev,
 716                               *display_wm, *cursor_wm,
 717                               display, cursor);
 718 }
 719
 720 static bool vlv_compute_drain_latency(struct drm_crtc *crtc,
 721                                       int pixel_size,
 722                                       int *prec_mult,
 723                                       int *drain_latency)
 724 {
 725         struct drm_device *dev = crtc->dev;
 726         int entries;
 727         int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
 728
 729         if (WARN(clock == 0, "Pixel clock is zero!\n"))
 730                 return false;
 731
 732         if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
 733                 return false;
 734
 735         entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
 736         if (IS_CHERRYVIEW(dev))
 737                 *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_32 :
 738                                                DRAIN_LATENCY_PRECISION_16;
 739         else
 740                 *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 :
 741                                                DRAIN_LATENCY_PRECISION_32;
 742         *drain_latency = (64 * (*prec_mult) * 4) / entries;
 743
 744         if (*drain_latency > DRAIN_LATENCY_MASK)
 745                 *drain_latency = DRAIN_LATENCY_MASK;
 746
 747         return true;
 748 }
 749
 750 /*
 751  * Update drain latency registers of memory arbiter
 752  *
 753  * Valleyview SoC has a new memory arbiter and needs drain latency registers
 754  * to be programmed. Each plane has a drain latency multiplier and a drain
 755  * latency value.
 756  */
 757
 758 static void vlv_update_drain_latency(struct drm_crtc *crtc)
 759 {
 760         struct drm_device *dev = crtc->dev;
 761         struct drm_i915_private *dev_priv = dev->dev_private;
 762         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 763         int pixel_size;
 764         int drain_latency;
 765         enum pipe pipe = intel_crtc->pipe;
 766         int plane_prec, prec_mult, plane_dl;
 767         const int high_precision = IS_CHERRYVIEW(dev) ?
 768                 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
 769
 770         plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_HIGH |
 771                    DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_HIGH |
 772                    (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT));
 773
 774         if (!intel_crtc_active(crtc)) {
 775                 I915_WRITE(VLV_DDL(pipe), plane_dl);
 776                 return;
 777         }
 778
 779         /* Primary plane Drain Latency */
 780         pixel_size = crtc->primary->fb->bits_per_pixel / 8;     /* BPP */
 781         if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
 782                 plane_prec = (prec_mult == high_precision) ?
 783                                            DDL_PLANE_PRECISION_HIGH :
 784                                            DDL_PLANE_PRECISION_LOW;
 785                 plane_dl |= plane_prec | drain_latency;
 786         }
 787
 788         /* Cursor Drain Latency
 789          * BPP is always 4 for cursor
 790          */
 791         pixel_size = 4;
 792
 793         /* Program cursor DL only if it is enabled */
 794         if (intel_crtc->cursor_base &&
 795             vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
 796                 plane_prec = (prec_mult == high_precision) ?
 797                                            DDL_CURSOR_PRECISION_HIGH :
 798                                            DDL_CURSOR_PRECISION_LOW;
 799                 plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT);
 800         }
 801
 802         I915_WRITE(VLV_DDL(pipe), plane_dl);
 803 }
 804
 805 #define single_plane_enabled(mask) is_power_of_2(mask)
 806
 807 static void valleyview_update_wm(struct drm_crtc *crtc)
 808 {
 809         struct drm_device *dev = crtc->dev;
 810         static const int sr_latency_ns = 12000;
 811         struct drm_i915_private *dev_priv = dev->dev_private;
 812         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
 813         int plane_sr, cursor_sr;
 814         int ignore_plane_sr, ignore_cursor_sr;
 815         unsigned int enabled = 0;
 816         bool cxsr_enabled;
 817
 818         vlv_update_drain_latency(crtc);
 819
 820         if (g4x_compute_wm0(dev, PIPE_A,
 821                             &valleyview_wm_info, pessimal_latency_ns,
 822                             &valleyview_cursor_wm_info, pessimal_latency_ns,
 823                             &planea_wm, &cursora_wm))
 824                 enabled |= 1 << PIPE_A;
 825
 826         if (g4x_compute_wm0(dev, PIPE_B,
 827                             &valleyview_wm_info, pessimal_latency_ns,
 828                             &valleyview_cursor_wm_info, pessimal_latency_ns,
 829                             &planeb_wm, &cursorb_wm))
 830                 enabled |= 1 << PIPE_B;
 831
 832         if (single_plane_enabled(enabled) &&
 833             g4x_compute_srwm(dev, ffs(enabled) - 1,
 834                              sr_latency_ns,
 835                              &valleyview_wm_info,
 836                              &valleyview_cursor_wm_info,
 837                              &plane_sr, &ignore_cursor_sr) &&
 838             g4x_compute_srwm(dev, ffs(enabled) - 1,
 839                              2*sr_latency_ns,
 840                              &valleyview_wm_info,
 841                              &valleyview_cursor_wm_info,
 842                              &ignore_plane_sr, &cursor_sr)) {
 843                 cxsr_enabled = true;
 844         } else {
 845                 cxsr_enabled = false;
 846                 intel_set_memory_cxsr(dev_priv, false);
 847                 plane_sr = cursor_sr = 0;
 848         }
 849
 850         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
 851                       "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
 852                       planea_wm, cursora_wm,
 853                       planeb_wm, cursorb_wm,
 854                       plane_sr, cursor_sr);
 855
 856         I915_WRITE(DSPFW1,
 857                    (plane_sr << DSPFW_SR_SHIFT) |
 858                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
 859                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
 860                    (planea_wm << DSPFW_PLANEA_SHIFT));
 861         I915_WRITE(DSPFW2,
 862                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
 863                    (cursora_wm << DSPFW_CURSORA_SHIFT));
 864         I915_WRITE(DSPFW3,
 865                    (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
 866                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
 867
 868         if (cxsr_enabled)
 869                 intel_set_memory_cxsr(dev_priv, true);
 870 }
 871
 872 static void cherryview_update_wm(struct drm_crtc *crtc)
 873 {
 874         struct drm_device *dev = crtc->dev;
 875         static const int sr_latency_ns = 12000;
 876         struct drm_i915_private *dev_priv = dev->dev_private;
 877         int planea_wm, planeb_wm, planec_wm;
 878         int cursora_wm, cursorb_wm, cursorc_wm;
 879         int plane_sr, cursor_sr;
 880         int ignore_plane_sr, ignore_cursor_sr;
 881         unsigned int enabled = 0;
 882         bool cxsr_enabled;
 883
 884         vlv_update_drain_latency(crtc);
 885
 886         if (g4x_compute_wm0(dev, PIPE_A,
 887                             &valleyview_wm_info, pessimal_latency_ns,
 888                             &valleyview_cursor_wm_info, pessimal_latency_ns,
 889                             &planea_wm, &cursora_wm))
 890                 enabled |= 1 << PIPE_A;
 891
 892         if (g4x_compute_wm0(dev, PIPE_B,
 893                             &valleyview_wm_info, pessimal_latency_ns,
 894                             &valleyview_cursor_wm_info, pessimal_latency_ns,
 895                             &planeb_wm, &cursorb_wm))
 896                 enabled |= 1 << PIPE_B;
 897
 898         if (g4x_compute_wm0(dev, PIPE_C,
 899                             &valleyview_wm_info, pessimal_latency_ns,
 900                             &valleyview_cursor_wm_info, pessimal_latency_ns,
 901                             &planec_wm, &cursorc_wm))
 902                 enabled |= 1 << PIPE_C;
 903
 904         if (single_plane_enabled(enabled) &&
 905             g4x_compute_srwm(dev, ffs(enabled) - 1,
 906                              sr_latency_ns,
 907                              &valleyview_wm_info,
 908                              &valleyview_cursor_wm_info,
 909                              &plane_sr, &ignore_cursor_sr) &&
 910             g4x_compute_srwm(dev, ffs(enabled) - 1,
 911                              2*sr_latency_ns,
 912                              &valleyview_wm_info,
 913                              &valleyview_cursor_wm_info,
 914                              &ignore_plane_sr, &cursor_sr)) {
 915                 cxsr_enabled = true;
 916         } else {
 917                 cxsr_enabled = false;
 918                 intel_set_memory_cxsr(dev_priv, false);
 919                 plane_sr = cursor_sr = 0;
 920         }
 921
 922         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
 923                       "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, "
 924                       "SR: plane=%d, cursor=%d\n",
 925                       planea_wm, cursora_wm,
 926                       planeb_wm, cursorb_wm,
 927                       planec_wm, cursorc_wm,
 928                       plane_sr, cursor_sr);
 929
 930         I915_WRITE(DSPFW1,
 931                    (plane_sr << DSPFW_SR_SHIFT) |
 932                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
 933                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
 934                    (planea_wm << DSPFW_PLANEA_SHIFT));
 935         I915_WRITE(DSPFW2,
 936                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
 937                    (cursora_wm << DSPFW_CURSORA_SHIFT));
 938         I915_WRITE(DSPFW3,
 939                    (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
 940                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
 941         I915_WRITE(DSPFW9_CHV,
 942                    (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK |
 943                                               DSPFW_CURSORC_MASK)) |
 944                    (planec_wm << DSPFW_PLANEC_SHIFT) |
 945                    (cursorc_wm << DSPFW_CURSORC_SHIFT));
 946
 947         if (cxsr_enabled)
 948                 intel_set_memory_cxsr(dev_priv, true);
 949 }
 950
 951 static void valleyview_update_sprite_wm(struct drm_plane *plane,
 952                                         struct drm_crtc *crtc,
 953                                         uint32_t sprite_width,
 954                                         uint32_t sprite_height,
 955                                         int pixel_size,
 956                                         bool enabled, bool scaled)
 957 {
 958         struct drm_device *dev = crtc->dev;
 959         struct drm_i915_private *dev_priv = dev->dev_private;
 960         int pipe = to_intel_plane(plane)->pipe;
 961         int sprite = to_intel_plane(plane)->plane;
 962         int drain_latency;
 963         int plane_prec;
 964         int sprite_dl;
 965         int prec_mult;
 966         const int high_precision = IS_CHERRYVIEW(dev) ?
 967                 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
 968
 969         sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_HIGH(sprite) |
 970                     (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite)));
 971
 972         if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult,
 973                                                  &drain_latency)) {
 974                 plane_prec = (prec_mult == high_precision) ?
 975                                            DDL_SPRITE_PRECISION_HIGH(sprite) :
 976                                            DDL_SPRITE_PRECISION_LOW(sprite);
 977                 sprite_dl |= plane_prec |
 978                              (drain_latency << DDL_SPRITE_SHIFT(sprite));
 979         }
 980
 981         I915_WRITE(VLV_DDL(pipe), sprite_dl);
 982 }
 983
 984 static void g4x_update_wm(struct drm_crtc *crtc)
 985 {
 986         struct drm_device *dev = crtc->dev;
 987         static const int sr_latency_ns = 12000;
 988         struct drm_i915_private *dev_priv = dev->dev_private;
 989         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
 990         int plane_sr, cursor_sr;
 991         unsigned int enabled = 0;
 992         bool cxsr_enabled;
 993
 994         if (g4x_compute_wm0(dev, PIPE_A,
 995                             &g4x_wm_info, pessimal_latency_ns,
 996                             &g4x_cursor_wm_info, pessimal_latency_ns,
 997                             &planea_wm, &cursora_wm))
 998                 enabled |= 1 << PIPE_A;
 999
1000         if (g4x_compute_wm0(dev, PIPE_B,
1001                             &g4x_wm_info, pessimal_latency_ns,
1002                             &g4x_cursor_wm_info, pessimal_latency_ns,
1003                             &planeb_wm, &cursorb_wm))
1004                 enabled |= 1 << PIPE_B;
1005
1006         if (single_plane_enabled(enabled) &&
1007             g4x_compute_srwm(dev, ffs(enabled) - 1,
1008                              sr_latency_ns,
1009                              &g4x_wm_info,
1010                              &g4x_cursor_wm_info,
1011                              &plane_sr, &cursor_sr)) {
1012                 cxsr_enabled = true;
1013         } else {
1014                 cxsr_enabled = false;
1015                 intel_set_memory_cxsr(dev_priv, false);
1016                 plane_sr = cursor_sr = 0;
1017         }
1018
1019         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
1020                       "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1021                       planea_wm, cursora_wm,
1022                       planeb_wm, cursorb_wm,
1023                       plane_sr, cursor_sr);
1024
1025         I915_WRITE(DSPFW1,
1026                    (plane_sr << DSPFW_SR_SHIFT) |
1027                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1028                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
1029                    (planea_wm << DSPFW_PLANEA_SHIFT));
1030         I915_WRITE(DSPFW2,
1031                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1032                    (cursora_wm << DSPFW_CURSORA_SHIFT));
1033         /* HPLL off in SR has some issues on G4x... disable it */
1034         I915_WRITE(DSPFW3,
1035                    (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1036                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1037
1038         if (cxsr_enabled)
1039                 intel_set_memory_cxsr(dev_priv, true);
1040 }
1041
1042 static void i965_update_wm(struct drm_crtc *unused_crtc)
1043 {
1044         struct drm_device *dev = unused_crtc->dev;
1045         struct drm_i915_private *dev_priv = dev->dev_private;
1046         struct drm_crtc *crtc;
1047         int srwm = 1;
1048         int cursor_sr = 16;
1049         bool cxsr_enabled;
1050
1051         /* Calc sr entries for one plane configs */
1052         crtc = single_enabled_crtc(dev);
1053         if (crtc) {
1054                 /* self-refresh has much higher latency */
1055                 static const int sr_latency_ns = 12000;
1056                 const struct drm_display_mode *adjusted_mode =
1057                         &to_intel_crtc(crtc)->config->base.adjusted_mode;
1058                 int clock = adjusted_mode->crtc_clock;
1059                 int htotal = adjusted_mode->crtc_htotal;
1060                 int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
1061                 int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
1062                 unsigned long line_time_us;
1063                 int entries;
1064
1065                 line_time_us = max(htotal * 1000 / clock, 1);
1066
1067                 /* Use ns/us then divide to preserve precision */
1068                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1069                         pixel_size * hdisplay;
1070                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1071                 srwm = I965_FIFO_SIZE - entries;
1072                 if (srwm < 0)
1073                         srwm = 1;
1074                 srwm &= 0x1ff;
1075                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1076                               entries, srwm);
1077
1078                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1079                         pixel_size * to_intel_crtc(crtc)->cursor_width;
1080                 entries = DIV_ROUND_UP(entries,
1081                                           i965_cursor_wm_info.cacheline_size);
1082                 cursor_sr = i965_cursor_wm_info.fifo_size -
1083                         (entries + i965_cursor_wm_info.guard_size);
1084
1085                 if (cursor_sr > i965_cursor_wm_info.max_wm)
1086                         cursor_sr = i965_cursor_wm_info.max_wm;
1087
1088                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1089                               "cursor %d\n", srwm, cursor_sr);
1090
1091                 cxsr_enabled = true;
1092         } else {
1093                 cxsr_enabled = false;
1094                 /* Turn off self refresh if both pipes are enabled */
1095                 intel_set_memory_cxsr(dev_priv, false);
1096         }
1097
1098         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1099                       srwm);
1100
1101         /* 965 has limitations... */
1102         I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
1103                    (8 << DSPFW_CURSORB_SHIFT) |
1104                    (8 << DSPFW_PLANEB_SHIFT) |
1105                    (8 << DSPFW_PLANEA_SHIFT));
1106         I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) |
1107                    (8 << DSPFW_PLANEC_SHIFT_OLD));
1108         /* update cursor SR watermark */
1109         I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1110
1111         if (cxsr_enabled)
1112                 intel_set_memory_cxsr(dev_priv, true);
1113 }
1114
1115 static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1116 {
1117         struct drm_device *dev = unused_crtc->dev;
1118         struct drm_i915_private *dev_priv = dev->dev_private;
1119         const struct intel_watermark_params *wm_info;
1120         uint32_t fwater_lo;
1121         uint32_t fwater_hi;
1122         int cwm, srwm = 1;
1123         int fifo_size;
1124         int planea_wm, planeb_wm;
1125         struct drm_crtc *crtc, *enabled = NULL;
1126
1127         if (IS_I945GM(dev))
1128                 wm_info = &i945_wm_info;
1129         else if (!IS_GEN2(dev))
1130                 wm_info = &i915_wm_info;
1131         else
1132                 wm_info = &i830_a_wm_info;
1133
1134         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1135         crtc = intel_get_crtc_for_plane(dev, 0);
1136         if (intel_crtc_active(crtc)) {
1137                 const struct drm_display_mode *adjusted_mode;
1138                 int cpp = crtc->primary->fb->bits_per_pixel / 8;
1139                 if (IS_GEN2(dev))
1140                         cpp = 4;
1141
1142                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1143                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1144                                                wm_info, fifo_size, cpp,
1145                                                pessimal_latency_ns);
1146                 enabled = crtc;
1147         } else {
1148                 planea_wm = fifo_size - wm_info->guard_size;
1149                 if (planea_wm > (long)wm_info->max_wm)
1150                         planea_wm = wm_info->max_wm;
1151         }
1152
1153         if (IS_GEN2(dev))
1154                 wm_info = &i830_bc_wm_info;
1155
1156         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1157         crtc = intel_get_crtc_for_plane(dev, 1);
1158         if (intel_crtc_active(crtc)) {
1159                 const struct drm_display_mode *adjusted_mode;
1160                 int cpp = crtc->primary->fb->bits_per_pixel / 8;
1161                 if (IS_GEN2(dev))
1162                         cpp = 4;
1163
1164                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1165                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1166                                                wm_info, fifo_size, cpp,
1167                                                pessimal_latency_ns);
1168                 if (enabled == NULL)
1169                         enabled = crtc;
1170                 else
1171                         enabled = NULL;
1172         } else {
1173                 planeb_wm = fifo_size - wm_info->guard_size;
1174                 if (planeb_wm > (long)wm_info->max_wm)
1175                         planeb_wm = wm_info->max_wm;
1176         }
1177
1178         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1179
1180         if (IS_I915GM(dev) && enabled) {
1181                 struct drm_i915_gem_object *obj;
1182
1183                 obj = intel_fb_obj(enabled->primary->fb);
1184
1185                 /* self-refresh seems busted with untiled */
1186                 if (obj->tiling_mode == I915_TILING_NONE)
1187                         enabled = NULL;
1188         }
1189
1190         /*
1191          * Overlay gets an aggressive default since video jitter is bad.
1192          */
1193         cwm = 2;
1194
1195         /* Play safe and disable self-refresh before adjusting watermarks. */
1196         intel_set_memory_cxsr(dev_priv, false);
1197
1198         /* Calc sr entries for one plane configs */
1199         if (HAS_FW_BLC(dev) && enabled) {
1200                 /* self-refresh has much higher latency */
1201                 static const int sr_latency_ns = 6000;
1202                 const struct drm_display_mode *adjusted_mode =
1203                         &to_intel_crtc(enabled)->config->base.adjusted_mode;
1204                 int clock = adjusted_mode->crtc_clock;
1205                 int htotal = adjusted_mode->crtc_htotal;
1206                 int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
1207                 int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
1208                 unsigned long line_time_us;
1209                 int entries;
1210
1211                 line_time_us = max(htotal * 1000 / clock, 1);
1212
1213                 /* Use ns/us then divide to preserve precision */
1214                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1215                         pixel_size * hdisplay;
1216                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1217                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1218                 srwm = wm_info->fifo_size - entries;
1219                 if (srwm < 0)
1220                         srwm = 1;
1221
1222                 if (IS_I945G(dev) || IS_I945GM(dev))
1223                         I915_WRITE(FW_BLC_SELF,
1224                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1225                 else if (IS_I915GM(dev))
1226                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1227         }
1228
1229         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1230                       planea_wm, planeb_wm, cwm, srwm);
1231
1232         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1233         fwater_hi = (cwm & 0x1f);
1234
1235         /* Set request length to 8 cachelines per fetch */
1236         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1237         fwater_hi = fwater_hi | (1 << 8);
1238
1239         I915_WRITE(FW_BLC, fwater_lo);
1240         I915_WRITE(FW_BLC2, fwater_hi);
1241
1242         if (enabled)
1243                 intel_set_memory_cxsr(dev_priv, true);
1244 }
1245
1246 static void i845_update_wm(struct drm_crtc *unused_crtc)
1247 {
1248         struct drm_device *dev = unused_crtc->dev;
1249         struct drm_i915_private *dev_priv = dev->dev_private;
1250         struct drm_crtc *crtc;
1251         const struct drm_display_mode *adjusted_mode;
1252         uint32_t fwater_lo;
1253         int planea_wm;
1254
1255         crtc = single_enabled_crtc(dev);
1256         if (crtc == NULL)
1257                 return;
1258
1259         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1260         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1261                                        &i845_wm_info,
1262                                        dev_priv->display.get_fifo_size(dev, 0),
1263                                        4, pessimal_latency_ns);
1264         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1265         fwater_lo |= (3<<8) | planea_wm;
1266
1267         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1268
1269         I915_WRITE(FW_BLC, fwater_lo);
1270 }
1271
1272 static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
1273                                     struct drm_crtc *crtc)
1274 {
1275         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1276         uint32_t pixel_rate;
1277
1278         pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock;
1279
1280         /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1281          * adjust the pixel_rate here. */
1282
1283         if (intel_crtc->config->pch_pfit.enabled) {
1284                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
1285                 uint32_t pfit_size = intel_crtc->config->pch_pfit.size;
1286
1287                 pipe_w = intel_crtc->config->pipe_src_w;
1288                 pipe_h = intel_crtc->config->pipe_src_h;
1289                 pfit_w = (pfit_size >> 16) & 0xFFFF;
1290                 pfit_h = pfit_size & 0xFFFF;
1291                 if (pipe_w < pfit_w)
1292                         pipe_w = pfit_w;
1293                 if (pipe_h < pfit_h)
1294                         pipe_h = pfit_h;
1295
1296                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1297                                      pfit_w * pfit_h);
1298         }
1299
1300         return pixel_rate;
1301 }
1302
1303 /* latency must be in 0.1us units. */
1304 static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
1305                                uint32_t latency)
1306 {
1307         uint64_t ret;
1308
1309         if (WARN(latency == 0, "Latency value missing\n"))
1310                 return UINT_MAX;
1311
1312         ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1313         ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1314
1315         return ret;
1316 }
1317
1318 /* latency must be in 0.1us units. */
1319 static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
1320                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1321                                uint32_t latency)
1322 {
1323         uint32_t ret;
1324
1325         if (WARN(latency == 0, "Latency value missing\n"))
1326                 return UINT_MAX;
1327
1328         ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1329         ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1330         ret = DIV_ROUND_UP(ret, 64) + 2;
1331         return ret;
1332 }
1333
1334 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
1335                            uint8_t bytes_per_pixel)
1336 {
1337         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1338 }
1339
1340 struct skl_pipe_wm_parameters {
1341         bool active;
1342         uint32_t pipe_htotal;
1343         uint32_t pixel_rate; /* in KHz */
1344         struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
1345         struct intel_plane_wm_parameters cursor;
1346 };
1347
1348 struct ilk_pipe_wm_parameters {
1349         bool active;
1350         uint32_t pipe_htotal;
1351         uint32_t pixel_rate;
1352         struct intel_plane_wm_parameters pri;
1353         struct intel_plane_wm_parameters spr;
1354         struct intel_plane_wm_parameters cur;
1355 };
1356
1357 struct ilk_wm_maximums {
1358         uint16_t pri;
1359         uint16_t spr;
1360         uint16_t cur;
1361         uint16_t fbc;
1362 };
1363
1364 /* used in computing the new watermarks state */
1365 struct intel_wm_config {
1366         unsigned int num_pipes_active;
1367         bool sprites_enabled;
1368         bool sprites_scaled;
1369 };
1370
1371 /*
1372  * For both WM_PIPE and WM_LP.
1373  * mem_value must be in 0.1us units.
1374  */
1375 static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
1376                                    uint32_t mem_value,
1377                                    bool is_lp)
1378 {
1379         uint32_t method1, method2;
1380
1381         if (!params->active || !params->pri.enabled)
1382                 return 0;
1383
1384         method1 = ilk_wm_method1(params->pixel_rate,
1385                                  params->pri.bytes_per_pixel,
1386                                  mem_value);
1387
1388         if (!is_lp)
1389                 return method1;
1390
1391         method2 = ilk_wm_method2(params->pixel_rate,
1392                                  params->pipe_htotal,
1393                                  params->pri.horiz_pixels,
1394                                  params->pri.bytes_per_pixel,
1395                                  mem_value);
1396
1397         return min(method1, method2);
1398 }
1399
1400 /*
1401  * For both WM_PIPE and WM_LP.
1402  * mem_value must be in 0.1us units.
1403  */
1404 static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
1405                                    uint32_t mem_value)
1406 {
1407         uint32_t method1, method2;
1408
1409         if (!params->active || !params->spr.enabled)
1410                 return 0;
1411
1412         method1 = ilk_wm_method1(params->pixel_rate,
1413                                  params->spr.bytes_per_pixel,
1414                                  mem_value);
1415         method2 = ilk_wm_method2(params->pixel_rate,
1416                                  params->pipe_htotal,
1417                                  params->spr.horiz_pixels,
1418                                  params->spr.bytes_per_pixel,
1419                                  mem_value);
1420         return min(method1, method2);
1421 }
1422
1423 /*
1424  * For both WM_PIPE and WM_LP.
1425  * mem_value must be in 0.1us units.
1426  */
1427 static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
1428                                    uint32_t mem_value)
1429 {
1430         if (!params->active || !params->cur.enabled)
1431                 return 0;
1432
1433         return ilk_wm_method2(params->pixel_rate,
1434                               params->pipe_htotal,
1435                               params->cur.horiz_pixels,
1436                               params->cur.bytes_per_pixel,
1437                               mem_value);
1438 }
1439
1440 /* Only for WM_LP. */
1441 static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1442                                    uint32_t pri_val)
1443 {
1444         if (!params->active || !params->pri.enabled)
1445                 return 0;
1446
1447         return ilk_wm_fbc(pri_val,
1448                           params->pri.horiz_pixels,
1449                           params->pri.bytes_per_pixel);
1450 }
1451
1452 static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1453 {
1454         if (INTEL_INFO(dev)->gen >= 8)
1455                 return 3072;
1456         else if (INTEL_INFO(dev)->gen >= 7)
1457                 return 768;
1458         else
1459                 return 512;
1460 }
1461
1462 static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1463                                          int level, bool is_sprite)
1464 {
1465         if (INTEL_INFO(dev)->gen >= 8)
1466                 /* BDW primary/sprite plane watermarks */
1467                 return level == 0 ? 255 : 2047;
1468         else if (INTEL_INFO(dev)->gen >= 7)
1469                 /* IVB/HSW primary/sprite plane watermarks */
1470                 return level == 0 ? 127 : 1023;
1471         else if (!is_sprite)
1472                 /* ILK/SNB primary plane watermarks */
1473                 return level == 0 ? 127 : 511;
1474         else
1475                 /* ILK/SNB sprite plane watermarks */
1476                 return level == 0 ? 63 : 255;
1477 }
1478
1479 static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
1480                                           int level)
1481 {
1482         if (INTEL_INFO(dev)->gen >= 7)
1483                 return level == 0 ? 63 : 255;
1484         else
1485                 return level == 0 ? 31 : 63;
1486 }
1487
1488 static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
1489 {
1490         if (INTEL_INFO(dev)->gen >= 8)
1491                 return 31;
1492         else
1493                 return 15;
1494 }
1495
1496 /* Calculate the maximum primary/sprite plane watermark */
1497 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1498                                      int level,
1499                                      const struct intel_wm_config *config,
1500                                      enum intel_ddb_partitioning ddb_partitioning,
1501                                      bool is_sprite)
1502 {
1503         unsigned int fifo_size = ilk_display_fifo_size(dev);
1504
1505         /* if sprites aren't enabled, sprites get nothing */
1506         if (is_sprite && !config->sprites_enabled)
1507                 return 0;
1508
1509         /* HSW allows LP1+ watermarks even with multiple pipes */
1510         if (level == 0 || config->num_pipes_active > 1) {
1511                 fifo_size /= INTEL_INFO(dev)->num_pipes;
1512
1513                 /*
1514                  * For some reason the non self refresh
1515                  * FIFO size is only half of the self
1516                  * refresh FIFO size on ILK/SNB.
1517                  */
1518                 if (INTEL_INFO(dev)->gen <= 6)
1519                         fifo_size /= 2;
1520         }
1521
1522         if (config->sprites_enabled) {
1523                 /* level 0 is always calculated with 1:1 split */
1524                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1525                         if (is_sprite)
1526                                 fifo_size *= 5;
1527                         fifo_size /= 6;
1528                 } else {
1529                         fifo_size /= 2;
1530                 }
1531         }
1532
1533         /* clamp to max that the registers can hold */
1534         return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
1535 }
1536
1537 /* Calculate the maximum cursor plane watermark */
1538 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1539                                       int level,
1540                                       const struct intel_wm_config *config)
1541 {
1542         /* HSW LP1+ watermarks w/ multiple pipes */
1543         if (level > 0 && config->num_pipes_active > 1)
1544                 return 64;
1545
1546         /* otherwise just report max that registers can hold */
1547         return ilk_cursor_wm_reg_max(dev, level);
1548 }
1549
1550 static void ilk_compute_wm_maximums(const struct drm_device *dev,
1551                                     int level,
1552                                     const struct intel_wm_config *config,
1553                                     enum intel_ddb_partitioning ddb_partitioning,
1554                                     struct ilk_wm_maximums *max)
1555 {
1556         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1557         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1558         max->cur = ilk_cursor_wm_max(dev, level, config);
1559         max->fbc = ilk_fbc_wm_reg_max(dev);
1560 }
1561
1562 static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
1563                                         int level,
1564                                         struct ilk_wm_maximums *max)
1565 {
1566         max->pri = ilk_plane_wm_reg_max(dev, level, false);
1567         max->spr = ilk_plane_wm_reg_max(dev, level, true);
1568         max->cur = ilk_cursor_wm_reg_max(dev, level);
1569         max->fbc = ilk_fbc_wm_reg_max(dev);
1570 }
1571
1572 static bool ilk_validate_wm_level(int level,
1573                                   const struct ilk_wm_maximums *max,
1574                                   struct intel_wm_level *result)
1575 {
1576         bool ret;
1577
1578         /* already determined to be invalid? */
1579         if (!result->enable)
1580                 return false;
1581
1582         result->enable = result->pri_val <= max->pri &&
1583                          result->spr_val <= max->spr &&
1584                          result->cur_val <= max->cur;
1585
1586         ret = result->enable;
1587
1588         /*
1589          * HACK until we can pre-compute everything,
1590          * and thus fail gracefully if LP0 watermarks
1591          * are exceeded...
1592          */
1593         if (level == 0 && !result->enable) {
1594                 if (result->pri_val > max->pri)
1595                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
1596                                       level, result->pri_val, max->pri);
1597                 if (result->spr_val > max->spr)
1598                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
1599                                       level, result->spr_val, max->spr);
1600                 if (result->cur_val > max->cur)
1601                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
1602                                       level, result->cur_val, max->cur);
1603
1604                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
1605                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
1606                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
1607                 result->enable = true;
1608         }
1609
1610         return ret;
1611 }
1612
1613 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
1614                                  int level,
1615                                  const struct ilk_pipe_wm_parameters *p,
1616                                  struct intel_wm_level *result)
1617 {
1618         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
1619         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
1620         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
1621
1622         /* WM1+ latency values stored in 0.5us units */
1623         if (level > 0) {
1624                 pri_latency *= 5;
1625                 spr_latency *= 5;
1626                 cur_latency *= 5;
1627         }
1628
1629         result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
1630         result->spr_val = ilk_compute_spr_wm(p, spr_latency);
1631         result->cur_val = ilk_compute_cur_wm(p, cur_latency);
1632         result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
1633         result->enable = true;
1634 }
1635
1636 static uint32_t
1637 hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
1638 {
1639         struct drm_i915_private *dev_priv = dev->dev_private;
1640         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1641         struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
1642         u32 linetime, ips_linetime;
1643
1644         if (!intel_crtc_active(crtc))
1645                 return 0;
1646
1647         /* The WM are computed with base on how long it takes to fill a single
1648          * row at the given clock rate, multiplied by 8.
1649          * */
1650         linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
1651                                      mode->crtc_clock);
1652         ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
1653                                          intel_ddi_get_cdclk_freq(dev_priv));
1654
1655         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
1656                PIPE_WM_LINETIME_TIME(linetime);
1657 }
1658
1659 static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
1660 {
1661         struct drm_i915_private *dev_priv = dev->dev_private;
1662
1663         if (IS_GEN9(dev)) {
1664                 uint32_t val;
1665                 int ret, i;
1666                 int level, max_level = ilk_wm_max_level(dev);
1667
1668                 /* read the first set of memory latencies[0:3] */
1669                 val = 0; /* data0 to be programmed to 0 for first set */
1670                 mutex_lock(&dev_priv->rps.hw_lock);
1671                 ret = sandybridge_pcode_read(dev_priv,
1672                                              GEN9_PCODE_READ_MEM_LATENCY,
1673                                              &val);
1674                 mutex_unlock(&dev_priv->rps.hw_lock);
1675
1676                 if (ret) {
1677                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
1678                         return;
1679                 }
1680
1681                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
1682                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
1683                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1684                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
1685                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1686                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
1687                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1688
1689                 /* read the second set of memory latencies[4:7] */
1690                 val = 1; /* data0 to be programmed to 1 for second set */
1691                 mutex_lock(&dev_priv->rps.hw_lock);
1692                 ret = sandybridge_pcode_read(dev_priv,
1693                                              GEN9_PCODE_READ_MEM_LATENCY,
1694                                              &val);
1695                 mutex_unlock(&dev_priv->rps.hw_lock);
1696                 if (ret) {
1697                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
1698                         return;
1699                 }
1700
1701                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
1702                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
1703                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1704                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
1705                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1706                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
1707                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1708
1709                 /*
1710                  * WaWmMemoryReadLatency:skl
1711                  *
1712                  * punit doesn't take into account the read latency so we need
1713                  * to add 2us to the various latency levels we retrieve from
1714                  * the punit.
1715                  *   - W0 is a bit special in that it's the only level that
1716                  *   can't be disabled if we want to have display working, so
1717                  *   we always add 2us there.
1718                  *   - For levels >=1, punit returns 0us latency when they are
1719                  *   disabled, so we respect that and don't add 2us then
1720                  *
1721                  * Additionally, if a level n (n > 1) has a 0us latency, all
1722                  * levels m (m >= n) need to be disabled. We make sure to
1723                  * sanitize the values out of the punit to satisfy this
1724                  * requirement.
1725                  */
1726                 wm[0] += 2;
1727                 for (level = 1; level <= max_level; level++)
1728                         if (wm[level] != 0)
1729                                 wm[level] += 2;
1730                         else {
1731                                 for (i = level + 1; i <= max_level; i++)
1732                                         wm[i] = 0;
1733
1734                                 break;
1735                         }
1736         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
1737                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
1738
1739                 wm[0] = (sskpd >> 56) & 0xFF;
1740                 if (wm[0] == 0)
1741                         wm[0] = sskpd & 0xF;
1742                 wm[1] = (sskpd >> 4) & 0xFF;
1743                 wm[2] = (sskpd >> 12) & 0xFF;
1744                 wm[3] = (sskpd >> 20) & 0x1FF;
1745                 wm[4] = (sskpd >> 32) & 0x1FF;
1746         } else if (INTEL_INFO(dev)->gen >= 6) {
1747                 uint32_t sskpd = I915_READ(MCH_SSKPD);
1748
1749                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
1750                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
1751                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
1752                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
1753         } else if (INTEL_INFO(dev)->gen >= 5) {
1754                 uint32_t mltr = I915_READ(MLTR_ILK);
1755
1756                 /* ILK primary LP0 latency is 700 ns */
1757                 wm[0] = 7;
1758                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
1759                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
1760         }
1761 }
1762
1763 static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
1764 {
1765         /* ILK sprite LP0 latency is 1300 ns */
1766         if (INTEL_INFO(dev)->gen == 5)
1767                 wm[0] = 13;
1768 }
1769
1770 static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
1771 {
1772         /* ILK cursor LP0 latency is 1300 ns */
1773         if (INTEL_INFO(dev)->gen == 5)
1774                 wm[0] = 13;
1775
1776         /* WaDoubleCursorLP3Latency:ivb */
1777         if (IS_IVYBRIDGE(dev))
1778                 wm[3] *= 2;
1779 }
1780
1781 int ilk_wm_max_level(const struct drm_device *dev)
1782 {
1783         /* how many WM levels are we expecting */
1784         if (IS_GEN9(dev))
1785                 return 7;
1786         else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
1787                 return 4;
1788         else if (INTEL_INFO(dev)->gen >= 6)
1789                 return 3;
1790         else
1791                 return 2;
1792 }
1793
1794 static void intel_print_wm_latency(struct drm_device *dev,
1795                                    const char *name,
1796                                    const uint16_t wm[8])
1797 {
1798         int level, max_level = ilk_wm_max_level(dev);
1799
1800         for (level = 0; level <= max_level; level++) {
1801                 unsigned int latency = wm[level];
1802
1803                 if (latency == 0) {
1804                         DRM_ERROR("%s WM%d latency not provided\n",
1805                                   name, level);
1806                         continue;
1807                 }
1808
1809                 /*
1810                  * - latencies are in us on gen9.
1811                  * - before then, WM1+ latency values are in 0.5us units
1812                  */
1813                 if (IS_GEN9(dev))
1814                         latency *= 10;
1815                 else if (level > 0)
1816                         latency *= 5;
1817
1818                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
1819                               name, level, wm[level],
1820                               latency / 10, latency % 10);
1821         }
1822 }
1823
1824 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
1825                                     uint16_t wm[5], uint16_t min)
1826 {
1827         int level, max_level = ilk_wm_max_level(dev_priv->dev);
1828
1829         if (wm[0] >= min)
1830                 return false;
1831
1832         wm[0] = max(wm[0], min);
1833         for (level = 1; level <= max_level; level++)
1834                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
1835
1836         return true;
1837 }
1838
1839 static void snb_wm_latency_quirk(struct drm_device *dev)
1840 {
1841         struct drm_i915_private *dev_priv = dev->dev_private;
1842         bool changed;
1843
1844         /*
1845          * The BIOS provided WM memory latency values are often
1846          * inadequate for high resolution displays. Adjust them.
1847          */
1848         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
1849                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
1850                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
1851
1852         if (!changed)
1853                 return;
1854
1855         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
1856         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
1857         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
1858         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
1859 }
1860
1861 static void ilk_setup_wm_latency(struct drm_device *dev)
1862 {
1863         struct drm_i915_private *dev_priv = dev->dev_private;
1864
1865         intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
1866
1867         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
1868                sizeof(dev_priv->wm.pri_latency));
1869         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
1870                sizeof(dev_priv->wm.pri_latency));
1871
1872         intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
1873         intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
1874
1875         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
1876         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
1877         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
1878
1879         if (IS_GEN6(dev))
1880                 snb_wm_latency_quirk(dev);
1881 }
1882
1883 static void skl_setup_wm_latency(struct drm_device *dev)
1884 {
1885         struct drm_i915_private *dev_priv = dev->dev_private;
1886
1887         intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
1888         intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
1889 }
1890
1891 static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
1892                                       struct ilk_pipe_wm_parameters *p)
1893 {
1894         struct drm_device *dev = crtc->dev;
1895         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1896         enum pipe pipe = intel_crtc->pipe;
1897         struct drm_plane *plane;
1898
1899         if (!intel_crtc_active(crtc))
1900                 return;
1901
1902         p->active = true;
1903         p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
1904         p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
1905         p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
1906         p->cur.bytes_per_pixel = 4;
1907         p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
1908         p->cur.horiz_pixels = intel_crtc->cursor_width;
1909         /* TODO: for now, assume primary and cursor planes are always enabled. */
1910         p->pri.enabled = true;
1911         p->cur.enabled = true;
1912
1913         drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
1914                 struct intel_plane *intel_plane = to_intel_plane(plane);
1915
1916                 if (intel_plane->pipe == pipe) {
1917                         p->spr = intel_plane->wm;
1918                         break;
1919                 }
1920         }
1921 }
1922
1923 static void ilk_compute_wm_config(struct drm_device *dev,
1924                                   struct intel_wm_config *config)
1925 {
1926         struct intel_crtc *intel_crtc;
1927
1928         /* Compute the currently _active_ config */
1929         for_each_intel_crtc(dev, intel_crtc) {
1930                 const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
1931
1932                 if (!wm->pipe_enabled)
1933                         continue;
1934
1935                 config->sprites_enabled |= wm->sprites_enabled;
1936                 config->sprites_scaled |= wm->sprites_scaled;
1937                 config->num_pipes_active++;
1938         }
1939 }
1940
1941 /* Compute new watermarks for the pipe */
1942 static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
1943                                   const struct ilk_pipe_wm_parameters *params,
1944                                   struct intel_pipe_wm *pipe_wm)
1945 {
1946         struct drm_device *dev = crtc->dev;
1947         const struct drm_i915_private *dev_priv = dev->dev_private;
1948         int level, max_level = ilk_wm_max_level(dev);
1949         /* LP0 watermark maximums depend on this pipe alone */
1950         struct intel_wm_config config = {
1951                 .num_pipes_active = 1,
1952                 .sprites_enabled = params->spr.enabled,
1953                 .sprites_scaled = params->spr.scaled,
1954         };
1955         struct ilk_wm_maximums max;
1956
1957         pipe_wm->pipe_enabled = params->active;
1958         pipe_wm->sprites_enabled = params->spr.enabled;
1959         pipe_wm->sprites_scaled = params->spr.scaled;
1960
1961         /* ILK/SNB: LP2+ watermarks only w/o sprites */
1962         if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
1963                 max_level = 1;
1964
1965         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
1966         if (params->spr.scaled)
1967                 max_level = 0;
1968
1969         ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
1970
1971         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
1972                 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
1973
1974         /* LP0 watermarks always use 1/2 DDB partitioning */
1975         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
1976
1977         /* At least LP0 must be valid */
1978         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
1979                 return false;
1980
1981         ilk_compute_wm_reg_maximums(dev, 1, &max);
1982
1983         for (level = 1; level <= max_level; level++) {
1984                 struct intel_wm_level wm = {};
1985
1986                 ilk_compute_wm_level(dev_priv, level, params, &wm);
1987
1988                 /*
1989                  * Disable any watermark level that exceeds the
1990                  * register maximums since such watermarks are
1991                  * always invalid.
1992                  */
1993                 if (!ilk_validate_wm_level(level, &max, &wm))
1994                         break;
1995
1996                 pipe_wm->wm[level] = wm;
1997         }
1998
1999         return true;
2000 }
2001
2002 /*
2003  * Merge the watermarks from all active pipes for a specific level.
2004  */
2005 static void ilk_merge_wm_level(struct drm_device *dev,
2006                                int level,
2007                                struct intel_wm_level *ret_wm)
2008 {
2009         const struct intel_crtc *intel_crtc;
2010
2011         ret_wm->enable = true;
2012
2013         for_each_intel_crtc(dev, intel_crtc) {
2014                 const struct intel_pipe_wm *active = &intel_crtc->wm.active;
2015                 const struct intel_wm_level *wm = &active->wm[level];
2016
2017                 if (!active->pipe_enabled)
2018                         continue;
2019
2020                 /*
2021                  * The watermark values may have been used in the past,
2022                  * so we must maintain them in the registers for some
2023                  * time even if the level is now disabled.
2024                  */
2025                 if (!wm->enable)
2026                         ret_wm->enable = false;
2027
2028                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2029                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2030                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2031                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2032         }
2033 }
2034
2035 /*
2036  * Merge all low power watermarks for all active pipes.
2037  */
2038 static void ilk_wm_merge(struct drm_device *dev,
2039                          const struct intel_wm_config *config,
2040                          const struct ilk_wm_maximums *max,
2041                          struct intel_pipe_wm *merged)
2042 {
2043         int level, max_level = ilk_wm_max_level(dev);
2044         int last_enabled_level = max_level;
2045
2046         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2047         if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2048             config->num_pipes_active > 1)
2049                 return;
2050
2051         /* ILK: FBC WM must be disabled always */
2052         merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
2053
2054         /* merge each WM1+ level */
2055         for (level = 1; level <= max_level; level++) {
2056                 struct intel_wm_level *wm = &merged->wm[level];
2057
2058                 ilk_merge_wm_level(dev, level, wm);
2059
2060                 if (level > last_enabled_level)
2061                         wm->enable = false;
2062                 else if (!ilk_validate_wm_level(level, max, wm))
2063                         /* make sure all following levels get disabled */
2064                         last_enabled_level = level - 1;
2065
2066                 /*
2067                  * The spec says it is preferred to disable
2068                  * FBC WMs instead of disabling a WM level.
2069                  */
2070                 if (wm->fbc_val > max->fbc) {
2071                         if (wm->enable)
2072                                 merged->fbc_wm_enabled = false;
2073                         wm->fbc_val = 0;
2074                 }
2075         }
2076
2077         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2078         /*
2079          * FIXME this is racy. FBC might get enabled later.
2080          * What we should check here is whether FBC can be
2081          * enabled sometime later.
2082          */
2083         if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
2084                 for (level = 2; level <= max_level; level++) {
2085                         struct intel_wm_level *wm = &merged->wm[level];
2086
2087                         wm->enable = false;
2088                 }
2089         }
2090 }
2091
2092 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2093 {
2094         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2095         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2096 }
2097
2098 /* The value we need to program into the WM_LPx latency field */
2099 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2100 {
2101         struct drm_i915_private *dev_priv = dev->dev_private;
2102
2103         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2104                 return 2 * level;
2105         else
2106                 return dev_priv->wm.pri_latency[level];
2107 }
2108
2109 static void ilk_compute_wm_results(struct drm_device *dev,
2110                                    const struct intel_pipe_wm *merged,
2111                                    enum intel_ddb_partitioning partitioning,
2112                                    struct ilk_wm_values *results)
2113 {
2114         struct intel_crtc *intel_crtc;
2115         int level, wm_lp;
2116
2117         results->enable_fbc_wm = merged->fbc_wm_enabled;
2118         results->partitioning = partitioning;
2119
2120         /* LP1+ register values */
2121         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2122                 const struct intel_wm_level *r;
2123
2124                 level = ilk_wm_lp_to_level(wm_lp, merged);
2125
2126                 r = &merged->wm[level];
2127
2128                 /*
2129                  * Maintain the watermark values even if the level is
2130                  * disabled. Doing otherwise could cause underruns.
2131                  */
2132                 results->wm_lp[wm_lp - 1] =
2133                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2134                         (r->pri_val << WM1_LP_SR_SHIFT) |
2135                         r->cur_val;
2136
2137                 if (r->enable)
2138                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2139
2140                 if (INTEL_INFO(dev)->gen >= 8)
2141                         results->wm_lp[wm_lp - 1] |=
2142                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2143                 else
2144                         results->wm_lp[wm_lp - 1] |=
2145                                 r->fbc_val << WM1_LP_FBC_SHIFT;
2146
2147                 /*
2148                  * Always set WM1S_LP_EN when spr_val != 0, even if the
2149                  * level is disabled. Doing otherwise could cause underruns.
2150                  */
2151                 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2152                         WARN_ON(wm_lp != 1);
2153                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2154                 } else
2155                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2156         }
2157
2158         /* LP0 register values */
2159         for_each_intel_crtc(dev, intel_crtc) {
2160                 enum pipe pipe = intel_crtc->pipe;
2161                 const struct intel_wm_level *r =
2162                         &intel_crtc->wm.active.wm[0];
2163
2164                 if (WARN_ON(!r->enable))
2165                         continue;
2166
2167                 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
2168
2169                 results->wm_pipe[pipe] =
2170                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2171                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2172                         r->cur_val;
2173         }
2174 }
2175
2176 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
2177  * case both are at the same level. Prefer r1 in case they're the same. */
2178 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2179                                                   struct intel_pipe_wm *r1,
2180                                                   struct intel_pipe_wm *r2)
2181 {
2182         int level, max_level = ilk_wm_max_level(dev);
2183         int level1 = 0, level2 = 0;
2184
2185         for (level = 1; level <= max_level; level++) {
2186                 if (r1->wm[level].enable)
2187                         level1 = level;
2188                 if (r2->wm[level].enable)
2189                         level2 = level;
2190         }
2191
2192         if (level1 == level2) {
2193                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2194                         return r2;
2195                 else
2196                         return r1;
2197         } else if (level1 > level2) {
2198                 return r1;
2199         } else {
2200                 return r2;
2201         }
2202 }
2203
2204 /* dirty bits used to track which watermarks need changes */
2205 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2206 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2207 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2208 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2209 #define WM_DIRTY_FBC (1 << 24)
2210 #define WM_DIRTY_DDB (1 << 25)
2211
2212 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
2213                                          const struct ilk_wm_values *old,
2214                                          const struct ilk_wm_values *new)
2215 {
2216         unsigned int dirty = 0;
2217         enum pipe pipe;
2218         int wm_lp;
2219
2220         for_each_pipe(dev_priv, pipe) {
2221                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2222                         dirty |= WM_DIRTY_LINETIME(pipe);
2223                         /* Must disable LP1+ watermarks too */
2224                         dirty |= WM_DIRTY_LP_ALL;
2225                 }
2226
2227                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2228                         dirty |= WM_DIRTY_PIPE(pipe);
2229                         /* Must disable LP1+ watermarks too */
2230                         dirty |= WM_DIRTY_LP_ALL;
2231                 }
2232         }
2233
2234         if (old->enable_fbc_wm != new->enable_fbc_wm) {
2235                 dirty |= WM_DIRTY_FBC;
2236                 /* Must disable LP1+ watermarks too */
2237                 dirty |= WM_DIRTY_LP_ALL;
2238         }
2239
2240         if (old->partitioning != new->partitioning) {
2241                 dirty |= WM_DIRTY_DDB;
2242                 /* Must disable LP1+ watermarks too */
2243                 dirty |= WM_DIRTY_LP_ALL;
2244         }
2245
2246         /* LP1+ watermarks already deemed dirty, no need to continue */
2247         if (dirty & WM_DIRTY_LP_ALL)
2248                 return dirty;
2249
2250         /* Find the lowest numbered LP1+ watermark in need of an update... */
2251         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2252                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2253                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2254                         break;
2255         }
2256
2257         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2258         for (; wm_lp <= 3; wm_lp++)
2259                 dirty |= WM_DIRTY_LP(wm_lp);
2260
2261         return dirty;
2262 }
2263
2264 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2265                                unsigned int dirty)
2266 {
2267         struct ilk_wm_values *previous = &dev_priv->wm.hw;
2268         bool changed = false;
2269
2270         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2271                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2272                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2273                 changed = true;
2274         }
2275         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2276                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2277                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2278                 changed = true;
2279         }
2280         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2281                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2282                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2283                 changed = true;
2284         }
2285
2286         /*
2287          * Don't touch WM1S_LP_EN here.
2288          * Doing so could cause underruns.
2289          */
2290
2291         return changed;
2292 }
2293
2294 /*
2295  * The spec says we shouldn't write when we don't need, because every write
2296  * causes WMs to be re-evaluated, expending some power.
2297  */
2298 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2299                                 struct ilk_wm_values *results)
2300 {
2301         struct drm_device *dev = dev_priv->dev;
2302         struct ilk_wm_values *previous = &dev_priv->wm.hw;
2303         unsigned int dirty;
2304         uint32_t val;
2305
2306         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
2307         if (!dirty)
2308                 return;
2309
2310         _ilk_disable_lp_wm(dev_priv, dirty);
2311
2312         if (dirty & WM_DIRTY_PIPE(PIPE_A))
2313                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2314         if (dirty & WM_DIRTY_PIPE(PIPE_B))
2315                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
2316         if (dirty & WM_DIRTY_PIPE(PIPE_C))
2317                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2318
2319         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
2320                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
2321         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
2322                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
2323         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
2324                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2325
2326         if (dirty & WM_DIRTY_DDB) {
2327                 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2328                         val = I915_READ(WM_MISC);
2329                         if (results->partitioning == INTEL_DDB_PART_1_2)
2330                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
2331                         else
2332                                 val |= WM_MISC_DATA_PARTITION_5_6;
2333                         I915_WRITE(WM_MISC, val);
2334                 } else {
2335                         val = I915_READ(DISP_ARB_CTL2);
2336                         if (results->partitioning == INTEL_DDB_PART_1_2)
2337                                 val &= ~DISP_DATA_PARTITION_5_6;
2338                         else
2339                                 val |= DISP_DATA_PARTITION_5_6;
2340                         I915_WRITE(DISP_ARB_CTL2, val);
2341                 }
2342         }
2343
2344         if (dirty & WM_DIRTY_FBC) {
2345                 val = I915_READ(DISP_ARB_CTL);
2346                 if (results->enable_fbc_wm)
2347                         val &= ~DISP_FBC_WM_DIS;
2348                 else
2349                         val |= DISP_FBC_WM_DIS;
2350                 I915_WRITE(DISP_ARB_CTL, val);
2351         }
2352
2353         if (dirty & WM_DIRTY_LP(1) &&
2354             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2355                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2356
2357         if (INTEL_INFO(dev)->gen >= 7) {
2358                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2359                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2360                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2361                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2362         }
2363
2364         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
2365                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
2366         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
2367                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
2368         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
2369                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
2370
2371         dev_priv->wm.hw = *results;
2372 }
2373
2374 static bool ilk_disable_lp_wm(struct drm_device *dev)
2375 {
2376         struct drm_i915_private *dev_priv = dev->dev_private;
2377
2378         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2379 }
2380
2381 /*
2382  * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
2383  * different active planes.
2384  */
2385
2386 #define SKL_DDB_SIZE            896     /* in blocks */
2387
2388 static void
2389 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2390                                    struct drm_crtc *for_crtc,
2391                                    const struct intel_wm_config *config,
2392                                    const struct skl_pipe_wm_parameters *params,
2393                                    struct skl_ddb_entry *alloc /* out */)
2394 {
2395         struct drm_crtc *crtc;
2396         unsigned int pipe_size, ddb_size;
2397         int nth_active_pipe;
2398
2399         if (!params->active) {
2400                 alloc->start = 0;
2401                 alloc->end = 0;
2402                 return;
2403         }
2404
2405         ddb_size = SKL_DDB_SIZE;
2406
2407         ddb_size -= 4; /* 4 blocks for bypass path allocation */
2408
2409         nth_active_pipe = 0;
2410         for_each_crtc(dev, crtc) {
2411                 if (!intel_crtc_active(crtc))
2412                         continue;
2413
2414                 if (crtc == for_crtc)
2415                         break;
2416
2417                 nth_active_pipe++;
2418         }
2419
2420         pipe_size = ddb_size / config->num_pipes_active;
2421         alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
2422         alloc->end = alloc->start + pipe_size;
2423 }
2424
2425 static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
2426 {
2427         if (config->num_pipes_active == 1)
2428                 return 32;
2429
2430         return 8;
2431 }
2432
2433 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
2434 {
2435         entry->start = reg & 0x3ff;
2436         entry->end = (reg >> 16) & 0x3ff;
2437         if (entry->end)
2438                 entry->end += 1;
2439 }
2440
2441 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
2442                           struct skl_ddb_allocation *ddb /* out */)
2443 {
2444         struct drm_device *dev = dev_priv->dev;
2445         enum pipe pipe;
2446         int plane;
2447         u32 val;
2448
2449         for_each_pipe(dev_priv, pipe) {
2450                 for_each_plane(pipe, plane) {
2451                         val = I915_READ(PLANE_BUF_CFG(pipe, plane));
2452                         skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
2453                                                    val);
2454                 }
2455
2456                 val = I915_READ(CUR_BUF_CFG(pipe));
2457                 skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
2458         }
2459 }
2460
2461 static unsigned int
2462 skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p)
2463 {
2464         return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
2465 }
2466
2467 /*
2468  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
2469  * a 8192x4096@32bpp framebuffer:
2470  *   3 * 4096 * 8192  * 4 < 2^32
2471  */
2472 static unsigned int
2473 skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
2474                                  const struct skl_pipe_wm_parameters *params)
2475 {
2476         unsigned int total_data_rate = 0;
2477         int plane;
2478
2479         for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2480                 const struct intel_plane_wm_parameters *p;
2481
2482                 p = &params->plane[plane];
2483                 if (!p->enabled)
2484                         continue;
2485
2486                 total_data_rate += skl_plane_relative_data_rate(p);
2487         }
2488
2489         return total_data_rate;
2490 }
2491
2492 static void
2493 skl_allocate_pipe_ddb(struct drm_crtc *crtc,
2494                       const struct intel_wm_config *config,
2495                       const struct skl_pipe_wm_parameters *params,
2496                       struct skl_ddb_allocation *ddb /* out */)
2497 {
2498         struct drm_device *dev = crtc->dev;
2499         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2500         enum pipe pipe = intel_crtc->pipe;
2501         struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
2502         uint16_t alloc_size, start, cursor_blocks;
2503         unsigned int total_data_rate;
2504         int plane;
2505
2506         skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
2507         alloc_size = skl_ddb_entry_size(alloc);
2508         if (alloc_size == 0) {
2509                 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
2510                 memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
2511                 return;
2512         }
2513
2514         cursor_blocks = skl_cursor_allocation(config);
2515         ddb->cursor[pipe].start = alloc->end - cursor_blocks;
2516         ddb->cursor[pipe].end = alloc->end;
2517
2518         alloc_size -= cursor_blocks;
2519         alloc->end -= cursor_blocks;
2520
2521         /*
2522          * Each active plane get a portion of the remaining space, in
2523          * proportion to the amount of data they need to fetch from memory.
2524          *
2525          * FIXME: we may not allocate every single block here.
2526          */
2527         total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
2528
2529         start = alloc->start;
2530         for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2531                 const struct intel_plane_wm_parameters *p;
2532                 unsigned int data_rate;
2533                 uint16_t plane_blocks;
2534
2535                 p = &params->plane[plane];
2536                 if (!p->enabled)
2537                         continue;
2538
2539                 data_rate = skl_plane_relative_data_rate(p);
2540
2541                 /*
2542                  * promote the expression to 64 bits to avoid overflowing, the
2543                  * result is < available as data_rate / total_data_rate < 1
2544                  */
2545                 plane_blocks = div_u64((uint64_t)alloc_size * data_rate,
2546                                        total_data_rate);
2547
2548                 ddb->plane[pipe][plane].start = start;
2549                 ddb->plane[pipe][plane].end = start + plane_blocks;
2550
2551                 start += plane_blocks;
2552         }
2553
2554 }
2555
2556 static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
2557 {
2558         /* TODO: Take into account the scalers once we support them */
2559         return config->base.adjusted_mode.crtc_clock;
2560 }
2561
2562 /*
2563  * The max latency should be 257 (max the punit can code is 255 and we add 2us
2564  * for the read latency) and bytes_per_pixel should always be <= 8, so that
2565  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
2566  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
2567 */
2568 static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
2569                                uint32_t latency)
2570 {
2571         uint32_t wm_intermediate_val, ret;
2572
2573         if (latency == 0)
2574                 return UINT_MAX;
2575
2576         wm_intermediate_val = latency * pixel_rate * bytes_per_pixel;
2577         ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
2578
2579         return ret;
2580 }
2581
2582 static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
2583                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
2584                                uint32_t latency)
2585 {
2586         uint32_t ret, plane_bytes_per_line, wm_intermediate_val;
2587
2588         if (latency == 0)
2589                 return UINT_MAX;
2590
2591         plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
2592         wm_intermediate_val = latency * pixel_rate;
2593         ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
2594                                 plane_bytes_per_line;
2595
2596         return ret;
2597 }
2598
2599 static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
2600                                        const struct intel_crtc *intel_crtc)
2601 {
2602         struct drm_device *dev = intel_crtc->base.dev;
2603         struct drm_i915_private *dev_priv = dev->dev_private;
2604         const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
2605         enum pipe pipe = intel_crtc->pipe;
2606
2607         if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
2608                    sizeof(new_ddb->plane[pipe])))
2609                 return true;
2610
2611         if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
2612                     sizeof(new_ddb->cursor[pipe])))
2613                 return true;
2614
2615         return false;
2616 }
2617
2618 static void skl_compute_wm_global_parameters(struct drm_device *dev,
2619                                              struct intel_wm_config *config)
2620 {
2621         struct drm_crtc *crtc;
2622         struct drm_plane *plane;
2623
2624         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2625                 config->num_pipes_active += intel_crtc_active(crtc);
2626
2627         /* FIXME: I don't think we need those two global parameters on SKL */
2628         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2629                 struct intel_plane *intel_plane = to_intel_plane(plane);
2630
2631                 config->sprites_enabled |= intel_plane->wm.enabled;
2632                 config->sprites_scaled |= intel_plane->wm.scaled;
2633         }
2634 }
2635
2636 static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
2637                                            struct skl_pipe_wm_parameters *p)
2638 {
2639         struct drm_device *dev = crtc->dev;
2640         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2641         enum pipe pipe = intel_crtc->pipe;
2642         struct drm_plane *plane;
2643         int i = 1; /* Index for sprite planes start */
2644
2645         p->active = intel_crtc_active(crtc);
2646         if (p->active) {
2647                 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
2648                 p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
2649
2650                 /*
2651                  * For now, assume primary and cursor planes are always enabled.
2652                  */
2653                 p->plane[0].enabled = true;
2654                 p->plane[0].bytes_per_pixel =
2655                         crtc->primary->fb->bits_per_pixel / 8;
2656                 p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
2657                 p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
2658
2659                 p->cursor.enabled = true;
2660                 p->cursor.bytes_per_pixel = 4;
2661                 p->cursor.horiz_pixels = intel_crtc->cursor_width ?
2662                                          intel_crtc->cursor_width : 64;
2663         }
2664
2665         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2666                 struct intel_plane *intel_plane = to_intel_plane(plane);
2667
2668                 if (intel_plane->pipe == pipe &&
2669                         plane->type == DRM_PLANE_TYPE_OVERLAY)
2670                         p->plane[i++] = intel_plane->wm;
2671         }
2672 }
2673
2674 static bool skl_compute_plane_wm(struct skl_pipe_wm_parameters *p,
2675                                  struct intel_plane_wm_parameters *p_params,
2676                                  uint16_t ddb_allocation,
2677                                  uint32_t mem_value,
2678                                  uint16_t *out_blocks, /* out */
2679                                  uint8_t *out_lines /* out */)
2680 {
2681         uint32_t method1, method2, plane_bytes_per_line, res_blocks, res_lines;
2682         uint32_t result_bytes;
2683
2684         if (mem_value == 0 || !p->active || !p_params->enabled)
2685                 return false;
2686
2687         method1 = skl_wm_method1(p->pixel_rate,
2688                                  p_params->bytes_per_pixel,
2689                                  mem_value);
2690         method2 = skl_wm_method2(p->pixel_rate,
2691                                  p->pipe_htotal,
2692                                  p_params->horiz_pixels,
2693                                  p_params->bytes_per_pixel,
2694                                  mem_value);
2695
2696         plane_bytes_per_line = p_params->horiz_pixels *
2697                                         p_params->bytes_per_pixel;
2698
2699         /* For now xtile and linear */
2700         if (((ddb_allocation * 512) / plane_bytes_per_line) >= 1)
2701                 result_bytes = min(method1, method2);
2702         else
2703                 result_bytes = method1;
2704
2705         res_blocks = DIV_ROUND_UP(result_bytes, 512) + 1;
2706         res_lines = DIV_ROUND_UP(result_bytes, plane_bytes_per_line);
2707
2708         if (res_blocks > ddb_allocation || res_lines > 31)
2709                 return false;
2710
2711         *out_blocks = res_blocks;
2712         *out_lines = res_lines;
2713
2714         return true;
2715 }
2716
2717 static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
2718                                  struct skl_ddb_allocation *ddb,
2719                                  struct skl_pipe_wm_parameters *p,
2720                                  enum pipe pipe,
2721                                  int level,
2722                                  int num_planes,
2723                                  struct skl_wm_level *result)
2724 {
2725         uint16_t latency = dev_priv->wm.skl_latency[level];
2726         uint16_t ddb_blocks;
2727         int i;
2728
2729         for (i = 0; i < num_planes; i++) {
2730                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
2731
2732                 result->plane_en[i] = skl_compute_plane_wm(p, &p->plane[i],
2733                                                 ddb_blocks,
2734                                                 latency,
2735                                                 &result->plane_res_b[i],
2736                                                 &result->plane_res_l[i]);
2737         }
2738
2739         ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
2740         result->cursor_en = skl_compute_plane_wm(p, &p->cursor, ddb_blocks,
2741                                                  latency, &result->cursor_res_b,
2742                                                  &result->cursor_res_l);
2743 }
2744
2745 static uint32_t
2746 skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
2747 {
2748         if (!intel_crtc_active(crtc))
2749                 return 0;
2750
2751         return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
2752
2753 }
2754
2755 static void skl_compute_transition_wm(struct drm_crtc *crtc,
2756                                       struct skl_pipe_wm_parameters *params,
2757                                       struct skl_wm_level *trans_wm /* out */)
2758 {
2759         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2760         int i;
2761
2762         if (!params->active)
2763                 return;
2764
2765         /* Until we know more, just disable transition WMs */
2766         for (i = 0; i < intel_num_planes(intel_crtc); i++)
2767                 trans_wm->plane_en[i] = false;
2768         trans_wm->cursor_en = false;
2769 }
2770
2771 static void skl_compute_pipe_wm(struct drm_crtc *crtc,
2772                                 struct skl_ddb_allocation *ddb,
2773                                 struct skl_pipe_wm_parameters *params,
2774                                 struct skl_pipe_wm *pipe_wm)
2775 {
2776         struct drm_device *dev = crtc->dev;
2777         const struct drm_i915_private *dev_priv = dev->dev_private;
2778         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2779         int level, max_level = ilk_wm_max_level(dev);
2780
2781         for (level = 0; level <= max_level; level++) {
2782                 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
2783                                      level, intel_num_planes(intel_crtc),
2784                                      &pipe_wm->wm[level]);
2785         }
2786         pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
2787
2788         skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
2789 }
2790
2791 static void skl_compute_wm_results(struct drm_device *dev,
2792                                    struct skl_pipe_wm_parameters *p,
2793                                    struct skl_pipe_wm *p_wm,
2794                                    struct skl_wm_values *r,
2795                                    struct intel_crtc *intel_crtc)
2796 {
2797         int level, max_level = ilk_wm_max_level(dev);
2798         enum pipe pipe = intel_crtc->pipe;
2799         uint32_t temp;
2800         int i;
2801
2802         for (level = 0; level <= max_level; level++) {
2803                 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
2804                         temp = 0;
2805
2806                         temp |= p_wm->wm[level].plane_res_l[i] <<
2807                                         PLANE_WM_LINES_SHIFT;
2808                         temp |= p_wm->wm[level].plane_res_b[i];
2809                         if (p_wm->wm[level].plane_en[i])
2810                                 temp |= PLANE_WM_EN;
2811
2812                         r->plane[pipe][i][level] = temp;
2813                 }
2814
2815                 temp = 0;
2816
2817                 temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
2818                 temp |= p_wm->wm[level].cursor_res_b;
2819
2820                 if (p_wm->wm[level].cursor_en)
2821                         temp |= PLANE_WM_EN;
2822
2823                 r->cursor[pipe][level] = temp;
2824
2825         }
2826
2827         /* transition WMs */
2828         for (i = 0; i < intel_num_planes(intel_crtc); i++) {
2829                 temp = 0;
2830                 temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
2831                 temp |= p_wm->trans_wm.plane_res_b[i];
2832                 if (p_wm->trans_wm.plane_en[i])
2833                         temp |= PLANE_WM_EN;
2834
2835                 r->plane_trans[pipe][i] = temp;
2836         }
2837
2838         temp = 0;
2839         temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
2840         temp |= p_wm->trans_wm.cursor_res_b;
2841         if (p_wm->trans_wm.cursor_en)
2842                 temp |= PLANE_WM_EN;
2843
2844         r->cursor_trans[pipe] = temp;
2845
2846         r->wm_linetime[pipe] = p_wm->linetime;
2847 }
2848
2849 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
2850                                 const struct skl_ddb_entry *entry)
2851 {
2852         if (entry->end)
2853                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
2854         else
2855                 I915_WRITE(reg, 0);
2856 }
2857
2858 static void skl_write_wm_values(struct drm_i915_private *dev_priv,
2859                                 const struct skl_wm_values *new)
2860 {
2861         struct drm_device *dev = dev_priv->dev;
2862         struct intel_crtc *crtc;
2863
2864         list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
2865                 int i, level, max_level = ilk_wm_max_level(dev);
2866                 enum pipe pipe = crtc->pipe;
2867
2868                 if (!new->dirty[pipe])
2869                         continue;
2870
2871                 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
2872
2873                 for (level = 0; level <= max_level; level++) {
2874                         for (i = 0; i < intel_num_planes(crtc); i++)
2875                                 I915_WRITE(PLANE_WM(pipe, i, level),
2876                                            new->plane[pipe][i][level]);
2877                         I915_WRITE(CUR_WM(pipe, level),
2878                                    new->cursor[pipe][level]);
2879                 }
2880                 for (i = 0; i < intel_num_planes(crtc); i++)
2881                         I915_WRITE(PLANE_WM_TRANS(pipe, i),
2882                                    new->plane_trans[pipe][i]);
2883                 I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
2884
2885                 for (i = 0; i < intel_num_planes(crtc); i++)
2886                         skl_ddb_entry_write(dev_priv,
2887                                             PLANE_BUF_CFG(pipe, i),
2888                                             &new->ddb.plane[pipe][i]);
2889
2890                 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
2891                                     &new->ddb.cursor[pipe]);
2892         }
2893 }
2894
2895 /*
2896  * When setting up a new DDB allocation arrangement, we need to correctly
2897  * sequence the times at which the new allocations for the pipes are taken into
2898  * account or we'll have pipes fetching from space previously allocated to
2899  * another pipe.
2900  *
2901  * Roughly the sequence looks like:
2902  *  1. re-allocate the pipe(s) with the allocation being reduced and not
2903  *     overlapping with a previous light-up pipe (another way to put it is:
2904  *     pipes with their new allocation strickly included into their old ones).
2905  *  2. re-allocate the other pipes that get their allocation reduced
2906  *  3. allocate the pipes having their allocation increased
2907  *
2908  * Steps 1. and 2. are here to take care of the following case:
2909  * - Initially DDB looks like this:
2910  *     |   B    |   C    |
2911  * - enable pipe A.
2912  * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
2913  *   allocation
2914  *     |  A  |  B  |  C  |
2915  *
2916  * We need to sequence the re-allocation: C, B, A (and not B, C, A).
2917  */
2918
2919 static void
2920 skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
2921 {
2922         struct drm_device *dev = dev_priv->dev;
2923         int plane;
2924
2925         DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
2926
2927         for_each_plane(pipe, plane) {
2928                 I915_WRITE(PLANE_SURF(pipe, plane),
2929                            I915_READ(PLANE_SURF(pipe, plane)));
2930         }
2931         I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
2932 }
2933
2934 static bool
2935 skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
2936                             const struct skl_ddb_allocation *new,
2937                             enum pipe pipe)
2938 {
2939         uint16_t old_size, new_size;
2940
2941         old_size = skl_ddb_entry_size(&old->pipe[pipe]);
2942         new_size = skl_ddb_entry_size(&new->pipe[pipe]);
2943
2944         return old_size != new_size &&
2945                new->pipe[pipe].start >= old->pipe[pipe].start &&
2946                new->pipe[pipe].end <= old->pipe[pipe].end;
2947 }
2948
2949 static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
2950                                 struct skl_wm_values *new_values)
2951 {
2952         struct drm_device *dev = dev_priv->dev;
2953         struct skl_ddb_allocation *cur_ddb, *new_ddb;
2954         bool reallocated[I915_MAX_PIPES] = {false, false, false};
2955         struct intel_crtc *crtc;
2956         enum pipe pipe;
2957
2958         new_ddb = &new_values->ddb;
2959         cur_ddb = &dev_priv->wm.skl_hw.ddb;
2960
2961         /*
2962          * First pass: flush the pipes with the new allocation contained into
2963          * the old space.
2964          *
2965          * We'll wait for the vblank on those pipes to ensure we can safely
2966          * re-allocate the freed space without this pipe fetching from it.
2967          */
2968         for_each_intel_crtc(dev, crtc) {
2969                 if (!crtc->active)
2970                         continue;
2971
2972                 pipe = crtc->pipe;
2973
2974                 if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
2975                         continue;
2976
2977                 skl_wm_flush_pipe(dev_priv, pipe, 1);
2978                 intel_wait_for_vblank(dev, pipe);
2979
2980                 reallocated[pipe] = true;
2981         }
2982
2983
2984         /*
2985          * Second pass: flush the pipes that are having their allocation
2986          * reduced, but overlapping with a previous allocation.
2987          *
2988          * Here as well we need to wait for the vblank to make sure the freed
2989          * space is not used anymore.
2990          */
2991         for_each_intel_crtc(dev, crtc) {
2992                 if (!crtc->active)
2993                         continue;
2994
2995                 pipe = crtc->pipe;
2996
2997                 if (reallocated[pipe])
2998                         continue;
2999
3000                 if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
3001                     skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
3002                         skl_wm_flush_pipe(dev_priv, pipe, 2);
3003                         intel_wait_for_vblank(dev, pipe);
3004                         reallocated[pipe] = true;
3005                 }
3006         }
3007
3008         /*
3009          * Third pass: flush the pipes that got more space allocated.
3010          *
3011          * We don't need to actively wait for the update here, next vblank
3012          * will just get more DDB space with the correct WM values.
3013          */
3014         for_each_intel_crtc(dev, crtc) {
3015                 if (!crtc->active)
3016                         continue;
3017
3018                 pipe = crtc->pipe;
3019
3020                 /*
3021                  * At this point, only the pipes more space than before are
3022                  * left to re-allocate.
3023                  */
3024                 if (reallocated[pipe])
3025                         continue;
3026
3027                 skl_wm_flush_pipe(dev_priv, pipe, 3);
3028         }
3029 }
3030
3031 static bool skl_update_pipe_wm(struct drm_crtc *crtc,
3032                                struct skl_pipe_wm_parameters *params,
3033                                struct intel_wm_config *config,
3034                                struct skl_ddb_allocation *ddb, /* out */
3035                                struct skl_pipe_wm *pipe_wm /* out */)
3036 {
3037         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3038
3039         skl_compute_wm_pipe_parameters(crtc, params);
3040         skl_allocate_pipe_ddb(crtc, config, params, ddb);
3041         skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
3042
3043         if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
3044                 return false;
3045
3046         intel_crtc->wm.skl_active = *pipe_wm;
3047         return true;
3048 }
3049
3050 static void skl_update_other_pipe_wm(struct drm_device *dev,
3051                                      struct drm_crtc *crtc,
3052                                      struct intel_wm_config *config,
3053                                      struct skl_wm_values *r)
3054 {
3055         struct intel_crtc *intel_crtc;
3056         struct intel_crtc *this_crtc = to_intel_crtc(crtc);
3057
3058         /*
3059          * If the WM update hasn't changed the allocation for this_crtc (the
3060          * crtc we are currently computing the new WM values for), other
3061          * enabled crtcs will keep the same allocation and we don't need to
3062          * recompute anything for them.
3063          */
3064         if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
3065                 return;
3066
3067         /*
3068          * Otherwise, because of this_crtc being freshly enabled/disabled, the
3069          * other active pipes need new DDB allocation and WM values.
3070          */
3071         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
3072                                 base.head) {
3073                 struct skl_pipe_wm_parameters params = {};
3074                 struct skl_pipe_wm pipe_wm = {};
3075                 bool wm_changed;
3076
3077                 if (this_crtc->pipe == intel_crtc->pipe)
3078                         continue;
3079
3080                 if (!intel_crtc->active)
3081                         continue;
3082
3083                 wm_changed = skl_update_pipe_wm(&intel_crtc->base,
3084                                                 &params, config,
3085                                                 &r->ddb, &pipe_wm);
3086
3087                 /*
3088                  * If we end up re-computing the other pipe WM values, it's
3089                  * because it was really needed, so we expect the WM values to
3090                  * be different.
3091                  */
3092                 WARN_ON(!wm_changed);
3093
3094                 skl_compute_wm_results(dev, &params, &pipe_wm, r, intel_crtc);
3095                 r->dirty[intel_crtc->pipe] = true;
3096         }
3097 }
3098
3099 static void skl_update_wm(struct drm_crtc *crtc)
3100 {
3101         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3102         struct drm_device *dev = crtc->dev;
3103         struct drm_i915_private *dev_priv = dev->dev_private;
3104         struct skl_pipe_wm_parameters params = {};
3105         struct skl_wm_values *results = &dev_priv->wm.skl_results;
3106         struct skl_pipe_wm pipe_wm = {};
3107         struct intel_wm_config config = {};
3108
3109         memset(results, 0, sizeof(*results));
3110
3111         skl_compute_wm_global_parameters(dev, &config);
3112
3113         if (!skl_update_pipe_wm(crtc, &params, &config,
3114                                 &results->ddb, &pipe_wm))
3115                 return;
3116
3117         skl_compute_wm_results(dev, &params, &pipe_wm, results, intel_crtc);
3118         results->dirty[intel_crtc->pipe] = true;
3119
3120         skl_update_other_pipe_wm(dev, crtc, &config, results);
3121         skl_write_wm_values(dev_priv, results);
3122         skl_flush_wm_values(dev_priv, results);
3123
3124         /* store the new configuration */
3125         dev_priv->wm.skl_hw = *results;
3126 }
3127
3128 static void
3129 skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
3130                      uint32_t sprite_width, uint32_t sprite_height,
3131                      int pixel_size, bool enabled, bool scaled)
3132 {
3133         struct intel_plane *intel_plane = to_intel_plane(plane);
3134
3135         intel_plane->wm.enabled = enabled;
3136         intel_plane->wm.scaled = scaled;
3137         intel_plane->wm.horiz_pixels = sprite_width;
3138         intel_plane->wm.vert_pixels = sprite_height;
3139         intel_plane->wm.bytes_per_pixel = pixel_size;
3140
3141         skl_update_wm(crtc);
3142 }
3143
3144 static void ilk_update_wm(struct drm_crtc *crtc)
3145 {
3146         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3147         struct drm_device *dev = crtc->dev;
3148         struct drm_i915_private *dev_priv = dev->dev_private;
3149         struct ilk_wm_maximums max;
3150         struct ilk_pipe_wm_parameters params = {};
3151         struct ilk_wm_values results = {};
3152         enum intel_ddb_partitioning partitioning;
3153         struct intel_pipe_wm pipe_wm = {};
3154         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3155         struct intel_wm_config config = {};
3156
3157         ilk_compute_wm_parameters(crtc, &params);
3158
3159         intel_compute_pipe_wm(crtc, &params, &pipe_wm);
3160
3161         if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
3162                 return;
3163
3164         intel_crtc->wm.active = pipe_wm;
3165
3166         ilk_compute_wm_config(dev, &config);
3167
3168         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
3169         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
3170
3171         /* 5/6 split only in single pipe config on IVB+ */
3172         if (INTEL_INFO(dev)->gen >= 7 &&
3173             config.num_pipes_active == 1 && config.sprites_enabled) {
3174                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
3175                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
3176
3177                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
3178         } else {
3179                 best_lp_wm = &lp_wm_1_2;
3180         }
3181
3182         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3183                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3184
3185         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
3186
3187         ilk_write_wm_values(dev_priv, &results);
3188 }
3189
3190 static void
3191 ilk_update_sprite_wm(struct drm_plane *plane,
3192                      struct drm_crtc *crtc,
3193                      uint32_t sprite_width, uint32_t sprite_height,
3194                      int pixel_size, bool enabled, bool scaled)
3195 {
3196         struct drm_device *dev = plane->dev;
3197         struct intel_plane *intel_plane = to_intel_plane(plane);
3198
3199         intel_plane->wm.enabled = enabled;
3200         intel_plane->wm.scaled = scaled;
3201         intel_plane->wm.horiz_pixels = sprite_width;
3202         intel_plane->wm.vert_pixels = sprite_width;
3203         intel_plane->wm.bytes_per_pixel = pixel_size;
3204
3205         /*
3206          * IVB workaround: must disable low power watermarks for at least
3207          * one frame before enabling scaling.  LP watermarks can be re-enabled
3208          * when scaling is disabled.
3209          *
3210          * WaCxSRDisabledForSpriteScaling:ivb
3211          */
3212         if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
3213                 intel_wait_for_vblank(dev, intel_plane->pipe);
3214
3215         ilk_update_wm(crtc);
3216 }
3217
3218 static void skl_pipe_wm_active_state(uint32_t val,
3219                                      struct skl_pipe_wm *active,
3220                                      bool is_transwm,
3221                                      bool is_cursor,
3222                                      int i,
3223                                      int level)
3224 {
3225         bool is_enabled = (val & PLANE_WM_EN) != 0;
3226
3227         if (!is_transwm) {
3228                 if (!is_cursor) {
3229                         active->wm[level].plane_en[i] = is_enabled;
3230                         active->wm[level].plane_res_b[i] =
3231                                         val & PLANE_WM_BLOCKS_MASK;
3232                         active->wm[level].plane_res_l[i] =
3233                                         (val >> PLANE_WM_LINES_SHIFT) &
3234                                                 PLANE_WM_LINES_MASK;
3235                 } else {
3236                         active->wm[level].cursor_en = is_enabled;
3237                         active->wm[level].cursor_res_b =
3238                                         val & PLANE_WM_BLOCKS_MASK;
3239                         active->wm[level].cursor_res_l =
3240                                         (val >> PLANE_WM_LINES_SHIFT) &
3241                                                 PLANE_WM_LINES_MASK;
3242                 }
3243         } else {
3244                 if (!is_cursor) {
3245                         active->trans_wm.plane_en[i] = is_enabled;
3246                         active->trans_wm.plane_res_b[i] =
3247                                         val & PLANE_WM_BLOCKS_MASK;
3248                         active->trans_wm.plane_res_l[i] =
3249                                         (val >> PLANE_WM_LINES_SHIFT) &
3250                                                 PLANE_WM_LINES_MASK;
3251                 } else {
3252                         active->trans_wm.cursor_en = is_enabled;
3253                         active->trans_wm.cursor_res_b =
3254                                         val & PLANE_WM_BLOCKS_MASK;
3255                         active->trans_wm.cursor_res_l =
3256                                         (val >> PLANE_WM_LINES_SHIFT) &
3257                                                 PLANE_WM_LINES_MASK;
3258                 }
3259         }
3260 }
3261
3262 static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3263 {
3264         struct drm_device *dev = crtc->dev;
3265         struct drm_i915_private *dev_priv = dev->dev_private;
3266         struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
3267         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3268         struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
3269         enum pipe pipe = intel_crtc->pipe;
3270         int level, i, max_level;
3271         uint32_t temp;
3272
3273         max_level = ilk_wm_max_level(dev);
3274
3275         hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3276
3277         for (level = 0; level <= max_level; level++) {
3278                 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3279                         hw->plane[pipe][i][level] =
3280                                         I915_READ(PLANE_WM(pipe, i, level));
3281                 hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
3282         }
3283
3284         for (i = 0; i < intel_num_planes(intel_crtc); i++)
3285                 hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
3286         hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
3287
3288         if (!intel_crtc_active(crtc))
3289                 return;
3290
3291         hw->dirty[pipe] = true;
3292
3293         active->linetime = hw->wm_linetime[pipe];
3294
3295         for (level = 0; level <= max_level; level++) {
3296                 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3297                         temp = hw->plane[pipe][i][level];
3298                         skl_pipe_wm_active_state(temp, active, false,
3299                                                 false, i, level);
3300                 }
3301                 temp = hw->cursor[pipe][level];
3302                 skl_pipe_wm_active_state(temp, active, false, true, i, level);
3303         }
3304
3305         for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3306                 temp = hw->plane_trans[pipe][i];
3307                 skl_pipe_wm_active_state(temp, active, true, false, i, 0);
3308         }
3309
3310         temp = hw->cursor_trans[pipe];
3311         skl_pipe_wm_active_state(temp, active, true, true, i, 0);
3312 }
3313
3314 void skl_wm_get_hw_state(struct drm_device *dev)
3315 {
3316         struct drm_i915_private *dev_priv = dev->dev_private;
3317         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
3318         struct drm_crtc *crtc;
3319
3320         skl_ddb_get_hw_state(dev_priv, ddb);
3321         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3322                 skl_pipe_wm_get_hw_state(crtc);
3323 }
3324
3325 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3326 {
3327         struct drm_device *dev = crtc->dev;
3328         struct drm_i915_private *dev_priv = dev->dev_private;
3329         struct ilk_wm_values *hw = &dev_priv->wm.hw;
3330         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3331         struct intel_pipe_wm *active = &intel_crtc->wm.active;
3332         enum pipe pipe = intel_crtc->pipe;
3333         static const unsigned int wm0_pipe_reg[] = {
3334                 [PIPE_A] = WM0_PIPEA_ILK,
3335                 [PIPE_B] = WM0_PIPEB_ILK,
3336                 [PIPE_C] = WM0_PIPEC_IVB,
3337         };
3338
3339         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
3340         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3341                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3342
3343         active->pipe_enabled = intel_crtc_active(crtc);
3344
3345         if (active->pipe_enabled) {
3346                 u32 tmp = hw->wm_pipe[pipe];
3347
3348                 /*
3349                  * For active pipes LP0 watermark is marked as
3350                  * enabled, and LP1+ watermaks as disabled since
3351                  * we can't really reverse compute them in case
3352                  * multiple pipes are active.
3353                  */
3354                 active->wm[0].enable = true;
3355                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
3356                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
3357                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
3358                 active->linetime = hw->wm_linetime[pipe];
3359         } else {
3360                 int level, max_level = ilk_wm_max_level(dev);
3361
3362                 /*
3363                  * For inactive pipes, all watermark levels
3364                  * should be marked as enabled but zeroed,
3365                  * which is what we'd compute them to.
3366                  */
3367                 for (level = 0; level <= max_level; level++)
3368                         active->wm[level].enable = true;
3369         }
3370 }
3371
3372 void ilk_wm_get_hw_state(struct drm_device *dev)
3373 {
3374         struct drm_i915_private *dev_priv = dev->dev_private;
3375         struct ilk_wm_values *hw = &dev_priv->wm.hw;
3376         struct drm_crtc *crtc;
3377
3378         for_each_crtc(dev, crtc)
3379                 ilk_pipe_wm_get_hw_state(crtc);
3380
3381         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
3382         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
3383         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3384
3385         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
3386         if (INTEL_INFO(dev)->gen >= 7) {
3387                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
3388                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
3389         }
3390
3391         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3392                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
3393                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3394         else if (IS_IVYBRIDGE(dev))
3395                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
3396                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3397
3398         hw->enable_fbc_wm =
3399                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3400 }
3401
3402 /**
3403  * intel_update_watermarks - update FIFO watermark values based on current modes
3404  *
3405  * Calculate watermark values for the various WM regs based on current mode
3406  * and plane configuration.
3407  *
3408  * There are several cases to deal with here:
3409  *   - normal (i.e. non-self-refresh)
3410  *   - self-refresh (SR) mode
3411  *   - lines are large relative to FIFO size (buffer can hold up to 2)
3412  *   - lines are small relative to FIFO size (buffer can hold more than 2
3413  *     lines), so need to account for TLB latency
3414  *
3415  *   The normal calculation is:
3416  *     watermark = dotclock * bytes per pixel * latency
3417  *   where latency is platform & configuration dependent (we assume pessimal
3418  *   values here).
3419  *
3420  *   The SR calculation is:
3421  *     watermark = (trunc(latency/line time)+1) * surface width *
3422  *       bytes per pixel
3423  *   where
3424  *     line time = htotal / dotclock
3425  *     surface width = hdisplay for normal plane and 64 for cursor
3426  *   and latency is assumed to be high, as above.
3427  *
3428  * The final value programmed to the register should always be rounded up,
3429  * and include an extra 2 entries to account for clock crossings.
3430  *
3431  * We don't use the sprite, so we can ignore that.  And on Crestline we have
3432  * to set the non-SR watermarks to 8.
3433  */
3434 void intel_update_watermarks(struct drm_crtc *crtc)
3435 {
3436         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3437
3438         if (dev_priv->display.update_wm)
3439                 dev_priv->display.update_wm(crtc);
3440 }
3441
3442 void intel_update_sprite_watermarks(struct drm_plane *plane,
3443                                     struct drm_crtc *crtc,
3444                                     uint32_t sprite_width,
3445                                     uint32_t sprite_height,
3446                                     int pixel_size,
3447                                     bool enabled, bool scaled)
3448 {
3449         struct drm_i915_private *dev_priv = plane->dev->dev_private;
3450
3451         if (dev_priv->display.update_sprite_wm)
3452                 dev_priv->display.update_sprite_wm(plane, crtc,
3453                                                    sprite_width, sprite_height,
3454                                                    pixel_size, enabled, scaled);
3455 }
3456
3457 static struct drm_i915_gem_object *
3458 intel_alloc_context_page(struct drm_device *dev)
3459 {
3460         struct drm_i915_gem_object *ctx;
3461         int ret;
3462
3463         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3464
3465         ctx = i915_gem_alloc_object(dev, 4096);
3466         if (!ctx) {
3467                 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
3468                 return NULL;
3469         }
3470
3471         ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0);
3472         if (ret) {
3473                 DRM_ERROR("failed to pin power context: %d\n", ret);
3474                 goto err_unref;
3475         }
3476
3477         ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
3478         if (ret) {
3479                 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
3480                 goto err_unpin;
3481         }
3482
3483         return ctx;
3484
3485 err_unpin:
3486         i915_gem_object_ggtt_unpin(ctx);
3487 err_unref:
3488         drm_gem_object_unreference(&ctx->base);
3489         return NULL;
3490 }
3491
3492 /**
3493  * Lock protecting IPS related data structures
3494  */
3495 DEFINE_SPINLOCK(mchdev_lock);
3496
3497 /* Global for IPS driver to get at the current i915 device. Protected by
3498  * mchdev_lock. */
3499 static struct drm_i915_private *i915_mch_dev;
3500
3501 bool ironlake_set_drps(struct drm_device *dev, u8 val)
3502 {
3503         struct drm_i915_private *dev_priv = dev->dev_private;
3504         u16 rgvswctl;
3505
3506         assert_spin_locked(&mchdev_lock);
3507
3508         rgvswctl = I915_READ16(MEMSWCTL);
3509         if (rgvswctl & MEMCTL_CMD_STS) {
3510                 DRM_DEBUG("gpu busy, RCS change rejected\n");
3511                 return false; /* still busy with another command */
3512         }
3513
3514         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
3515                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
3516         I915_WRITE16(MEMSWCTL, rgvswctl);
3517         POSTING_READ16(MEMSWCTL);
3518
3519         rgvswctl |= MEMCTL_CMD_STS;
3520         I915_WRITE16(MEMSWCTL, rgvswctl);
3521
3522         return true;
3523 }
3524
3525 static void ironlake_enable_drps(struct drm_device *dev)
3526 {
3527         struct drm_i915_private *dev_priv = dev->dev_private;
3528         u32 rgvmodectl = I915_READ(MEMMODECTL);
3529         u8 fmax, fmin, fstart, vstart;
3530
3531         spin_lock_irq(&mchdev_lock);
3532
3533         /* Enable temp reporting */
3534         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
3535         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
3536
3537         /* 100ms RC evaluation intervals */
3538         I915_WRITE(RCUPEI, 100000);
3539         I915_WRITE(RCDNEI, 100000);
3540
3541         /* Set max/min thresholds to 90ms and 80ms respectively */
3542         I915_WRITE(RCBMAXAVG, 90000);
3543         I915_WRITE(RCBMINAVG, 80000);
3544
3545         I915_WRITE(MEMIHYST, 1);
3546
3547         /* Set up min, max, and cur for interrupt handling */
3548         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
3549         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
3550         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
3551                 MEMMODE_FSTART_SHIFT;
3552
3553         vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
3554                 PXVFREQ_PX_SHIFT;
3555
3556         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
3557         dev_priv->ips.fstart = fstart;
3558
3559         dev_priv->ips.max_delay = fstart;
3560         dev_priv->ips.min_delay = fmin;
3561         dev_priv->ips.cur_delay = fstart;
3562
3563         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
3564                          fmax, fmin, fstart);
3565
3566         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
3567
3568         /*
3569          * Interrupts will be enabled in ironlake_irq_postinstall
3570          */
3571
3572         I915_WRITE(VIDSTART, vstart);
3573         POSTING_READ(VIDSTART);
3574
3575         rgvmodectl |= MEMMODE_SWMODE_EN;
3576         I915_WRITE(MEMMODECTL, rgvmodectl);
3577
3578         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
3579                 DRM_ERROR("stuck trying to change perf mode\n");
3580         mdelay(1);
3581
3582         ironlake_set_drps(dev, fstart);
3583
3584         dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
3585                 I915_READ(0x112e0);
3586         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3587         dev_priv->ips.last_count2 = I915_READ(0x112f4);
3588         dev_priv->ips.last_time2 = ktime_get_raw_ns();
3589
3590         spin_unlock_irq(&mchdev_lock);
3591 }
3592
3593 static void ironlake_disable_drps(struct drm_device *dev)
3594 {
3595         struct drm_i915_private *dev_priv = dev->dev_private;
3596         u16 rgvswctl;
3597
3598         spin_lock_irq(&mchdev_lock);
3599
3600         rgvswctl = I915_READ16(MEMSWCTL);
3601
3602         /* Ack interrupts, disable EFC interrupt */
3603         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
3604         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
3605         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
3606         I915_WRITE(DEIIR, DE_PCU_EVENT);
3607         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
3608
3609         /* Go back to the starting frequency */
3610         ironlake_set_drps(dev, dev_priv->ips.fstart);
3611         mdelay(1);
3612         rgvswctl |= MEMCTL_CMD_STS;
3613         I915_WRITE(MEMSWCTL, rgvswctl);
3614         mdelay(1);
3615
3616         spin_unlock_irq(&mchdev_lock);
3617 }
3618
3619 /* There's a funny hw issue where the hw returns all 0 when reading from
3620  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
3621  * ourselves, instead of doing a rmw cycle (which might result in us clearing
3622  * all limits and the gpu stuck at whatever frequency it is at atm).
3623  */
3624 static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3625 {
3626         u32 limits;
3627
3628         /* Only set the down limit when we've reached the lowest level to avoid
3629          * getting more interrupts, otherwise leave this clear. This prevents a
3630          * race in the hw when coming out of rc6: There's a tiny window where
3631          * the hw runs at the minimal clock before selecting the desired
3632          * frequency, if the down threshold expires in that window we will not
3633          * receive a down interrupt. */
3634         limits = dev_priv->rps.max_freq_softlimit << 24;
3635         if (val <= dev_priv->rps.min_freq_softlimit)
3636                 limits |= dev_priv->rps.min_freq_softlimit << 16;
3637
3638         return limits;
3639 }
3640
3641 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
3642 {
3643         int new_power;
3644
3645         new_power = dev_priv->rps.power;
3646         switch (dev_priv->rps.power) {
3647         case LOW_POWER:
3648                 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
3649                         new_power = BETWEEN;
3650                 break;
3651
3652         case BETWEEN:
3653                 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
3654                         new_power = LOW_POWER;
3655                 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
3656                         new_power = HIGH_POWER;
3657                 break;
3658
3659         case HIGH_POWER:
3660                 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
3661                         new_power = BETWEEN;
3662                 break;
3663         }
3664         /* Max/min bins are special */
3665         if (val == dev_priv->rps.min_freq_softlimit)
3666                 new_power = LOW_POWER;
3667         if (val == dev_priv->rps.max_freq_softlimit)
3668                 new_power = HIGH_POWER;
3669         if (new_power == dev_priv->rps.power)
3670                 return;
3671
3672         /* Note the units here are not exactly 1us, but 1280ns. */
3673         switch (new_power) {
3674         case LOW_POWER:
3675                 /* Upclock if more than 95% busy over 16ms */
3676                 I915_WRITE(GEN6_RP_UP_EI, 12500);
3677                 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
3678
3679                 /* Downclock if less than 85% busy over 32ms */
3680                 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3681                 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
3682
3683                 I915_WRITE(GEN6_RP_CONTROL,
3684                            GEN6_RP_MEDIA_TURBO |
3685                            GEN6_RP_MEDIA_HW_NORMAL_MODE |
3686                            GEN6_RP_MEDIA_IS_GFX |
3687                            GEN6_RP_ENABLE |
3688                            GEN6_RP_UP_BUSY_AVG |
3689                            GEN6_RP_DOWN_IDLE_AVG);
3690                 break;
3691
3692         case BETWEEN:
3693                 /* Upclock if more than 90% busy over 13ms */
3694                 I915_WRITE(GEN6_RP_UP_EI, 10250);
3695                 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
3696
3697                 /* Downclock if less than 75% busy over 32ms */
3698                 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3699                 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
3700
3701                 I915_WRITE(GEN6_RP_CONTROL,
3702                            GEN6_RP_MEDIA_TURBO |
3703                            GEN6_RP_MEDIA_HW_NORMAL_MODE |
3704                            GEN6_RP_MEDIA_IS_GFX |
3705                            GEN6_RP_ENABLE |
3706                            GEN6_RP_UP_BUSY_AVG |
3707                            GEN6_RP_DOWN_IDLE_AVG);
3708                 break;
3709
3710         case HIGH_POWER:
3711                 /* Upclock if more than 85% busy over 10ms */
3712                 I915_WRITE(GEN6_RP_UP_EI, 8000);
3713                 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
3714
3715                 /* Downclock if less than 60% busy over 32ms */
3716                 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3717                 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
3718
3719                 I915_WRITE(GEN6_RP_CONTROL,
3720                            GEN6_RP_MEDIA_TURBO |
3721                            GEN6_RP_MEDIA_HW_NORMAL_MODE |
3722                            GEN6_RP_MEDIA_IS_GFX |
3723                            GEN6_RP_ENABLE |
3724                            GEN6_RP_UP_BUSY_AVG |
3725                            GEN6_RP_DOWN_IDLE_AVG);
3726                 break;
3727         }
3728
3729         dev_priv->rps.power = new_power;
3730         dev_priv->rps.last_adj = 0;
3731 }
3732
3733 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
3734 {
3735         u32 mask = 0;
3736
3737         if (val > dev_priv->rps.min_freq_softlimit)
3738                 mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
3739         if (val < dev_priv->rps.max_freq_softlimit)
3740                 mask |= GEN6_PM_RP_UP_THRESHOLD;
3741
3742         mask |= dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED);
3743         mask &= dev_priv->pm_rps_events;
3744
3745         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
3746 }
3747
3748 /* gen6_set_rps is called to update the frequency request, but should also be
3749  * called when the range (min_delay and max_delay) is modified so that we can
3750  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
3751 static void gen6_set_rps(struct drm_device *dev, u8 val)
3752 {
3753         struct drm_i915_private *dev_priv = dev->dev_private;
3754
3755         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3756         WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3757         WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3758
3759         /* min/max delay may still have been modified so be sure to
3760          * write the limits value.
3761          */
3762         if (val != dev_priv->rps.cur_freq) {
3763                 gen6_set_rps_thresholds(dev_priv, val);
3764
3765                 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3766                         I915_WRITE(GEN6_RPNSWREQ,
3767                                    HSW_FREQUENCY(val));
3768                 else
3769                         I915_WRITE(GEN6_RPNSWREQ,
3770                                    GEN6_FREQUENCY(val) |
3771                                    GEN6_OFFSET(0) |
3772                                    GEN6_AGGRESSIVE_TURBO);
3773         }
3774
3775         /* Make sure we continue to get interrupts
3776          * until we hit the minimum or maximum frequencies.
3777          */
3778         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
3779         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3780
3781         POSTING_READ(GEN6_RPNSWREQ);
3782
3783         dev_priv->rps.cur_freq = val;
3784         trace_intel_gpu_freq_change(val * 50);
3785 }
3786
3787 static void valleyview_set_rps(struct drm_device *dev, u8 val)
3788 {
3789         struct drm_i915_private *dev_priv = dev->dev_private;
3790
3791         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3792         WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3793         WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3794
3795         if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
3796                       "Odd GPU freq value\n"))
3797                 val &= ~1;
3798
3799         if (val != dev_priv->rps.cur_freq)
3800                 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
3801
3802         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3803
3804         dev_priv->rps.cur_freq = val;
3805         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3806 }
3807
3808 /* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
3809  *
3810  * * If Gfx is Idle, then
3811  * 1. Mask Turbo interrupts
3812  * 2. Bring up Gfx clock
3813  * 3. Change the freq to Rpn and wait till P-Unit updates freq
3814  * 4. Clear the Force GFX CLK ON bit so that Gfx can down
3815  * 5. Unmask Turbo interrupts
3816 */
3817 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
3818 {
3819         struct drm_device *dev = dev_priv->dev;
3820
3821         /* CHV and latest VLV don't need to force the gfx clock */
3822         if (IS_CHERRYVIEW(dev) || dev->pdev->revision >= 0xd) {
3823                 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3824                 return;
3825         }
3826
3827         /*
3828          * When we are idle.  Drop to min voltage state.
3829          */
3830
3831         if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
3832                 return;
3833
3834         /* Mask turbo interrupt so that they will not come in between */
3835         I915_WRITE(GEN6_PMINTRMSK,
3836                    gen6_sanitize_rps_pm_mask(dev_priv, ~0));
3837
3838         vlv_force_gfx_clock(dev_priv, true);
3839
3840         dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
3841
3842         vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
3843                                         dev_priv->rps.min_freq_softlimit);
3844
3845         if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
3846                                 & GENFREQSTATUS) == 0, 100))
3847                 DRM_ERROR("timed out waiting for Punit\n");
3848
3849         vlv_force_gfx_clock(dev_priv, false);
3850
3851         I915_WRITE(GEN6_PMINTRMSK,
3852                    gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
3853 }
3854
3855 void gen6_rps_idle(struct drm_i915_private *dev_priv)
3856 {
3857         struct drm_device *dev = dev_priv->dev;
3858
3859         mutex_lock(&dev_priv->rps.hw_lock);
3860         if (dev_priv->rps.enabled) {
3861                 if (IS_VALLEYVIEW(dev))
3862                         vlv_set_rps_idle(dev_priv);
3863                 else
3864                         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3865                 dev_priv->rps.last_adj = 0;
3866         }
3867         mutex_unlock(&dev_priv->rps.hw_lock);
3868 }
3869
3870 void gen6_rps_boost(struct drm_i915_private *dev_priv)
3871 {
3872         mutex_lock(&dev_priv->rps.hw_lock);
3873         if (dev_priv->rps.enabled) {
3874                 intel_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
3875                 dev_priv->rps.last_adj = 0;
3876         }
3877         mutex_unlock(&dev_priv->rps.hw_lock);
3878 }
3879
3880 void intel_set_rps(struct drm_device *dev, u8 val)
3881 {
3882         if (IS_VALLEYVIEW(dev))
3883                 valleyview_set_rps(dev, val);
3884         else
3885                 gen6_set_rps(dev, val);
3886 }
3887
3888 static void gen9_disable_rps(struct drm_device *dev)
3889 {
3890         struct drm_i915_private *dev_priv = dev->dev_private;
3891
3892         I915_WRITE(GEN6_RC_CONTROL, 0);
3893         I915_WRITE(GEN9_PG_ENABLE, 0);
3894 }
3895
3896 static void gen6_disable_rps(struct drm_device *dev)
3897 {
3898         struct drm_i915_private *dev_priv = dev->dev_private;
3899
3900         I915_WRITE(GEN6_RC_CONTROL, 0);
3901         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
3902 }
3903
3904 static void cherryview_disable_rps(struct drm_device *dev)
3905 {
3906         struct drm_i915_private *dev_priv = dev->dev_private;
3907
3908         I915_WRITE(GEN6_RC_CONTROL, 0);
3909 }
3910
3911 static void valleyview_disable_rps(struct drm_device *dev)
3912 {
3913         struct drm_i915_private *dev_priv = dev->dev_private;
3914
3915         /* we're doing forcewake before Disabling RC6,
3916          * This what the BIOS expects when going into suspend */
3917         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
3918
3919         I915_WRITE(GEN6_RC_CONTROL, 0);
3920
3921         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
3922 }
3923
3924 static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
3925 {
3926         if (IS_VALLEYVIEW(dev)) {
3927                 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
3928                         mode = GEN6_RC_CTL_RC6_ENABLE;
3929                 else
3930                         mode = 0;
3931         }
3932         if (HAS_RC6p(dev))
3933                 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
3934                               (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
3935                               (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
3936                               (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
3937
3938         else
3939                 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
3940                               (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
3941 }
3942
3943 static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3944 {
3945         /* No RC6 before Ironlake */
3946         if (INTEL_INFO(dev)->gen < 5)
3947                 return 0;
3948
3949         /* RC6 is only on Ironlake mobile not on desktop */
3950         if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev))
3951                 return 0;
3952
3953         /* Respect the kernel parameter if it is set */
3954         if (enable_rc6 >= 0) {
3955                 int mask;
3956
3957                 if (HAS_RC6p(dev))
3958                         mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
3959                                INTEL_RC6pp_ENABLE;
3960                 else
3961                         mask = INTEL_RC6_ENABLE;
3962
3963                 if ((enable_rc6 & mask) != enable_rc6)
3964                         DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
3965                                       enable_rc6 & mask, enable_rc6, mask);
3966
3967                 return enable_rc6 & mask;
3968         }
3969
3970         /* Disable RC6 on Ironlake */
3971         if (INTEL_INFO(dev)->gen == 5)
3972                 return 0;
3973
3974         if (IS_IVYBRIDGE(dev))
3975                 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
3976
3977         return INTEL_RC6_ENABLE;
3978 }
3979
3980 int intel_enable_rc6(const struct drm_device *dev)
3981 {
3982         return i915.enable_rc6;
3983 }
3984
3985 static void gen6_init_rps_frequencies(struct drm_device *dev)
3986 {
3987         struct drm_i915_private *dev_priv = dev->dev_private;
3988         uint32_t rp_state_cap;
3989         u32 ddcc_status = 0;
3990         int ret;
3991
3992         rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3993         /* All of these values are in units of 50MHz */
3994         dev_priv->rps.cur_freq          = 0;
3995         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
3996         dev_priv->rps.rp0_freq          = (rp_state_cap >>  0) & 0xff;
3997         dev_priv->rps.rp1_freq          = (rp_state_cap >>  8) & 0xff;
3998         dev_priv->rps.min_freq          = (rp_state_cap >> 16) & 0xff;
3999         /* hw_max = RP0 until we check for overclocking */
4000         dev_priv->rps.max_freq          = dev_priv->rps.rp0_freq;
4001
4002         dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
4003         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
4004                 ret = sandybridge_pcode_read(dev_priv,
4005                                         HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
4006                                         &ddcc_status);
4007                 if (0 == ret)
4008                         dev_priv->rps.efficient_freq =
4009                                 (ddcc_status >> 8) & 0xff;
4010         }
4011
4012         /* Preserve min/max settings in case of re-init */
4013         if (dev_priv->rps.max_freq_softlimit == 0)
4014                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4015
4016         if (dev_priv->rps.min_freq_softlimit == 0) {
4017                 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4018                         dev_priv->rps.min_freq_softlimit =
4019                                 /* max(RPe, 450 MHz) */
4020                                 max(dev_priv->rps.efficient_freq, (u8) 9);
4021                 else
4022                         dev_priv->rps.min_freq_softlimit =
4023                                 dev_priv->rps.min_freq;
4024         }
4025 }
4026
4027 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
4028 static void gen9_enable_rps(struct drm_device *dev)
4029 {
4030         struct drm_i915_private *dev_priv = dev->dev_private;
4031
4032         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4033
4034         gen6_init_rps_frequencies(dev);
4035
4036         I915_WRITE(GEN6_RPNSWREQ, 0xc800000);
4037         I915_WRITE(GEN6_RC_VIDEO_FREQ, 0xc800000);
4038
4039         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 0xf4240);
4040         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, 0x12060000);
4041         I915_WRITE(GEN6_RP_UP_THRESHOLD, 0xe808);
4042         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 0x3bd08);
4043         I915_WRITE(GEN6_RP_UP_EI, 0x101d0);
4044         I915_WRITE(GEN6_RP_DOWN_EI, 0x55730);
4045         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
4046         I915_WRITE(GEN6_PMINTRMSK, 0x6);
4047         I915_WRITE(GEN6_RP_CONTROL, GEN6_RP_MEDIA_TURBO |
4048                    GEN6_RP_MEDIA_HW_MODE | GEN6_RP_MEDIA_IS_GFX |
4049                    GEN6_RP_ENABLE | GEN6_RP_UP_BUSY_AVG |
4050                    GEN6_RP_DOWN_IDLE_AVG);
4051
4052         gen6_enable_rps_interrupts(dev);
4053
4054         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4055 }
4056
4057 static void gen9_enable_rc6(struct drm_device *dev)
4058 {
4059         struct drm_i915_private *dev_priv = dev->dev_private;
4060         struct intel_engine_cs *ring;
4061         uint32_t rc6_mask = 0;
4062         int unused;
4063
4064         /* 1a: Software RC state - RC0 */
4065         I915_WRITE(GEN6_RC_STATE, 0);
4066
4067         /* 1b: Get forcewake during program sequence. Although the driver
4068          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4069         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4070
4071         /* 2a: Disable RC states. */
4072         I915_WRITE(GEN6_RC_CONTROL, 0);
4073
4074         /* 2b: Program RC6 thresholds.*/
4075         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
4076         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4077         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4078         for_each_ring(ring, dev_priv, unused)
4079                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4080         I915_WRITE(GEN6_RC_SLEEP, 0);
4081         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
4082
4083         /* 2c: Program Coarse Power Gating Policies. */
4084         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
4085         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
4086
4087         /* 3a: Enable RC6 */
4088         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4089                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4090         DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4091                         "on" : "off");
4092         I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4093                                    GEN6_RC_CTL_EI_MODE(1) |
4094                                    rc6_mask);
4095
4096         /* 3b: Enable Coarse Power Gating only when RC6 is enabled */
4097         I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? 3 : 0);
4098
4099         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4100
4101 }
4102
4103 static void gen8_enable_rps(struct drm_device *dev)
4104 {
4105         struct drm_i915_private *dev_priv = dev->dev_private;
4106         struct intel_engine_cs *ring;
4107         uint32_t rc6_mask = 0;
4108         int unused;
4109
4110         /* 1a: Software RC state - RC0 */
4111         I915_WRITE(GEN6_RC_STATE, 0);
4112
4113         /* 1c & 1d: Get forcewake during program sequence. Although the driver
4114          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4115         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4116
4117         /* 2a: Disable RC states. */
4118         I915_WRITE(GEN6_RC_CONTROL, 0);
4119
4120         /* Initialize rps frequencies */
4121         gen6_init_rps_frequencies(dev);
4122
4123         /* 2b: Program RC6 thresholds.*/
4124         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4125         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4126         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4127         for_each_ring(ring, dev_priv, unused)
4128                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4129         I915_WRITE(GEN6_RC_SLEEP, 0);
4130         if (IS_BROADWELL(dev))
4131                 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
4132         else
4133                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4134
4135         /* 3: Enable RC6 */
4136         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4137                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4138         intel_print_rc6_info(dev, rc6_mask);
4139         if (IS_BROADWELL(dev))
4140                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4141                                 GEN7_RC_CTL_TO_MODE |
4142                                 rc6_mask);
4143         else
4144                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4145                                 GEN6_RC_CTL_EI_MODE(1) |
4146                                 rc6_mask);
4147
4148         /* 4 Program defaults and thresholds for RPS*/
4149         I915_WRITE(GEN6_RPNSWREQ,
4150                    HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4151         I915_WRITE(GEN6_RC_VIDEO_FREQ,
4152                    HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4153         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
4154         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
4155
4156         /* Docs recommend 900MHz, and 300 MHz respectively */
4157         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
4158                    dev_priv->rps.max_freq_softlimit << 24 |
4159                    dev_priv->rps.min_freq_softlimit << 16);
4160
4161         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
4162         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
4163         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
4164         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
4165
4166         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4167
4168         /* 5: Enable RPS */
4169         I915_WRITE(GEN6_RP_CONTROL,
4170                    GEN6_RP_MEDIA_TURBO |
4171                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4172                    GEN6_RP_MEDIA_IS_GFX |
4173                    GEN6_RP_ENABLE |
4174                    GEN6_RP_UP_BUSY_AVG |
4175                    GEN6_RP_DOWN_IDLE_AVG);
4176
4177         /* 6: Ring frequency + overclocking (our driver does this later */
4178
4179         dev_priv->rps.power = HIGH_POWER; /* force a reset */
4180         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4181
4182         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4183 }
4184
4185 static void gen6_enable_rps(struct drm_device *dev)
4186 {
4187         struct drm_i915_private *dev_priv = dev->dev_private;
4188         struct intel_engine_cs *ring;
4189         u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
4190         u32 gtfifodbg;
4191         int rc6_mode;
4192         int i, ret;
4193
4194         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4195
4196         /* Here begins a magic sequence of register writes to enable
4197          * auto-downclocking.
4198          *
4199          * Perhaps there might be some value in exposing these to
4200          * userspace...
4201          */
4202         I915_WRITE(GEN6_RC_STATE, 0);
4203
4204         /* Clear the DBG now so we don't confuse earlier errors */
4205         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4206                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
4207                 I915_WRITE(GTFIFODBG, gtfifodbg);
4208         }
4209
4210         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4211
4212         /* Initialize rps frequencies */
4213         gen6_init_rps_frequencies(dev);
4214
4215         /* disable the counters and set deterministic thresholds */
4216         I915_WRITE(GEN6_RC_CONTROL, 0);
4217
4218         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
4219         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
4220         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
4221         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4222         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4223
4224         for_each_ring(ring, dev_priv, i)
4225                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4226
4227         I915_WRITE(GEN6_RC_SLEEP, 0);
4228         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4229         if (IS_IVYBRIDGE(dev))
4230                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
4231         else
4232                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
4233         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
4234         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
4235
4236         /* Check if we are enabling RC6 */
4237         rc6_mode = intel_enable_rc6(dev_priv->dev);
4238         if (rc6_mode & INTEL_RC6_ENABLE)
4239                 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
4240
4241         /* We don't use those on Haswell */
4242         if (!IS_HASWELL(dev)) {
4243                 if (rc6_mode & INTEL_RC6p_ENABLE)
4244                         rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
4245
4246                 if (rc6_mode & INTEL_RC6pp_ENABLE)
4247                         rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
4248         }
4249
4250         intel_print_rc6_info(dev, rc6_mask);
4251
4252         I915_WRITE(GEN6_RC_CONTROL,
4253                    rc6_mask |
4254                    GEN6_RC_CTL_EI_MODE(1) |
4255                    GEN6_RC_CTL_HW_ENABLE);
4256
4257         /* Power down if completely idle for over 50ms */
4258         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
4259         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4260
4261         ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
4262         if (ret)
4263                 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
4264
4265         ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
4266         if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
4267                 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
4268                                  (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
4269                                  (pcu_mbox & 0xff) * 50);
4270                 dev_priv->rps.max_freq = pcu_mbox & 0xff;
4271         }
4272
4273         dev_priv->rps.power = HIGH_POWER; /* force a reset */
4274         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4275
4276         rc6vids = 0;
4277         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
4278         if (IS_GEN6(dev) && ret) {
4279                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
4280         } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
4281                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
4282                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
4283                 rc6vids &= 0xffff00;
4284                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
4285                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
4286                 if (ret)
4287                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
4288         }
4289
4290         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4291 }
4292
4293 static void __gen6_update_ring_freq(struct drm_device *dev)
4294 {
4295         struct drm_i915_private *dev_priv = dev->dev_private;
4296         int min_freq = 15;
4297         unsigned int gpu_freq;
4298         unsigned int max_ia_freq, min_ring_freq;
4299         int scaling_factor = 180;
4300         struct cpufreq_policy *policy;
4301
4302         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4303
4304         policy = cpufreq_cpu_get(0);
4305         if (policy) {
4306                 max_ia_freq = policy->cpuinfo.max_freq;
4307                 cpufreq_cpu_put(policy);
4308         } else {
4309                 /*
4310                  * Default to measured freq if none found, PCU will ensure we
4311                  * don't go over
4312                  */
4313                 max_ia_freq = tsc_khz;
4314         }
4315
4316         /* Convert from kHz to MHz */
4317         max_ia_freq /= 1000;
4318
4319         min_ring_freq = I915_READ(DCLK) & 0xf;
4320         /* convert DDR frequency from units of 266.6MHz to bandwidth */
4321         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
4322
4323         /*
4324          * For each potential GPU frequency, load a ring frequency we'd like
4325          * to use for memory access.  We do this by specifying the IA frequency
4326          * the PCU should use as a reference to determine the ring frequency.
4327          */
4328         for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq;
4329              gpu_freq--) {
4330                 int diff = dev_priv->rps.max_freq - gpu_freq;
4331                 unsigned int ia_freq = 0, ring_freq = 0;
4332
4333                 if (INTEL_INFO(dev)->gen >= 8) {
4334                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
4335                         ring_freq = max(min_ring_freq, gpu_freq);
4336                 } else if (IS_HASWELL(dev)) {
4337                         ring_freq = mult_frac(gpu_freq, 5, 4);
4338                         ring_freq = max(min_ring_freq, ring_freq);
4339                         /* leave ia_freq as the default, chosen by cpufreq */
4340                 } else {
4341                         /* On older processors, there is no separate ring
4342                          * clock domain, so in order to boost the bandwidth
4343                          * of the ring, we need to upclock the CPU (ia_freq).
4344                          *
4345                          * For GPU frequencies less than 750MHz,
4346                          * just use the lowest ring freq.
4347                          */
4348                         if (gpu_freq < min_freq)
4349                                 ia_freq = 800;
4350                         else
4351                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
4352                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
4353                 }
4354
4355                 sandybridge_pcode_write(dev_priv,
4356                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
4357                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
4358                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
4359                                         gpu_freq);
4360         }
4361 }
4362
4363 void gen6_update_ring_freq(struct drm_device *dev)
4364 {
4365         struct drm_i915_private *dev_priv = dev->dev_private;
4366
4367         if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev))
4368                 return;
4369
4370         mutex_lock(&dev_priv->rps.hw_lock);
4371         __gen6_update_ring_freq(dev);
4372         mutex_unlock(&dev_priv->rps.hw_lock);
4373 }
4374
4375 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
4376 {
4377         struct drm_device *dev = dev_priv->dev;
4378         u32 val, rp0;
4379
4380         if (dev->pdev->revision >= 0x20) {
4381                 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
4382
4383                 switch (INTEL_INFO(dev)->eu_total) {
4384                 case 8:
4385                                 /* (2 * 4) config */
4386                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
4387                                 break;
4388                 case 12:
4389                                 /* (2 * 6) config */
4390                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
4391                                 break;
4392                 case 16:
4393                                 /* (2 * 8) config */
4394                 default:
4395                                 /* Setting (2 * 8) Min RP0 for any other combination */
4396                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
4397                                 break;
4398                 }
4399                 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
4400         } else {
4401                 /* For pre-production hardware */
4402                 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
4403                 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
4404                        PUNIT_GPU_STATUS_MAX_FREQ_MASK;
4405         }
4406         return rp0;
4407 }
4408
4409 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4410 {
4411         u32 val, rpe;
4412
4413         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
4414         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
4415
4416         return rpe;
4417 }
4418
4419 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
4420 {
4421         struct drm_device *dev = dev_priv->dev;
4422         u32 val, rp1;
4423
4424         if (dev->pdev->revision >= 0x20) {
4425                 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
4426                 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
4427         } else {
4428                 /* For pre-production hardware */
4429                 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4430                 rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
4431                        PUNIT_GPU_STATUS_MAX_FREQ_MASK);
4432         }
4433         return rp1;
4434 }
4435
4436 static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
4437 {
4438         struct drm_device *dev = dev_priv->dev;
4439         u32 val, rpn;
4440
4441         if (dev->pdev->revision >= 0x20) {
4442                 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
4443                 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
4444                        FB_GFX_FREQ_FUSE_MASK);
4445         } else { /* For pre-production hardware */
4446                 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
4447                 rpn = ((val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) &
4448                        PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK);
4449         }
4450
4451         return rpn;
4452 }
4453
4454 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
4455 {
4456         u32 val, rp1;
4457
4458         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
4459
4460         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
4461
4462         return rp1;
4463 }
4464
4465 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
4466 {
4467         u32 val, rp0;
4468
4469         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
4470
4471         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
4472         /* Clamp to max */
4473         rp0 = min_t(u32, rp0, 0xea);
4474
4475         return rp0;
4476 }
4477
4478 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4479 {
4480         u32 val, rpe;
4481
4482         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
4483         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
4484         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
4485         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
4486
4487         return rpe;
4488 }
4489
4490 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4491 {
4492         return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
4493 }
4494
4495 /* Check that the pctx buffer wasn't move under us. */
4496 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
4497 {
4498         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
4499
4500         WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
4501                              dev_priv->vlv_pctx->stolen->start);
4502 }
4503
4504
4505 /* Check that the pcbr address is not empty. */
4506 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
4507 {
4508         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
4509
4510         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
4511 }
4512
4513 static void cherryview_setup_pctx(struct drm_device *dev)
4514 {
4515         struct drm_i915_private *dev_priv = dev->dev_private;
4516         unsigned long pctx_paddr, paddr;
4517         struct i915_gtt *gtt = &dev_priv->gtt;
4518         u32 pcbr;
4519         int pctx_size = 32*1024;
4520
4521         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4522
4523         pcbr = I915_READ(VLV_PCBR);
4524         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
4525                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
4526                 paddr = (dev_priv->mm.stolen_base +
4527                          (gtt->stolen_size - pctx_size));
4528
4529                 pctx_paddr = (paddr & (~4095));
4530                 I915_WRITE(VLV_PCBR, pctx_paddr);
4531         }
4532
4533         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4534 }
4535
4536 static void valleyview_setup_pctx(struct drm_device *dev)
4537 {
4538         struct drm_i915_private *dev_priv = dev->dev_private;
4539         struct drm_i915_gem_object *pctx;
4540         unsigned long pctx_paddr;
4541         u32 pcbr;
4542         int pctx_size = 24*1024;
4543
4544         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4545
4546         pcbr = I915_READ(VLV_PCBR);
4547         if (pcbr) {
4548                 /* BIOS set it up already, grab the pre-alloc'd space */
4549                 int pcbr_offset;
4550
4551                 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
4552                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
4553                                                                       pcbr_offset,
4554                                                                       I915_GTT_OFFSET_NONE,
4555                                                                       pctx_size);
4556                 goto out;
4557         }
4558
4559         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
4560
4561         /*
4562          * From the Gunit register HAS:
4563          * The Gfx driver is expected to program this register and ensure
4564          * proper allocation within Gfx stolen memory.  For example, this
4565          * register should be programmed such than the PCBR range does not
4566          * overlap with other ranges, such as the frame buffer, protected
4567          * memory, or any other relevant ranges.
4568          */
4569         pctx = i915_gem_object_create_stolen(dev, pctx_size);
4570         if (!pctx) {
4571                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
4572                 return;
4573         }
4574
4575         pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
4576         I915_WRITE(VLV_PCBR, pctx_paddr);
4577
4578 out:
4579         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4580         dev_priv->vlv_pctx = pctx;
4581 }
4582
4583 static void valleyview_cleanup_pctx(struct drm_device *dev)
4584 {
4585         struct drm_i915_private *dev_priv = dev->dev_private;
4586
4587         if (WARN_ON(!dev_priv->vlv_pctx))
4588                 return;
4589
4590         drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
4591         dev_priv->vlv_pctx = NULL;
4592 }
4593
4594 static void valleyview_init_gt_powersave(struct drm_device *dev)
4595 {
4596         struct drm_i915_private *dev_priv = dev->dev_private;
4597         u32 val;
4598
4599         valleyview_setup_pctx(dev);
4600
4601         mutex_lock(&dev_priv->rps.hw_lock);
4602
4603         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4604         switch ((val >> 6) & 3) {
4605         case 0:
4606         case 1:
4607                 dev_priv->mem_freq = 800;
4608                 break;
4609         case 2:
4610                 dev_priv->mem_freq = 1066;
4611                 break;
4612         case 3:
4613                 dev_priv->mem_freq = 1333;
4614                 break;
4615         }
4616         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
4617
4618         dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
4619         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
4620         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
4621                          intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
4622                          dev_priv->rps.max_freq);
4623
4624         dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
4625         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
4626                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4627                          dev_priv->rps.efficient_freq);
4628
4629         dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
4630         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
4631                          intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
4632                          dev_priv->rps.rp1_freq);
4633
4634         dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
4635         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
4636                          intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
4637                          dev_priv->rps.min_freq);
4638
4639         /* Preserve min/max settings in case of re-init */
4640         if (dev_priv->rps.max_freq_softlimit == 0)
4641                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4642
4643         if (dev_priv->rps.min_freq_softlimit == 0)
4644                 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4645
4646         mutex_unlock(&dev_priv->rps.hw_lock);
4647 }
4648
4649 static void cherryview_init_gt_powersave(struct drm_device *dev)
4650 {
4651         struct drm_i915_private *dev_priv = dev->dev_private;
4652         u32 val;
4653
4654         cherryview_setup_pctx(dev);
4655
4656         mutex_lock(&dev_priv->rps.hw_lock);
4657
4658         mutex_lock(&dev_priv->dpio_lock);
4659         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
4660         mutex_unlock(&dev_priv->dpio_lock);
4661
4662         switch ((val >> 2) & 0x7) {
4663         case 0:
4664         case 1:
4665                 dev_priv->rps.cz_freq = 200;
4666                 dev_priv->mem_freq = 1600;
4667                 break;
4668         case 2:
4669                 dev_priv->rps.cz_freq = 267;
4670                 dev_priv->mem_freq = 1600;
4671                 break;
4672         case 3:
4673                 dev_priv->rps.cz_freq = 333;
4674                 dev_priv->mem_freq = 2000;
4675                 break;
4676         case 4:
4677                 dev_priv->rps.cz_freq = 320;
4678                 dev_priv->mem_freq = 1600;
4679                 break;
4680         case 5:
4681                 dev_priv->rps.cz_freq = 400;
4682                 dev_priv->mem_freq = 1600;
4683                 break;
4684         }
4685         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
4686
4687         dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
4688         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
4689         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
4690                          intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
4691                          dev_priv->rps.max_freq);
4692
4693         dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
4694         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
4695                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4696                          dev_priv->rps.efficient_freq);
4697
4698         dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
4699         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
4700                          intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
4701                          dev_priv->rps.rp1_freq);
4702
4703         dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
4704         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
4705                          intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
4706                          dev_priv->rps.min_freq);
4707
4708         WARN_ONCE((dev_priv->rps.max_freq |
4709                    dev_priv->rps.efficient_freq |
4710                    dev_priv->rps.rp1_freq |
4711                    dev_priv->rps.min_freq) & 1,
4712                   "Odd GPU freq values\n");
4713
4714         /* Preserve min/max settings in case of re-init */
4715         if (dev_priv->rps.max_freq_softlimit == 0)
4716                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4717
4718         if (dev_priv->rps.min_freq_softlimit == 0)
4719                 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4720
4721         mutex_unlock(&dev_priv->rps.hw_lock);
4722 }
4723
4724 static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
4725 {
4726         valleyview_cleanup_pctx(dev);
4727 }
4728
4729 static void cherryview_enable_rps(struct drm_device *dev)
4730 {
4731         struct drm_i915_private *dev_priv = dev->dev_private;
4732         struct intel_engine_cs *ring;
4733         u32 gtfifodbg, val, rc6_mode = 0, pcbr;
4734         int i;
4735
4736         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4737
4738         gtfifodbg = I915_READ(GTFIFODBG);
4739         if (gtfifodbg) {
4740                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4741                                  gtfifodbg);
4742                 I915_WRITE(GTFIFODBG, gtfifodbg);
4743         }
4744
4745         cherryview_check_pctx(dev_priv);
4746
4747         /* 1a & 1b: Get forcewake during program sequence. Although the driver
4748          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4749         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4750
4751         /*  Disable RC states. */
4752         I915_WRITE(GEN6_RC_CONTROL, 0);
4753
4754         /* 2a: Program RC6 thresholds.*/
4755         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4756         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4757         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4758
4759         for_each_ring(ring, dev_priv, i)
4760                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4761         I915_WRITE(GEN6_RC_SLEEP, 0);
4762
4763         /* TO threshold set to 1750 us ( 0x557 * 1.28 us) */
4764         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4765
4766         /* allows RC6 residency counter to work */
4767         I915_WRITE(VLV_COUNTER_CONTROL,
4768                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
4769                                       VLV_MEDIA_RC6_COUNT_EN |
4770                                       VLV_RENDER_RC6_COUNT_EN));
4771
4772         /* For now we assume BIOS is allocating and populating the PCBR  */
4773         pcbr = I915_READ(VLV_PCBR);
4774
4775         /* 3: Enable RC6 */
4776         if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
4777                                                 (pcbr >> VLV_PCBR_ADDR_SHIFT))
4778                 rc6_mode = GEN7_RC_CTL_TO_MODE;
4779
4780         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
4781
4782         /* 4 Program defaults and thresholds for RPS*/
4783         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4784         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4785         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4786         I915_WRITE(GEN6_RP_UP_EI, 66000);
4787         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4788
4789         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4790
4791         /* 5: Enable RPS */
4792         I915_WRITE(GEN6_RP_CONTROL,
4793                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4794                    GEN6_RP_MEDIA_IS_GFX |
4795                    GEN6_RP_ENABLE |
4796                    GEN6_RP_UP_BUSY_AVG |
4797                    GEN6_RP_DOWN_IDLE_AVG);
4798
4799         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4800
4801         /* RPS code assumes GPLL is used */
4802         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
4803
4804         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
4805         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
4806
4807         dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4808         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
4809                          intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
4810                          dev_priv->rps.cur_freq);
4811
4812         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
4813                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4814                          dev_priv->rps.efficient_freq);
4815
4816         valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4817
4818         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4819 }
4820
4821 static void valleyview_enable_rps(struct drm_device *dev)
4822 {
4823         struct drm_i915_private *dev_priv = dev->dev_private;
4824         struct intel_engine_cs *ring;
4825         u32 gtfifodbg, val, rc6_mode = 0;
4826         int i;
4827
4828         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4829
4830         valleyview_check_pctx(dev_priv);
4831
4832         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4833                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4834                                  gtfifodbg);
4835                 I915_WRITE(GTFIFODBG, gtfifodbg);
4836         }
4837
4838         /* If VLV, Forcewake all wells, else re-direct to regular path */
4839         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4840
4841         /*  Disable RC states. */
4842         I915_WRITE(GEN6_RC_CONTROL, 0);
4843
4844         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4845         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4846         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4847         I915_WRITE(GEN6_RP_UP_EI, 66000);
4848         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4849
4850         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4851
4852         I915_WRITE(GEN6_RP_CONTROL,
4853                    GEN6_RP_MEDIA_TURBO |
4854                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4855                    GEN6_RP_MEDIA_IS_GFX |
4856                    GEN6_RP_ENABLE |
4857                    GEN6_RP_UP_BUSY_AVG |
4858                    GEN6_RP_DOWN_IDLE_CONT);
4859
4860         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
4861         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4862         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4863
4864         for_each_ring(ring, dev_priv, i)
4865                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4866
4867         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4868
4869         /* allows RC6 residency counter to work */
4870         I915_WRITE(VLV_COUNTER_CONTROL,
4871                    _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN |
4872                                       VLV_RENDER_RC0_COUNT_EN |
4873                                       VLV_MEDIA_RC6_COUNT_EN |
4874                                       VLV_RENDER_RC6_COUNT_EN));
4875
4876         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4877                 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
4878
4879         intel_print_rc6_info(dev, rc6_mode);
4880
4881         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
4882
4883         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4884
4885         /* RPS code assumes GPLL is used */
4886         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
4887
4888         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
4889         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
4890
4891         dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4892         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
4893                          intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
4894                          dev_priv->rps.cur_freq);
4895
4896         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
4897                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4898                          dev_priv->rps.efficient_freq);
4899
4900         valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4901
4902         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4903 }
4904
4905 void ironlake_teardown_rc6(struct drm_device *dev)
4906 {
4907         struct drm_i915_private *dev_priv = dev->dev_private;
4908
4909         if (dev_priv->ips.renderctx) {
4910                 i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
4911                 drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
4912                 dev_priv->ips.renderctx = NULL;
4913         }
4914
4915         if (dev_priv->ips.pwrctx) {
4916                 i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
4917                 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
4918                 dev_priv->ips.pwrctx = NULL;
4919         }
4920 }
4921
4922 static void ironlake_disable_rc6(struct drm_device *dev)
4923 {
4924         struct drm_i915_private *dev_priv = dev->dev_private;
4925
4926         if (I915_READ(PWRCTXA)) {
4927                 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
4928                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
4929                 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
4930                          50);
4931
4932                 I915_WRITE(PWRCTXA, 0);
4933                 POSTING_READ(PWRCTXA);
4934
4935                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4936                 POSTING_READ(RSTDBYCTL);
4937         }
4938 }
4939
4940 static int ironlake_setup_rc6(struct drm_device *dev)
4941 {
4942         struct drm_i915_private *dev_priv = dev->dev_private;
4943
4944         if (dev_priv->ips.renderctx == NULL)
4945                 dev_priv->ips.renderctx = intel_alloc_context_page(dev);
4946         if (!dev_priv->ips.renderctx)
4947                 return -ENOMEM;
4948
4949         if (dev_priv->ips.pwrctx == NULL)
4950                 dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
4951         if (!dev_priv->ips.pwrctx) {
4952                 ironlake_teardown_rc6(dev);
4953                 return -ENOMEM;
4954         }
4955
4956         return 0;
4957 }
4958
4959 static void ironlake_enable_rc6(struct drm_device *dev)
4960 {
4961         struct drm_i915_private *dev_priv = dev->dev_private;
4962         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
4963         bool was_interruptible;
4964         int ret;
4965
4966         /* rc6 disabled by default due to repeated reports of hanging during
4967          * boot and resume.
4968          */
4969         if (!intel_enable_rc6(dev))
4970                 return;
4971
4972         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4973
4974         ret = ironlake_setup_rc6(dev);
4975         if (ret)
4976                 return;
4977
4978         was_interruptible = dev_priv->mm.interruptible;
4979         dev_priv->mm.interruptible = false;
4980
4981         /*
4982          * GPU can automatically power down the render unit if given a page
4983          * to save state.
4984          */
4985         ret = intel_ring_begin(ring, 6);
4986         if (ret) {
4987                 ironlake_teardown_rc6(dev);
4988                 dev_priv->mm.interruptible = was_interruptible;
4989                 return;
4990         }
4991
4992         intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
4993         intel_ring_emit(ring, MI_SET_CONTEXT);
4994         intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) |
4995                         MI_MM_SPACE_GTT |
4996                         MI_SAVE_EXT_STATE_EN |
4997                         MI_RESTORE_EXT_STATE_EN |
4998                         MI_RESTORE_INHIBIT);
4999         intel_ring_emit(ring, MI_SUSPEND_FLUSH);
5000         intel_ring_emit(ring, MI_NOOP);
5001         intel_ring_emit(ring, MI_FLUSH);
5002         intel_ring_advance(ring);
5003
5004         /*
5005          * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
5006          * does an implicit flush, combined with MI_FLUSH above, it should be
5007          * safe to assume that renderctx is valid
5008          */
5009         ret = intel_ring_idle(ring);
5010         dev_priv->mm.interruptible = was_interruptible;
5011         if (ret) {
5012                 DRM_ERROR("failed to enable ironlake power savings\n");
5013                 ironlake_teardown_rc6(dev);
5014                 return;
5015         }
5016
5017         I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) | PWRCTX_EN);
5018         I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
5019
5020         intel_print_rc6_info(dev, GEN6_RC_CTL_RC6_ENABLE);
5021 }
5022
5023 static unsigned long intel_pxfreq(u32 vidfreq)
5024 {
5025         unsigned long freq;
5026         int div = (vidfreq & 0x3f0000) >> 16;
5027         int post = (vidfreq & 0x3000) >> 12;
5028         int pre = (vidfreq & 0x7);
5029
5030         if (!pre)
5031                 return 0;
5032
5033         freq = ((div * 133333) / ((1<<post) * pre));
5034
5035         return freq;
5036 }
5037
5038 static const struct cparams {
5039         u16 i;
5040         u16 t;
5041         u16 m;
5042         u16 c;
5043 } cparams[] = {
5044         { 1, 1333, 301, 28664 },
5045         { 1, 1066, 294, 24460 },
5046         { 1, 800, 294, 25192 },
5047         { 0, 1333, 276, 27605 },
5048         { 0, 1066, 276, 27605 },
5049         { 0, 800, 231, 23784 },
5050 };
5051
5052 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
5053 {
5054         u64 total_count, diff, ret;
5055         u32 count1, count2, count3, m = 0, c = 0;
5056         unsigned long now = jiffies_to_msecs(jiffies), diff1;
5057         int i;
5058
5059         assert_spin_locked(&mchdev_lock);
5060
5061         diff1 = now - dev_priv->ips.last_time1;
5062
5063         /* Prevent division-by-zero if we are asking too fast.
5064          * Also, we don't get interesting results if we are polling
5065          * faster than once in 10ms, so just return the saved value
5066          * in such cases.
5067          */
5068         if (diff1 <= 10)
5069                 return dev_priv->ips.chipset_power;
5070
5071         count1 = I915_READ(DMIEC);
5072         count2 = I915_READ(DDREC);
5073         count3 = I915_READ(CSIEC);
5074
5075         total_count = count1 + count2 + count3;
5076
5077         /* FIXME: handle per-counter overflow */
5078         if (total_count < dev_priv->ips.last_count1) {
5079                 diff = ~0UL - dev_priv->ips.last_count1;
5080                 diff += total_count;
5081         } else {
5082                 diff = total_count - dev_priv->ips.last_count1;
5083         }
5084
5085         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
5086                 if (cparams[i].i == dev_priv->ips.c_m &&
5087                     cparams[i].t == dev_priv->ips.r_t) {
5088                         m = cparams[i].m;
5089                         c = cparams[i].c;
5090                         break;
5091                 }
5092         }
5093
5094         diff = div_u64(diff, diff1);
5095         ret = ((m * diff) + c);
5096         ret = div_u64(ret, 10);
5097
5098         dev_priv->ips.last_count1 = total_count;
5099         dev_priv->ips.last_time1 = now;
5100
5101         dev_priv->ips.chipset_power = ret;
5102
5103         return ret;
5104 }
5105
5106 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
5107 {
5108         struct drm_device *dev = dev_priv->dev;
5109         unsigned long val;
5110
5111         if (INTEL_INFO(dev)->gen != 5)
5112                 return 0;
5113
5114         spin_lock_irq(&mchdev_lock);
5115
5116         val = __i915_chipset_val(dev_priv);
5117
5118         spin_unlock_irq(&mchdev_lock);
5119
5120         return val;
5121 }
5122
5123 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
5124 {
5125         unsigned long m, x, b;
5126         u32 tsfs;
5127
5128         tsfs = I915_READ(TSFS);
5129
5130         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
5131         x = I915_READ8(TR1);
5132
5133         b = tsfs & TSFS_INTR_MASK;
5134
5135         return ((m * x) / 127) - b;
5136 }
5137
5138 static int _pxvid_to_vd(u8 pxvid)
5139 {
5140         if (pxvid == 0)
5141                 return 0;
5142
5143         if (pxvid >= 8 && pxvid < 31)
5144                 pxvid = 31;
5145
5146         return (pxvid + 2) * 125;
5147 }
5148
5149 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
5150 {
5151         struct drm_device *dev = dev_priv->dev;
5152         const int vd = _pxvid_to_vd(pxvid);
5153         const int vm = vd - 1125;
5154
5155         if (INTEL_INFO(dev)->is_mobile)
5156                 return vm > 0 ? vm : 0;
5157
5158         return vd;
5159 }
5160
5161 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
5162 {
5163         u64 now, diff, diffms;
5164         u32 count;
5165
5166         assert_spin_locked(&mchdev_lock);
5167
5168         now = ktime_get_raw_ns();
5169         diffms = now - dev_priv->ips.last_time2;
5170         do_div(diffms, NSEC_PER_MSEC);
5171
5172         /* Don't divide by 0 */
5173         if (!diffms)
5174                 return;
5175
5176         count = I915_READ(GFXEC);
5177
5178         if (count < dev_priv->ips.last_count2) {
5179                 diff = ~0UL - dev_priv->ips.last_count2;
5180                 diff += count;
5181         } else {
5182                 diff = count - dev_priv->ips.last_count2;
5183         }
5184
5185         dev_priv->ips.last_count2 = count;
5186         dev_priv->ips.last_time2 = now;
5187
5188         /* More magic constants... */
5189         diff = diff * 1181;
5190         diff = div_u64(diff, diffms * 10);
5191         dev_priv->ips.gfx_power = diff;
5192 }
5193
5194 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
5195 {
5196         struct drm_device *dev = dev_priv->dev;
5197
5198         if (INTEL_INFO(dev)->gen != 5)
5199                 return;
5200
5201         spin_lock_irq(&mchdev_lock);
5202
5203         __i915_update_gfx_val(dev_priv);
5204
5205         spin_unlock_irq(&mchdev_lock);
5206 }
5207
5208 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
5209 {
5210         unsigned long t, corr, state1, corr2, state2;
5211         u32 pxvid, ext_v;
5212
5213         assert_spin_locked(&mchdev_lock);
5214
5215         pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
5216         pxvid = (pxvid >> 24) & 0x7f;
5217         ext_v = pvid_to_extvid(dev_priv, pxvid);
5218
5219         state1 = ext_v;
5220
5221         t = i915_mch_val(dev_priv);
5222
5223         /* Revel in the empirically derived constants */
5224
5225         /* Correction factor in 1/100000 units */
5226         if (t > 80)
5227                 corr = ((t * 2349) + 135940);
5228         else if (t >= 50)
5229                 corr = ((t * 964) + 29317);
5230         else /* < 50 */
5231                 corr = ((t * 301) + 1004);
5232
5233         corr = corr * ((150142 * state1) / 10000 - 78642);
5234         corr /= 100000;
5235         corr2 = (corr * dev_priv->ips.corr);
5236
5237         state2 = (corr2 * state1) / 10000;
5238         state2 /= 100; /* convert to mW */
5239
5240         __i915_update_gfx_val(dev_priv);
5241
5242         return dev_priv->ips.gfx_power + state2;
5243 }
5244
5245 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
5246 {
5247         struct drm_device *dev = dev_priv->dev;
5248         unsigned long val;
5249
5250         if (INTEL_INFO(dev)->gen != 5)
5251                 return 0;
5252
5253         spin_lock_irq(&mchdev_lock);
5254
5255         val = __i915_gfx_val(dev_priv);
5256
5257         spin_unlock_irq(&mchdev_lock);
5258
5259         return val;
5260 }
5261
5262 /**
5263  * i915_read_mch_val - return value for IPS use
5264  *
5265  * Calculate and return a value for the IPS driver to use when deciding whether
5266  * we have thermal and power headroom to increase CPU or GPU power budget.
5267  */
5268 unsigned long i915_read_mch_val(void)
5269 {
5270         struct drm_i915_private *dev_priv;
5271         unsigned long chipset_val, graphics_val, ret = 0;
5272
5273         spin_lock_irq(&mchdev_lock);
5274         if (!i915_mch_dev)
5275                 goto out_unlock;
5276         dev_priv = i915_mch_dev;
5277
5278         chipset_val = __i915_chipset_val(dev_priv);
5279         graphics_val = __i915_gfx_val(dev_priv);
5280
5281         ret = chipset_val + graphics_val;
5282
5283 out_unlock:
5284         spin_unlock_irq(&mchdev_lock);
5285
5286         return ret;
5287 }
5288 EXPORT_SYMBOL_GPL(i915_read_mch_val);
5289
5290 /**
5291  * i915_gpu_raise - raise GPU frequency limit
5292  *
5293  * Raise the limit; IPS indicates we have thermal headroom.
5294  */
5295 bool i915_gpu_raise(void)
5296 {
5297         struct drm_i915_private *dev_priv;
5298         bool ret = true;
5299
5300         spin_lock_irq(&mchdev_lock);
5301         if (!i915_mch_dev) {
5302                 ret = false;
5303                 goto out_unlock;
5304         }
5305         dev_priv = i915_mch_dev;
5306
5307         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
5308                 dev_priv->ips.max_delay--;
5309
5310 out_unlock:
5311         spin_unlock_irq(&mchdev_lock);
5312
5313         return ret;
5314 }
5315 EXPORT_SYMBOL_GPL(i915_gpu_raise);
5316
5317 /**
5318  * i915_gpu_lower - lower GPU frequency limit
5319  *
5320  * IPS indicates we're close to a thermal limit, so throttle back the GPU
5321  * frequency maximum.
5322  */
5323 bool i915_gpu_lower(void)
5324 {
5325         struct drm_i915_private *dev_priv;
5326         bool ret = true;
5327
5328         spin_lock_irq(&mchdev_lock);
5329         if (!i915_mch_dev) {
5330                 ret = false;
5331                 goto out_unlock;
5332         }
5333         dev_priv = i915_mch_dev;
5334
5335         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
5336                 dev_priv->ips.max_delay++;
5337
5338 out_unlock:
5339         spin_unlock_irq(&mchdev_lock);
5340
5341         return ret;
5342 }
5343 EXPORT_SYMBOL_GPL(i915_gpu_lower);
5344
5345 /**
5346  * i915_gpu_busy - indicate GPU business to IPS
5347  *
5348  * Tell the IPS driver whether or not the GPU is busy.
5349  */
5350 bool i915_gpu_busy(void)
5351 {
5352         struct drm_i915_private *dev_priv;
5353         struct intel_engine_cs *ring;
5354         bool ret = false;
5355         int i;
5356
5357         spin_lock_irq(&mchdev_lock);
5358         if (!i915_mch_dev)
5359                 goto out_unlock;
5360         dev_priv = i915_mch_dev;
5361
5362         for_each_ring(ring, dev_priv, i)
5363                 ret |= !list_empty(&ring->request_list);
5364
5365 out_unlock:
5366         spin_unlock_irq(&mchdev_lock);
5367
5368         return ret;
5369 }
5370 EXPORT_SYMBOL_GPL(i915_gpu_busy);
5371
5372 /**
5373  * i915_gpu_turbo_disable - disable graphics turbo
5374  *
5375  * Disable graphics turbo by resetting the max frequency and setting the
5376  * current frequency to the default.
5377  */
5378 bool i915_gpu_turbo_disable(void)
5379 {
5380         struct drm_i915_private *dev_priv;
5381         bool ret = true;
5382
5383         spin_lock_irq(&mchdev_lock);
5384         if (!i915_mch_dev) {
5385                 ret = false;
5386                 goto out_unlock;
5387         }
5388         dev_priv = i915_mch_dev;
5389
5390         dev_priv->ips.max_delay = dev_priv->ips.fstart;
5391
5392         if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
5393                 ret = false;
5394
5395 out_unlock:
5396         spin_unlock_irq(&mchdev_lock);
5397
5398         return ret;
5399 }
5400 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
5401
5402 /**
5403  * Tells the intel_ips driver that the i915 driver is now loaded, if
5404  * IPS got loaded first.
5405  *
5406  * This awkward dance is so that neither module has to depend on the
5407  * other in order for IPS to do the appropriate communication of
5408  * GPU turbo limits to i915.
5409  */
5410 static void
5411 ips_ping_for_i915_load(void)
5412 {
5413         void (*link)(void);
5414
5415         link = symbol_get(ips_link_to_i915_driver);
5416         if (link) {
5417                 link();
5418                 symbol_put(ips_link_to_i915_driver);
5419         }
5420 }
5421
5422 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
5423 {
5424         /* We only register the i915 ips part with intel-ips once everything is
5425          * set up, to avoid intel-ips sneaking in and reading bogus values. */
5426         spin_lock_irq(&mchdev_lock);
5427         i915_mch_dev = dev_priv;
5428         spin_unlock_irq(&mchdev_lock);
5429
5430         ips_ping_for_i915_load();
5431 }
5432
5433 void intel_gpu_ips_teardown(void)
5434 {
5435         spin_lock_irq(&mchdev_lock);
5436         i915_mch_dev = NULL;
5437         spin_unlock_irq(&mchdev_lock);
5438 }
5439
5440 static void intel_init_emon(struct drm_device *dev)
5441 {
5442         struct drm_i915_private *dev_priv = dev->dev_private;
5443         u32 lcfuse;
5444         u8 pxw[16];
5445         int i;
5446
5447         /* Disable to program */
5448         I915_WRITE(ECR, 0);
5449         POSTING_READ(ECR);
5450
5451         /* Program energy weights for various events */
5452         I915_WRITE(SDEW, 0x15040d00);
5453         I915_WRITE(CSIEW0, 0x007f0000);
5454         I915_WRITE(CSIEW1, 0x1e220004);
5455         I915_WRITE(CSIEW2, 0x04000004);
5456
5457         for (i = 0; i < 5; i++)
5458                 I915_WRITE(PEW + (i * 4), 0);
5459         for (i = 0; i < 3; i++)
5460                 I915_WRITE(DEW + (i * 4), 0);
5461
5462         /* Program P-state weights to account for frequency power adjustment */
5463         for (i = 0; i < 16; i++) {
5464                 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
5465                 unsigned long freq = intel_pxfreq(pxvidfreq);
5466                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
5467                         PXVFREQ_PX_SHIFT;
5468                 unsigned long val;
5469
5470                 val = vid * vid;
5471                 val *= (freq / 1000);
5472                 val *= 255;
5473                 val /= (127*127*900);
5474                 if (val > 0xff)
5475                         DRM_ERROR("bad pxval: %ld\n", val);
5476                 pxw[i] = val;
5477         }
5478         /* Render standby states get 0 weight */
5479         pxw[14] = 0;
5480         pxw[15] = 0;
5481
5482         for (i = 0; i < 4; i++) {
5483                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
5484                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
5485                 I915_WRITE(PXW + (i * 4), val);
5486         }
5487
5488         /* Adjust magic regs to magic values (more experimental results) */
5489         I915_WRITE(OGW0, 0);
5490         I915_WRITE(OGW1, 0);
5491         I915_WRITE(EG0, 0x00007f00);
5492         I915_WRITE(EG1, 0x0000000e);
5493         I915_WRITE(EG2, 0x000e0000);
5494         I915_WRITE(EG3, 0x68000300);
5495         I915_WRITE(EG4, 0x42000000);
5496         I915_WRITE(EG5, 0x00140031);
5497         I915_WRITE(EG6, 0);
5498         I915_WRITE(EG7, 0);
5499
5500         for (i = 0; i < 8; i++)
5501                 I915_WRITE(PXWL + (i * 4), 0);
5502
5503         /* Enable PMON + select events */
5504         I915_WRITE(ECR, 0x80000019);
5505
5506         lcfuse = I915_READ(LCFUSE02);
5507
5508         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
5509 }
5510
5511 void intel_init_gt_powersave(struct drm_device *dev)
5512 {
5513         i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
5514
5515         if (IS_CHERRYVIEW(dev))
5516                 cherryview_init_gt_powersave(dev);
5517         else if (IS_VALLEYVIEW(dev))
5518                 valleyview_init_gt_powersave(dev);
5519 }
5520
5521 void intel_cleanup_gt_powersave(struct drm_device *dev)
5522 {
5523         if (IS_CHERRYVIEW(dev))
5524                 return;
5525         else if (IS_VALLEYVIEW(dev))
5526                 valleyview_cleanup_gt_powersave(dev);
5527 }
5528
5529 static void gen6_suspend_rps(struct drm_device *dev)
5530 {
5531         struct drm_i915_private *dev_priv = dev->dev_private;
5532
5533         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
5534
5535         /*
5536          * TODO: disable RPS interrupts on GEN9+ too once RPS support
5537          * is added for it.
5538          */
5539         if (INTEL_INFO(dev)->gen < 9)
5540                 gen6_disable_rps_interrupts(dev);
5541 }
5542
5543 /**
5544  * intel_suspend_gt_powersave - suspend PM work and helper threads
5545  * @dev: drm device
5546  *
5547  * We don't want to disable RC6 or other features here, we just want
5548  * to make sure any work we've queued has finished and won't bother
5549  * us while we're suspended.
5550  */
5551 void intel_suspend_gt_powersave(struct drm_device *dev)
5552 {
5553         struct drm_i915_private *dev_priv = dev->dev_private;
5554
5555         if (INTEL_INFO(dev)->gen < 6)
5556                 return;
5557
5558         gen6_suspend_rps(dev);
5559
5560         /* Force GPU to min freq during suspend */
5561         gen6_rps_idle(dev_priv);
5562 }
5563
5564 void intel_disable_gt_powersave(struct drm_device *dev)
5565 {
5566         struct drm_i915_private *dev_priv = dev->dev_private;
5567
5568         if (IS_IRONLAKE_M(dev)) {
5569                 ironlake_disable_drps(dev);
5570                 ironlake_disable_rc6(dev);
5571         } else if (INTEL_INFO(dev)->gen >= 6) {
5572                 intel_suspend_gt_powersave(dev);
5573
5574                 mutex_lock(&dev_priv->rps.hw_lock);
5575                 if (INTEL_INFO(dev)->gen >= 9)
5576                         gen9_disable_rps(dev);
5577                 else if (IS_CHERRYVIEW(dev))
5578                         cherryview_disable_rps(dev);
5579                 else if (IS_VALLEYVIEW(dev))
5580                         valleyview_disable_rps(dev);
5581                 else
5582                         gen6_disable_rps(dev);
5583
5584                 dev_priv->rps.enabled = false;
5585                 mutex_unlock(&dev_priv->rps.hw_lock);
5586         }
5587 }
5588
5589 static void intel_gen6_powersave_work(struct work_struct *work)
5590 {
5591         struct drm_i915_private *dev_priv =
5592                 container_of(work, struct drm_i915_private,
5593                              rps.delayed_resume_work.work);
5594         struct drm_device *dev = dev_priv->dev;
5595
5596         mutex_lock(&dev_priv->rps.hw_lock);
5597
5598         /*
5599          * TODO: reset/enable RPS interrupts on GEN9+ too, once RPS support is
5600          * added for it.
5601          */
5602         if (INTEL_INFO(dev)->gen < 9)
5603                 gen6_reset_rps_interrupts(dev);
5604
5605         if (IS_CHERRYVIEW(dev)) {
5606                 cherryview_enable_rps(dev);
5607         } else if (IS_VALLEYVIEW(dev)) {
5608                 valleyview_enable_rps(dev);
5609         } else if (INTEL_INFO(dev)->gen >= 9) {
5610                 gen9_enable_rc6(dev);
5611                 gen9_enable_rps(dev);
5612                 __gen6_update_ring_freq(dev);
5613         } else if (IS_BROADWELL(dev)) {
5614                 gen8_enable_rps(dev);
5615                 __gen6_update_ring_freq(dev);
5616         } else {
5617                 gen6_enable_rps(dev);
5618                 __gen6_update_ring_freq(dev);
5619         }
5620         dev_priv->rps.enabled = true;
5621
5622         if (INTEL_INFO(dev)->gen < 9)
5623                 gen6_enable_rps_interrupts(dev);
5624
5625         mutex_unlock(&dev_priv->rps.hw_lock);
5626
5627         intel_runtime_pm_put(dev_priv);
5628 }
5629
5630 void intel_enable_gt_powersave(struct drm_device *dev)
5631 {
5632         struct drm_i915_private *dev_priv = dev->dev_private;
5633
5634         /* Powersaving is controlled by the host when inside a VM */
5635         if (intel_vgpu_active(dev))
5636                 return;
5637
5638         if (IS_IRONLAKE_M(dev)) {
5639                 mutex_lock(&dev->struct_mutex);
5640                 ironlake_enable_drps(dev);
5641                 ironlake_enable_rc6(dev);
5642                 intel_init_emon(dev);
5643                 mutex_unlock(&dev->struct_mutex);
5644         } else if (INTEL_INFO(dev)->gen >= 6) {
5645                 /*
5646                  * PCU communication is slow and this doesn't need to be
5647                  * done at any specific time, so do this out of our fast path
5648                  * to make resume and init faster.
5649                  *
5650                  * We depend on the HW RC6 power context save/restore
5651                  * mechanism when entering D3 through runtime PM suspend. So
5652                  * disable RPM until RPS/RC6 is properly setup. We can only
5653                  * get here via the driver load/system resume/runtime resume
5654                  * paths, so the _noresume version is enough (and in case of
5655                  * runtime resume it's necessary).
5656                  */
5657                 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
5658                                            round_jiffies_up_relative(HZ)))
5659                         intel_runtime_pm_get_noresume(dev_priv);
5660         }
5661 }
5662
5663 void intel_reset_gt_powersave(struct drm_device *dev)
5664 {
5665         struct drm_i915_private *dev_priv = dev->dev_private;
5666
5667         if (INTEL_INFO(dev)->gen < 6)
5668                 return;
5669
5670         gen6_suspend_rps(dev);
5671         dev_priv->rps.enabled = false;
5672 }
5673
5674 static void ibx_init_clock_gating(struct drm_device *dev)
5675 {
5676         struct drm_i915_private *dev_priv = dev->dev_private;
5677
5678         /*
5679          * On Ibex Peak and Cougar Point, we need to disable clock
5680          * gating for the panel power sequencer or it will fail to
5681          * start up when no ports are active.
5682          */
5683         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
5684 }
5685
5686 static void g4x_disable_trickle_feed(struct drm_device *dev)
5687 {
5688         struct drm_i915_private *dev_priv = dev->dev_private;
5689         int pipe;
5690
5691         for_each_pipe(dev_priv, pipe) {
5692                 I915_WRITE(DSPCNTR(pipe),
5693                            I915_READ(DSPCNTR(pipe)) |
5694                            DISPPLANE_TRICKLE_FEED_DISABLE);
5695                 intel_flush_primary_plane(dev_priv, pipe);
5696         }
5697 }
5698
5699 static void ilk_init_lp_watermarks(struct drm_device *dev)
5700 {
5701         struct drm_i915_private *dev_priv = dev->dev_private;
5702
5703         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
5704         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
5705         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
5706
5707         /*
5708          * Don't touch WM1S_LP_EN here.
5709          * Doing so could cause underruns.
5710          */
5711 }
5712
5713 static void ironlake_init_clock_gating(struct drm_device *dev)
5714 {
5715         struct drm_i915_private *dev_priv = dev->dev_private;
5716         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
5717
5718         /*
5719          * Required for FBC
5720          * WaFbcDisableDpfcClockGating:ilk
5721          */
5722         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
5723                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
5724                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
5725
5726         I915_WRITE(PCH_3DCGDIS0,
5727                    MARIUNIT_CLOCK_GATE_DISABLE |
5728                    SVSMUNIT_CLOCK_GATE_DISABLE);
5729         I915_WRITE(PCH_3DCGDIS1,
5730                    VFMUNIT_CLOCK_GATE_DISABLE);
5731
5732         /*
5733          * According to the spec the following bits should be set in
5734          * order to enable memory self-refresh
5735          * The bit 22/21 of 0x42004
5736          * The bit 5 of 0x42020
5737          * The bit 15 of 0x45000
5738          */
5739         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5740                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
5741                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
5742         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
5743         I915_WRITE(DISP_ARB_CTL,
5744                    (I915_READ(DISP_ARB_CTL) |
5745                     DISP_FBC_WM_DIS));
5746
5747         ilk_init_lp_watermarks(dev);
5748
5749         /*
5750          * Based on the document from hardware guys the following bits
5751          * should be set unconditionally in order to enable FBC.
5752          * The bit 22 of 0x42000
5753          * The bit 22 of 0x42004
5754          * The bit 7,8,9 of 0x42020.
5755          */
5756         if (IS_IRONLAKE_M(dev)) {
5757                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
5758                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5759                            I915_READ(ILK_DISPLAY_CHICKEN1) |
5760                            ILK_FBCQ_DIS);
5761                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5762                            I915_READ(ILK_DISPLAY_CHICKEN2) |
5763                            ILK_DPARB_GATE);
5764         }
5765
5766         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5767
5768         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5769                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5770                    ILK_ELPIN_409_SELECT);
5771         I915_WRITE(_3D_CHICKEN2,
5772                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
5773                    _3D_CHICKEN2_WM_READ_PIPELINED);
5774
5775         /* WaDisableRenderCachePipelinedFlush:ilk */
5776         I915_WRITE(CACHE_MODE_0,
5777                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
5778
5779         /* WaDisable_RenderCache_OperationalFlush:ilk */
5780         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5781
5782         g4x_disable_trickle_feed(dev);
5783
5784         ibx_init_clock_gating(dev);
5785 }
5786
5787 static void cpt_init_clock_gating(struct drm_device *dev)
5788 {
5789         struct drm_i915_private *dev_priv = dev->dev_private;
5790         int pipe;
5791         uint32_t val;
5792
5793         /*
5794          * On Ibex Peak and Cougar Point, we need to disable clock
5795          * gating for the panel power sequencer or it will fail to
5796          * start up when no ports are active.
5797          */
5798         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
5799                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
5800                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
5801         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
5802                    DPLS_EDP_PPS_FIX_DIS);
5803         /* The below fixes the weird display corruption, a few pixels shifted
5804          * downward, on (only) LVDS of some HP laptops with IVY.
5805          */
5806         for_each_pipe(dev_priv, pipe) {
5807                 val = I915_READ(TRANS_CHICKEN2(pipe));
5808                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
5809                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
5810                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
5811                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
5812                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
5813                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
5814                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
5815                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
5816         }
5817         /* WADP0ClockGatingDisable */
5818         for_each_pipe(dev_priv, pipe) {
5819                 I915_WRITE(TRANS_CHICKEN1(pipe),
5820                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5821         }
5822 }
5823
5824 static void gen6_check_mch_setup(struct drm_device *dev)
5825 {
5826         struct drm_i915_private *dev_priv = dev->dev_private;
5827         uint32_t tmp;
5828
5829         tmp = I915_READ(MCH_SSKPD);
5830         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
5831                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
5832                               tmp);
5833 }
5834
5835 static void gen6_init_clock_gating(struct drm_device *dev)
5836 {
5837         struct drm_i915_private *dev_priv = dev->dev_private;
5838         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
5839
5840         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5841
5842         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5843                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5844                    ILK_ELPIN_409_SELECT);
5845
5846         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
5847         I915_WRITE(_3D_CHICKEN,
5848                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
5849
5850         /* WaDisable_RenderCache_OperationalFlush:snb */
5851         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5852
5853         /*
5854          * BSpec recoomends 8x4 when MSAA is used,
5855          * however in practice 16x4 seems fastest.
5856          *
5857          * Note that PS/WM thread counts depend on the WIZ hashing
5858          * disable bit, which we don't touch here, but it's good
5859          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
5860          */
5861         I915_WRITE(GEN6_GT_MODE,
5862                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5863
5864         ilk_init_lp_watermarks(dev);
5865
5866         I915_WRITE(CACHE_MODE_0,
5867                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
5868
5869         I915_WRITE(GEN6_UCGCTL1,
5870                    I915_READ(GEN6_UCGCTL1) |
5871                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
5872                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
5873
5874         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5875          * gating disable must be set.  Failure to set it results in
5876          * flickering pixels due to Z write ordering failures after
5877          * some amount of runtime in the Mesa "fire" demo, and Unigine
5878          * Sanctuary and Tropics, and apparently anything else with
5879          * alpha test or pixel discard.
5880          *
5881          * According to the spec, bit 11 (RCCUNIT) must also be set,
5882          * but we didn't debug actual testcases to find it out.
5883          *
5884          * WaDisableRCCUnitClockGating:snb
5885          * WaDisableRCPBUnitClockGating:snb
5886          */
5887         I915_WRITE(GEN6_UCGCTL2,
5888                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
5889                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5890
5891         /* WaStripsFansDisableFastClipPerformanceFix:snb */
5892         I915_WRITE(_3D_CHICKEN3,
5893                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
5894
5895         /*
5896          * Bspec says:
5897          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
5898          * 3DSTATE_SF number of SF output attributes is more than 16."
5899          */
5900         I915_WRITE(_3D_CHICKEN3,
5901                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
5902
5903         /*
5904          * According to the spec the following bits should be
5905          * set in order to enable memory self-refresh and fbc:
5906          * The bit21 and bit22 of 0x42000
5907          * The bit21 and bit22 of 0x42004
5908          * The bit5 and bit7 of 0x42020
5909          * The bit14 of 0x70180
5910          * The bit14 of 0x71180
5911          *
5912          * WaFbcAsynchFlipDisableFbcQueue:snb
5913          */
5914         I915_WRITE(ILK_DISPLAY_CHICKEN1,
5915                    I915_READ(ILK_DISPLAY_CHICKEN1) |
5916                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
5917         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5918                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5919                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
5920         I915_WRITE(ILK_DSPCLK_GATE_D,
5921                    I915_READ(ILK_DSPCLK_GATE_D) |
5922                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
5923                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
5924
5925         g4x_disable_trickle_feed(dev);
5926
5927         cpt_init_clock_gating(dev);
5928
5929         gen6_check_mch_setup(dev);
5930 }
5931
5932 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
5933 {
5934         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
5935
5936         /*
5937          * WaVSThreadDispatchOverride:ivb,vlv
5938          *
5939          * This actually overrides the dispatch
5940          * mode for all thread types.
5941          */
5942         reg &= ~GEN7_FF_SCHED_MASK;
5943         reg |= GEN7_FF_TS_SCHED_HW;
5944         reg |= GEN7_FF_VS_SCHED_HW;
5945         reg |= GEN7_FF_DS_SCHED_HW;
5946
5947         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
5948 }
5949
5950 static void lpt_init_clock_gating(struct drm_device *dev)
5951 {
5952         struct drm_i915_private *dev_priv = dev->dev_private;
5953
5954         /*
5955          * TODO: this bit should only be enabled when really needed, then
5956          * disabled when not needed anymore in order to save power.
5957          */
5958         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
5959                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
5960                            I915_READ(SOUTH_DSPCLK_GATE_D) |
5961                            PCH_LP_PARTITION_LEVEL_DISABLE);
5962
5963         /* WADPOClockGatingDisable:hsw */
5964         I915_WRITE(_TRANSA_CHICKEN1,
5965                    I915_READ(_TRANSA_CHICKEN1) |
5966                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5967 }
5968
5969 static void lpt_suspend_hw(struct drm_device *dev)
5970 {
5971         struct drm_i915_private *dev_priv = dev->dev_private;
5972
5973         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
5974                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
5975
5976                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
5977                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
5978         }
5979 }
5980
5981 static void broadwell_init_clock_gating(struct drm_device *dev)
5982 {
5983         struct drm_i915_private *dev_priv = dev->dev_private;
5984         enum pipe pipe;
5985
5986         I915_WRITE(WM3_LP_ILK, 0);
5987         I915_WRITE(WM2_LP_ILK, 0);
5988         I915_WRITE(WM1_LP_ILK, 0);
5989
5990         /* WaSwitchSolVfFArbitrationPriority:bdw */
5991         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
5992
5993         /* WaPsrDPAMaskVBlankInSRD:bdw */
5994         I915_WRITE(CHICKEN_PAR1_1,
5995                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
5996
5997         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5998         for_each_pipe(dev_priv, pipe) {
5999                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
6000                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
6001                            BDW_DPRS_MASK_VBLANK_SRD);
6002         }
6003
6004         /* WaVSRefCountFullforceMissDisable:bdw */
6005         /* WaDSRefCountFullforceMissDisable:bdw */
6006         I915_WRITE(GEN7_FF_THREAD_MODE,
6007                    I915_READ(GEN7_FF_THREAD_MODE) &
6008                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6009
6010         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6011                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6012
6013         /* WaDisableSDEUnitClockGating:bdw */
6014         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6015                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6016
6017         lpt_init_clock_gating(dev);
6018 }
6019
6020 static void haswell_init_clock_gating(struct drm_device *dev)
6021 {
6022         struct drm_i915_private *dev_priv = dev->dev_private;
6023
6024         ilk_init_lp_watermarks(dev);
6025
6026         /* L3 caching of data atomics doesn't work -- disable it. */
6027         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
6028         I915_WRITE(HSW_ROW_CHICKEN3,
6029                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
6030
6031         /* This is required by WaCatErrorRejectionIssue:hsw */
6032         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6033                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6034                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6035
6036         /* WaVSRefCountFullforceMissDisable:hsw */
6037         I915_WRITE(GEN7_FF_THREAD_MODE,
6038                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
6039
6040         /* WaDisable_RenderCache_OperationalFlush:hsw */
6041         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6042
6043         /* enable HiZ Raw Stall Optimization */
6044         I915_WRITE(CACHE_MODE_0_GEN7,
6045                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6046
6047         /* WaDisable4x2SubspanOptimization:hsw */
6048         I915_WRITE(CACHE_MODE_1,
6049                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6050
6051         /*
6052          * BSpec recommends 8x4 when MSAA is used,
6053          * however in practice 16x4 seems fastest.
6054          *
6055          * Note that PS/WM thread counts depend on the WIZ hashing
6056          * disable bit, which we don't touch here, but it's good
6057          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6058          */
6059         I915_WRITE(GEN7_GT_MODE,
6060                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6061
6062         /* WaSampleCChickenBitEnable:hsw */
6063         I915_WRITE(HALF_SLICE_CHICKEN3,
6064                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
6065
6066         /* WaSwitchSolVfFArbitrationPriority:hsw */
6067         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6068
6069         /* WaRsPkgCStateDisplayPMReq:hsw */
6070         I915_WRITE(CHICKEN_PAR1_1,
6071                    I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
6072
6073         lpt_init_clock_gating(dev);
6074 }
6075
6076 static void ivybridge_init_clock_gating(struct drm_device *dev)
6077 {
6078         struct drm_i915_private *dev_priv = dev->dev_private;
6079         uint32_t snpcr;
6080
6081         ilk_init_lp_watermarks(dev);
6082
6083         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6084
6085         /* WaDisableEarlyCull:ivb */
6086         I915_WRITE(_3D_CHICKEN3,
6087                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6088
6089         /* WaDisableBackToBackFlipFix:ivb */
6090         I915_WRITE(IVB_CHICKEN3,
6091                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6092                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6093
6094         /* WaDisablePSDDualDispatchEnable:ivb */
6095         if (IS_IVB_GT1(dev))
6096                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6097                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6098
6099         /* WaDisable_RenderCache_OperationalFlush:ivb */
6100         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6101
6102         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6103         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
6104                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
6105
6106         /* WaApplyL3ControlAndL3ChickenMode:ivb */
6107         I915_WRITE(GEN7_L3CNTLREG1,
6108                         GEN7_WA_FOR_GEN7_L3_CONTROL);
6109         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6110                    GEN7_WA_L3_CHICKEN_MODE);
6111         if (IS_IVB_GT1(dev))
6112                 I915_WRITE(GEN7_ROW_CHICKEN2,
6113                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6114         else {
6115                 /* must write both registers */
6116                 I915_WRITE(GEN7_ROW_CHICKEN2,
6117                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6118                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
6119                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6120         }
6121
6122         /* WaForceL3Serialization:ivb */
6123         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6124                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6125
6126         /*
6127          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6128          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
6129          */
6130         I915_WRITE(GEN6_UCGCTL2,
6131                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6132
6133         /* This is required by WaCatErrorRejectionIssue:ivb */
6134         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6135                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6136                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6137
6138         g4x_disable_trickle_feed(dev);
6139
6140         gen7_setup_fixed_func_scheduler(dev_priv);
6141
6142         if (0) { /* causes HiZ corruption on ivb:gt1 */
6143                 /* enable HiZ Raw Stall Optimization */
6144                 I915_WRITE(CACHE_MODE_0_GEN7,
6145                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6146         }
6147
6148         /* WaDisable4x2SubspanOptimization:ivb */
6149         I915_WRITE(CACHE_MODE_1,
6150                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6151
6152         /*
6153          * BSpec recommends 8x4 when MSAA is used,
6154          * however in practice 16x4 seems fastest.
6155          *
6156          * Note that PS/WM thread counts depend on the WIZ hashing
6157          * disable bit, which we don't touch here, but it's good
6158          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6159          */
6160         I915_WRITE(GEN7_GT_MODE,
6161                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6162
6163         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
6164         snpcr &= ~GEN6_MBC_SNPCR_MASK;
6165         snpcr |= GEN6_MBC_SNPCR_MED;
6166         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
6167
6168         if (!HAS_PCH_NOP(dev))
6169                 cpt_init_clock_gating(dev);
6170
6171         gen6_check_mch_setup(dev);
6172 }
6173
6174 static void valleyview_init_clock_gating(struct drm_device *dev)
6175 {
6176         struct drm_i915_private *dev_priv = dev->dev_private;
6177
6178         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6179
6180         /* WaDisableEarlyCull:vlv */
6181         I915_WRITE(_3D_CHICKEN3,
6182                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6183
6184         /* WaDisableBackToBackFlipFix:vlv */
6185         I915_WRITE(IVB_CHICKEN3,
6186                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6187                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6188
6189         /* WaPsdDispatchEnable:vlv */
6190         /* WaDisablePSDDualDispatchEnable:vlv */
6191         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6192                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
6193                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6194
6195         /* WaDisable_RenderCache_OperationalFlush:vlv */
6196         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6197
6198         /* WaForceL3Serialization:vlv */
6199         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6200                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6201
6202         /* WaDisableDopClockGating:vlv */
6203         I915_WRITE(GEN7_ROW_CHICKEN2,
6204                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6205
6206         /* This is required by WaCatErrorRejectionIssue:vlv */
6207         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6208                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6209                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6210
6211         gen7_setup_fixed_func_scheduler(dev_priv);
6212
6213         /*
6214          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6215          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
6216          */
6217         I915_WRITE(GEN6_UCGCTL2,
6218                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6219
6220         /* WaDisableL3Bank2xClockGate:vlv
6221          * Disabling L3 clock gating- MMIO 940c[25] = 1
6222          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
6223         I915_WRITE(GEN7_UCGCTL4,
6224                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
6225
6226         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6227
6228         /*
6229          * BSpec says this must be set, even though
6230          * WaDisable4x2SubspanOptimization isn't listed for VLV.
6231          */
6232         I915_WRITE(CACHE_MODE_1,
6233                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6234
6235         /*
6236          * BSpec recommends 8x4 when MSAA is used,
6237          * however in practice 16x4 seems fastest.
6238          *
6239          * Note that PS/WM thread counts depend on the WIZ hashing
6240          * disable bit, which we don't touch here, but it's good
6241          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6242          */
6243         I915_WRITE(GEN7_GT_MODE,
6244                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6245
6246         /*
6247          * WaIncreaseL3CreditsForVLVB0:vlv
6248          * This is the hardware default actually.
6249          */
6250         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
6251
6252         /*
6253          * WaDisableVLVClockGating_VBIIssue:vlv
6254          * Disable clock gating on th GCFG unit to prevent a delay
6255          * in the reporting of vblank events.
6256          */
6257         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6258 }
6259
6260 static void cherryview_init_clock_gating(struct drm_device *dev)
6261 {
6262         struct drm_i915_private *dev_priv = dev->dev_private;
6263
6264         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6265
6266         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6267
6268         /* WaVSRefCountFullforceMissDisable:chv */
6269         /* WaDSRefCountFullforceMissDisable:chv */
6270         I915_WRITE(GEN7_FF_THREAD_MODE,
6271                    I915_READ(GEN7_FF_THREAD_MODE) &
6272                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6273
6274         /* WaDisableSemaphoreAndSyncFlipWait:chv */
6275         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6276                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6277
6278         /* WaDisableCSUnitClockGating:chv */
6279         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6280                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6281
6282         /* WaDisableSDEUnitClockGating:chv */
6283         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6284                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6285 }
6286
6287 static void g4x_init_clock_gating(struct drm_device *dev)
6288 {
6289         struct drm_i915_private *dev_priv = dev->dev_private;
6290         uint32_t dspclk_gate;
6291
6292         I915_WRITE(RENCLK_GATE_D1, 0);
6293         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
6294                    GS_UNIT_CLOCK_GATE_DISABLE |
6295                    CL_UNIT_CLOCK_GATE_DISABLE);
6296         I915_WRITE(RAMCLK_GATE_D, 0);
6297         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
6298                 OVRUNIT_CLOCK_GATE_DISABLE |
6299                 OVCUNIT_CLOCK_GATE_DISABLE;
6300         if (IS_GM45(dev))
6301                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
6302         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
6303
6304         /* WaDisableRenderCachePipelinedFlush */
6305         I915_WRITE(CACHE_MODE_0,
6306                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6307
6308         /* WaDisable_RenderCache_OperationalFlush:g4x */
6309         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6310
6311         g4x_disable_trickle_feed(dev);
6312 }
6313
6314 static void crestline_init_clock_gating(struct drm_device *dev)
6315 {
6316         struct drm_i915_private *dev_priv = dev->dev_private;
6317
6318         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
6319         I915_WRITE(RENCLK_GATE_D2, 0);
6320         I915_WRITE(DSPCLK_GATE_D, 0);
6321         I915_WRITE(RAMCLK_GATE_D, 0);
6322         I915_WRITE16(DEUC, 0);
6323         I915_WRITE(MI_ARB_STATE,
6324                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6325
6326         /* WaDisable_RenderCache_OperationalFlush:gen4 */
6327         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6328 }
6329
6330 static void broadwater_init_clock_gating(struct drm_device *dev)
6331 {
6332         struct drm_i915_private *dev_priv = dev->dev_private;
6333
6334         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
6335                    I965_RCC_CLOCK_GATE_DISABLE |
6336                    I965_RCPB_CLOCK_GATE_DISABLE |
6337                    I965_ISC_CLOCK_GATE_DISABLE |
6338                    I965_FBC_CLOCK_GATE_DISABLE);
6339         I915_WRITE(RENCLK_GATE_D2, 0);
6340         I915_WRITE(MI_ARB_STATE,
6341                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6342
6343         /* WaDisable_RenderCache_OperationalFlush:gen4 */
6344         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6345 }
6346
6347 static void gen3_init_clock_gating(struct drm_device *dev)
6348 {
6349         struct drm_i915_private *dev_priv = dev->dev_private;
6350         u32 dstate = I915_READ(D_STATE);
6351
6352         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
6353                 DSTATE_DOT_CLOCK_GATING;
6354         I915_WRITE(D_STATE, dstate);
6355
6356         if (IS_PINEVIEW(dev))
6357                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
6358
6359         /* IIR "flip pending" means done if this bit is set */
6360         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
6361
6362         /* interrupts should cause a wake up from C3 */
6363         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
6364
6365         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
6366         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
6367
6368         I915_WRITE(MI_ARB_STATE,
6369                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6370 }
6371
6372 static void i85x_init_clock_gating(struct drm_device *dev)
6373 {
6374         struct drm_i915_private *dev_priv = dev->dev_private;
6375
6376         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
6377
6378         /* interrupts should cause a wake up from C3 */
6379         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
6380                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
6381
6382         I915_WRITE(MEM_MODE,
6383                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
6384 }
6385
6386 static void i830_init_clock_gating(struct drm_device *dev)
6387 {
6388         struct drm_i915_private *dev_priv = dev->dev_private;
6389
6390         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
6391
6392         I915_WRITE(MEM_MODE,
6393                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
6394                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
6395 }
6396
6397 void intel_init_clock_gating(struct drm_device *dev)
6398 {
6399         struct drm_i915_private *dev_priv = dev->dev_private;
6400
6401         if (dev_priv->display.init_clock_gating)
6402                 dev_priv->display.init_clock_gating(dev);
6403 }
6404
6405 void intel_suspend_hw(struct drm_device *dev)
6406 {
6407         if (HAS_PCH_LPT(dev))
6408                 lpt_suspend_hw(dev);
6409 }
6410
6411 /* Set up chip specific power management-related functions */
6412 void intel_init_pm(struct drm_device *dev)
6413 {
6414         struct drm_i915_private *dev_priv = dev->dev_private;
6415
6416         intel_fbc_init(dev_priv);
6417
6418         /* For cxsr */
6419         if (IS_PINEVIEW(dev))
6420                 i915_pineview_get_mem_freq(dev);
6421         else if (IS_GEN5(dev))
6422                 i915_ironlake_get_mem_freq(dev);
6423
6424         /* For FIFO watermark updates */
6425         if (INTEL_INFO(dev)->gen >= 9) {
6426                 skl_setup_wm_latency(dev);
6427
6428                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
6429                 dev_priv->display.update_wm = skl_update_wm;
6430                 dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
6431         } else if (HAS_PCH_SPLIT(dev)) {
6432                 ilk_setup_wm_latency(dev);
6433
6434                 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
6435                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
6436                     (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
6437                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
6438                         dev_priv->display.update_wm = ilk_update_wm;
6439                         dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6440                 } else {
6441                         DRM_DEBUG_KMS("Failed to read display plane latency. "
6442                                       "Disable CxSR\n");
6443                 }
6444
6445                 if (IS_GEN5(dev))
6446                         dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
6447                 else if (IS_GEN6(dev))
6448                         dev_priv->display.init_clock_gating = gen6_init_clock_gating;
6449                 else if (IS_IVYBRIDGE(dev))
6450                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
6451                 else if (IS_HASWELL(dev))
6452                         dev_priv->display.init_clock_gating = haswell_init_clock_gating;
6453                 else if (INTEL_INFO(dev)->gen == 8)
6454                         dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
6455         } else if (IS_CHERRYVIEW(dev)) {
6456                 dev_priv->display.update_wm = cherryview_update_wm;
6457                 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
6458                 dev_priv->display.init_clock_gating =
6459                         cherryview_init_clock_gating;
6460         } else if (IS_VALLEYVIEW(dev)) {
6461                 dev_priv->display.update_wm = valleyview_update_wm;
6462                 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
6463                 dev_priv->display.init_clock_gating =
6464                         valleyview_init_clock_gating;
6465         } else if (IS_PINEVIEW(dev)) {
6466                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
6467                                             dev_priv->is_ddr3,
6468                                             dev_priv->fsb_freq,
6469                                             dev_priv->mem_freq)) {
6470                         DRM_INFO("failed to find known CxSR latency "
6471                                  "(found ddr%s fsb freq %d, mem freq %d), "
6472                                  "disabling CxSR\n",
6473                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
6474                                  dev_priv->fsb_freq, dev_priv->mem_freq);
6475                         /* Disable CxSR and never update its watermark again */
6476                         intel_set_memory_cxsr(dev_priv, false);
6477                         dev_priv->display.update_wm = NULL;
6478                 } else
6479                         dev_priv->display.update_wm = pineview_update_wm;
6480                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6481         } else if (IS_G4X(dev)) {
6482                 dev_priv->display.update_wm = g4x_update_wm;
6483                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
6484         } else if (IS_GEN4(dev)) {
6485                 dev_priv->display.update_wm = i965_update_wm;
6486                 if (IS_CRESTLINE(dev))
6487                         dev_priv->display.init_clock_gating = crestline_init_clock_gating;
6488                 else if (IS_BROADWATER(dev))
6489                         dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
6490         } else if (IS_GEN3(dev)) {
6491                 dev_priv->display.update_wm = i9xx_update_wm;
6492                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
6493                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6494         } else if (IS_GEN2(dev)) {
6495                 if (INTEL_INFO(dev)->num_pipes == 1) {
6496                         dev_priv->display.update_wm = i845_update_wm;
6497                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
6498                 } else {
6499                         dev_priv->display.update_wm = i9xx_update_wm;
6500                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
6501                 }
6502
6503                 if (IS_I85X(dev) || IS_I865G(dev))
6504                         dev_priv->display.init_clock_gating = i85x_init_clock_gating;
6505                 else
6506                         dev_priv->display.init_clock_gating = i830_init_clock_gating;
6507         } else {
6508                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
6509         }
6510 }
6511
6512 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
6513 {
6514         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6515
6516         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6517                 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
6518                 return -EAGAIN;
6519         }
6520
6521         I915_WRITE(GEN6_PCODE_DATA, *val);
6522         I915_WRITE(GEN6_PCODE_DATA1, 0);
6523         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6524
6525         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6526                      500)) {
6527                 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
6528                 return -ETIMEDOUT;
6529         }
6530
6531         *val = I915_READ(GEN6_PCODE_DATA);
6532         I915_WRITE(GEN6_PCODE_DATA, 0);
6533
6534         return 0;
6535 }
6536
6537 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
6538 {
6539         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6540
6541         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6542                 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
6543                 return -EAGAIN;
6544         }
6545
6546         I915_WRITE(GEN6_PCODE_DATA, val);
6547         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6548
6549         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6550                      500)) {
6551                 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
6552                 return -ETIMEDOUT;
6553         }
6554
6555         I915_WRITE(GEN6_PCODE_DATA, 0);
6556
6557         return 0;
6558 }
6559
6560 static int vlv_gpu_freq_div(unsigned int czclk_freq)
6561 {
6562         switch (czclk_freq) {
6563         case 200:
6564                 return 10;
6565         case 267:
6566                 return 12;
6567         case 320:
6568         case 333:
6569                 return 16;
6570         case 400:
6571                 return 20;
6572         default:
6573                 return -1;
6574         }
6575 }
6576
6577 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
6578 {
6579         int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
6580
6581         div = vlv_gpu_freq_div(czclk_freq);
6582         if (div < 0)
6583                 return div;
6584
6585         return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
6586 }
6587
6588 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
6589 {
6590         int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
6591
6592         mul = vlv_gpu_freq_div(czclk_freq);
6593         if (mul < 0)
6594                 return mul;
6595
6596         return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
6597 }
6598
6599 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
6600 {
6601         int div, czclk_freq = dev_priv->rps.cz_freq;
6602
6603         div = vlv_gpu_freq_div(czclk_freq) / 2;
6604         if (div < 0)
6605                 return div;
6606
6607         return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
6608 }
6609
6610 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
6611 {
6612         int mul, czclk_freq = dev_priv->rps.cz_freq;
6613
6614         mul = vlv_gpu_freq_div(czclk_freq) / 2;
6615         if (mul < 0)
6616                 return mul;
6617
6618         /* CHV needs even values */
6619         return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
6620 }
6621
6622 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
6623 {
6624         if (IS_CHERRYVIEW(dev_priv->dev))
6625                 return chv_gpu_freq(dev_priv, val);
6626         else if (IS_VALLEYVIEW(dev_priv->dev))
6627                 return byt_gpu_freq(dev_priv, val);
6628         else
6629                 return val * GT_FREQUENCY_MULTIPLIER;
6630 }
6631
6632 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
6633 {
6634         if (IS_CHERRYVIEW(dev_priv->dev))
6635                 return chv_freq_opcode(dev_priv, val);
6636         else if (IS_VALLEYVIEW(dev_priv->dev))
6637                 return byt_freq_opcode(dev_priv, val);
6638         else
6639                 return val / GT_FREQUENCY_MULTIPLIER;
6640 }
6641
6642 void intel_pm_setup(struct drm_device *dev)
6643 {
6644         struct drm_i915_private *dev_priv = dev->dev_private;
6645
6646         mutex_init(&dev_priv->rps.hw_lock);
6647
6648         INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
6649                           intel_gen6_powersave_work);
6650
6651         dev_priv->pm.suspended = false;
6652 }