drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include "nvc0.h"
  26
  27 /*******************************************************************************
  28  * Graphics object classes
  29  ******************************************************************************/
  30
  31 struct nouveau_oclass
  32 nvc0_graph_sclass[] = {
  33         { 0x902d, &nouveau_object_ofuncs },
  34         { 0x9039, &nouveau_object_ofuncs },
  35         { 0x9097, &nouveau_object_ofuncs },
  36         { 0x90c0, &nouveau_object_ofuncs },
  37         {}
  38 };
  39
  40 /*******************************************************************************
  41  * PGRAPH context
  42  ******************************************************************************/
  43
  44 int
  45 nvc0_graph_context_ctor(struct nouveau_object *parent,
  46                         struct nouveau_object *engine,
  47                         struct nouveau_oclass *oclass, void *args, u32 size,
  48                         struct nouveau_object **pobject)
  49 {
  50         struct nouveau_vm *vm = nouveau_client(parent)->vm;
  51         struct nvc0_graph_priv *priv = (void *)engine;
  52         struct nvc0_graph_data *data = priv->mmio_data;
  53         struct nvc0_graph_mmio *mmio = priv->mmio_list;
  54         struct nvc0_graph_chan *chan;
  55         int ret, i;
  56
  57         /* allocate memory for context, and fill with default values */
  58         ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
  59                                            priv->size, 0x100,
  60                                            NVOBJ_FLAG_ZERO_ALLOC, &chan);
  61         *pobject = nv_object(chan);
  62         if (ret)
  63                 return ret;
  64
  65         /* allocate memory for a "mmio list" buffer that's used by the HUB
  66          * fuc to modify some per-context register settings on first load
  67          * of the context.
  68          */
  69         ret = nouveau_gpuobj_new(nv_object(chan), NULL, 0x1000, 0x100, 0,
  70                                 &chan->mmio);
  71         if (ret)
  72                 return ret;
  73
  74         ret = nouveau_gpuobj_map_vm(nv_gpuobj(chan->mmio), vm,
  75                                     NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
  76                                     &chan->mmio_vma);
  77         if (ret)
  78                 return ret;
  79
  80         /* allocate buffers referenced by mmio list */
  81         for (i = 0; data->size && i < ARRAY_SIZE(priv->mmio_data); i++) {
  82                 ret = nouveau_gpuobj_new(nv_object(chan), NULL, data->size,
  83                                          data->align, 0, &chan->data[i].mem);
  84                 if (ret)
  85                         return ret;
  86
  87                 ret = nouveau_gpuobj_map_vm(chan->data[i].mem, vm, data->access,
  88                                            &chan->data[i].vma);
  89                 if (ret)
  90                         return ret;
  91
  92                 data++;
  93         }
  94
  95         /* finally, fill in the mmio list and point the context at it */
  96         for (i = 0; mmio->addr && i < ARRAY_SIZE(priv->mmio_list); i++) {
  97                 u32 addr = mmio->addr;
  98                 u32 data = mmio->data;
  99
 100                 if (mmio->shift) {
 101                         u64 info = chan->data[mmio->buffer].vma.offset;
 102                         data |= info >> mmio->shift;
 103                 }
 104
 105                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
 106                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
 107                 mmio++;
 108         }
 109
 110         for (i = 0; i < priv->size; i += 4)
 111                 nv_wo32(chan, i, priv->data[i / 4]);
 112
 113         if (!priv->firmware) {
 114                 nv_wo32(chan, 0x00, chan->mmio_nr / 2);
 115                 nv_wo32(chan, 0x04, chan->mmio_vma.offset >> 8);
 116         } else {
 117                 nv_wo32(chan, 0xf4, 0);
 118                 nv_wo32(chan, 0xf8, 0);
 119                 nv_wo32(chan, 0x10, chan->mmio_nr / 2);
 120                 nv_wo32(chan, 0x14, lower_32_bits(chan->mmio_vma.offset));
 121                 nv_wo32(chan, 0x18, upper_32_bits(chan->mmio_vma.offset));
 122                 nv_wo32(chan, 0x1c, 1);
 123                 nv_wo32(chan, 0x20, 0);
 124                 nv_wo32(chan, 0x28, 0);
 125                 nv_wo32(chan, 0x2c, 0);
 126         }
 127
 128         return 0;
 129 }
 130
 131 void
 132 nvc0_graph_context_dtor(struct nouveau_object *object)
 133 {
 134         struct nvc0_graph_chan *chan = (void *)object;
 135         int i;
 136
 137         for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
 138                 nouveau_gpuobj_unmap(&chan->data[i].vma);
 139                 nouveau_gpuobj_ref(NULL, &chan->data[i].mem);
 140         }
 141
 142         nouveau_gpuobj_unmap(&chan->mmio_vma);
 143         nouveau_gpuobj_ref(NULL, &chan->mmio);
 144
 145         nouveau_graph_context_destroy(&chan->base);
 146 }
 147
 148 /*******************************************************************************
 149  * PGRAPH engine/subdev functions
 150  ******************************************************************************/
 151
 152 struct nvc0_graph_init
 153 nvc0_graph_init_regs[] = {
 154         { 0x400080,   1, 0x04, 0x003083c2 },
 155         { 0x400088,   1, 0x04, 0x00006fe7 },
 156         { 0x40008c,   1, 0x04, 0x00000000 },
 157         { 0x400090,   1, 0x04, 0x00000030 },
 158         { 0x40013c,   1, 0x04, 0x013901f7 },
 159         { 0x400140,   1, 0x04, 0x00000100 },
 160         { 0x400144,   1, 0x04, 0x00000000 },
 161         { 0x400148,   1, 0x04, 0x00000110 },
 162         { 0x400138,   1, 0x04, 0x00000000 },
 163         { 0x400130,   2, 0x04, 0x00000000 },
 164         { 0x400124,   1, 0x04, 0x00000002 },
 165         {}
 166 };
 167
 168 struct nvc0_graph_init
 169 nvc0_graph_init_unk40xx[] = {
 170         { 0x40415c,   1, 0x04, 0x00000000 },
 171         { 0x404170,   1, 0x04, 0x00000000 },
 172         {}
 173 };
 174
 175 struct nvc0_graph_init
 176 nvc0_graph_init_unk44xx[] = {
 177         { 0x404488,   2, 0x04, 0x00000000 },
 178         {}
 179 };
 180
 181 struct nvc0_graph_init
 182 nvc0_graph_init_unk78xx[] = {
 183         { 0x407808,   1, 0x04, 0x00000000 },
 184         {}
 185 };
 186
 187 struct nvc0_graph_init
 188 nvc0_graph_init_unk60xx[] = {
 189         { 0x406024,   1, 0x04, 0x00000000 },
 190         {}
 191 };
 192
 193 struct nvc0_graph_init
 194 nvc0_graph_init_unk58xx[] = {
 195         { 0x405844,   1, 0x04, 0x00ffffff },
 196         { 0x405850,   1, 0x04, 0x00000000 },
 197         { 0x405908,   1, 0x04, 0x00000000 },
 198         {}
 199 };
 200
 201 struct nvc0_graph_init
 202 nvc0_graph_init_unk80xx[] = {
 203         { 0x40803c,   1, 0x04, 0x00000000 },
 204         {}
 205 };
 206
 207 struct nvc0_graph_init
 208 nvc0_graph_init_gpc[] = {
 209         { 0x4184a0,   1, 0x04, 0x00000000 },
 210         { 0x418604,   1, 0x04, 0x00000000 },
 211         { 0x418680,   1, 0x04, 0x00000000 },
 212         { 0x418714,   1, 0x04, 0x80000000 },
 213         { 0x418384,   1, 0x04, 0x00000000 },
 214         { 0x418814,   3, 0x04, 0x00000000 },
 215         { 0x418b04,   1, 0x04, 0x00000000 },
 216         { 0x4188c8,   1, 0x04, 0x80000000 },
 217         { 0x4188cc,   1, 0x04, 0x00000000 },
 218         { 0x4188d0,   1, 0x04, 0x00010000 },
 219         { 0x4188d4,   1, 0x04, 0x00000001 },
 220         { 0x418910,   1, 0x04, 0x00010001 },
 221         { 0x418914,   1, 0x04, 0x00000301 },
 222         { 0x418918,   1, 0x04, 0x00800000 },
 223         { 0x418980,   1, 0x04, 0x77777770 },
 224         { 0x418984,   3, 0x04, 0x77777777 },
 225         { 0x418c04,   1, 0x04, 0x00000000 },
 226         { 0x418c88,   1, 0x04, 0x00000000 },
 227         { 0x418d00,   1, 0x04, 0x00000000 },
 228         { 0x418f08,   1, 0x04, 0x00000000 },
 229         { 0x418e00,   1, 0x04, 0x00000050 },
 230         { 0x418e08,   1, 0x04, 0x00000000 },
 231         { 0x41900c,   1, 0x04, 0x00000000 },
 232         { 0x419018,   1, 0x04, 0x00000000 },
 233         {}
 234 };
 235
 236 static struct nvc0_graph_init
 237 nvc0_graph_init_tpc[] = {
 238         { 0x419d08,   2, 0x04, 0x00000000 },
 239         { 0x419d10,   1, 0x04, 0x00000014 },
 240         { 0x419ab0,   1, 0x04, 0x00000000 },
 241         { 0x419ab8,   1, 0x04, 0x000000e7 },
 242         { 0x419abc,   2, 0x04, 0x00000000 },
 243         { 0x41980c,   3, 0x04, 0x00000000 },
 244         { 0x419844,   1, 0x04, 0x00000000 },
 245         { 0x41984c,   1, 0x04, 0x00005bc5 },
 246         { 0x419850,   4, 0x04, 0x00000000 },
 247         { 0x419c98,   1, 0x04, 0x00000000 },
 248         { 0x419ca8,   1, 0x04, 0x80000000 },
 249         { 0x419cb4,   1, 0x04, 0x00000000 },
 250         { 0x419cb8,   1, 0x04, 0x00008bf4 },
 251         { 0x419cbc,   1, 0x04, 0x28137606 },
 252         { 0x419cc0,   2, 0x04, 0x00000000 },
 253         { 0x419bd4,   1, 0x04, 0x00800000 },
 254         { 0x419bdc,   1, 0x04, 0x00000000 },
 255         { 0x419d2c,   1, 0x04, 0x00000000 },
 256         { 0x419c0c,   1, 0x04, 0x00000000 },
 257         { 0x419e00,   1, 0x04, 0x00000000 },
 258         { 0x419ea0,   1, 0x04, 0x00000000 },
 259         { 0x419ea4,   1, 0x04, 0x00000100 },
 260         { 0x419ea8,   1, 0x04, 0x00001100 },
 261         { 0x419eac,   1, 0x04, 0x11100702 },
 262         { 0x419eb0,   1, 0x04, 0x00000003 },
 263         { 0x419eb4,   4, 0x04, 0x00000000 },
 264         { 0x419ec8,   1, 0x04, 0x06060618 },
 265         { 0x419ed0,   1, 0x04, 0x0eff0e38 },
 266         { 0x419ed4,   1, 0x04, 0x011104f1 },
 267         { 0x419edc,   1, 0x04, 0x00000000 },
 268         { 0x419f00,   1, 0x04, 0x00000000 },
 269         { 0x419f2c,   1, 0x04, 0x00000000 },
 270         {}
 271 };
 272
 273 struct nvc0_graph_init
 274 nvc0_graph_init_unk88xx[] = {
 275         { 0x40880c,   1, 0x04, 0x00000000 },
 276         { 0x408910,   9, 0x04, 0x00000000 },
 277         { 0x408950,   1, 0x04, 0x00000000 },
 278         { 0x408954,   1, 0x04, 0x0000ffff },
 279         { 0x408984,   1, 0x04, 0x00000000 },
 280         { 0x408988,   1, 0x04, 0x08040201 },
 281         { 0x40898c,   1, 0x04, 0x80402010 },
 282         {}
 283 };
 284
 285 struct nvc0_graph_init
 286 nvc0_graph_tpc_0[] = {
 287         { 0x50405c,   1, 0x04, 0x00000001 },
 288         {}
 289 };
 290
 291 void
 292 nvc0_graph_mmio(struct nvc0_graph_priv *priv, struct nvc0_graph_init *init)
 293 {
 294         for (; init && init->count; init++) {
 295                 u32 addr = init->addr, i;
 296                 for (i = 0; i < init->count; i++) {
 297                         nv_wr32(priv, addr, init->data);
 298                         addr += init->pitch;
 299                 }
 300         }
 301 }
 302
 303 void
 304 nvc0_graph_icmd(struct nvc0_graph_priv *priv, struct nvc0_graph_init *init)
 305 {
 306         u32 addr, data;
 307         int i, j;
 308
 309         nv_wr32(priv, 0x400208, 0x80000000);
 310         for (i = 0; init->count; init++, i++) {
 311                 if (!i || data != init->data) {
 312                         nv_wr32(priv, 0x400204, init->data);
 313                         data = init->data;
 314                 }
 315
 316                 addr = init->addr;
 317                 for (j = 0; j < init->count; j++) {
 318                         nv_wr32(priv, 0x400200, addr);
 319                         addr += init->pitch;
 320                         while (nv_rd32(priv, 0x400700) & 0x00000002) {}
 321                 }
 322         }
 323         nv_wr32(priv, 0x400208, 0x00000000);
 324 }
 325
 326 void
 327 nvc0_graph_mthd(struct nvc0_graph_priv *priv, struct nvc0_graph_mthd *mthds)
 328 {
 329         struct nvc0_graph_mthd *mthd;
 330         struct nvc0_graph_init *init;
 331         int i = 0, j;
 332         u32 data;
 333
 334         while ((mthd = &mthds[i++]) && (init = mthd->init)) {
 335                 u32  addr = 0x80000000 | mthd->oclass;
 336                 for (data = 0; init->count; init++) {
 337                         if (data != init->data) {
 338                                 nv_wr32(priv, 0x40448c, init->data);
 339                                 data = init->data;
 340                         }
 341
 342                         addr = (addr & 0x8000ffff) | (init->addr << 14);
 343                         for (j = 0; j < init->count; j++) {
 344                                 nv_wr32(priv, 0x404488, addr);
 345                                 addr += init->pitch << 14;
 346                         }
 347                 }
 348         }
 349 }
 350
 351 u64
 352 nvc0_graph_units(struct nouveau_graph *graph)
 353 {
 354         struct nvc0_graph_priv *priv = (void *)graph;
 355         u64 cfg;
 356
 357         cfg  = (u32)priv->gpc_nr;
 358         cfg |= (u32)priv->tpc_total << 8;
 359         cfg |= (u64)priv->rop_nr << 32;
 360
 361         return cfg;
 362 }
 363
 364 static const struct nouveau_enum nve0_sked_error[] = {
 365         { 7, "CONSTANT_BUFFER_SIZE" },
 366         { 9, "LOCAL_MEMORY_SIZE_POS" },
 367         { 10, "LOCAL_MEMORY_SIZE_NEG" },
 368         { 11, "WARP_CSTACK_SIZE" },
 369         { 12, "TOTAL_TEMP_SIZE" },
 370         { 13, "REGISTER_COUNT" },
 371         { 18, "TOTAL_THREADS" },
 372         { 20, "PROGRAM_OFFSET" },
 373         { 21, "SHARED_MEMORY_SIZE" },
 374         { 25, "SHARED_CONFIG_TOO_SMALL" },
 375         { 26, "TOTAL_REGISTER_COUNT" },
 376         {}
 377 };
 378
 379 static const struct nouveau_enum nvc0_gpc_rop_error[] = {
 380         { 1, "RT_PITCH_OVERRUN" },
 381         { 4, "RT_WIDTH_OVERRUN" },
 382         { 5, "RT_HEIGHT_OVERRUN" },
 383         { 7, "ZETA_STORAGE_TYPE_MISMATCH" },
 384         { 8, "RT_STORAGE_TYPE_MISMATCH" },
 385         { 10, "RT_LINEAR_MISMATCH" },
 386         {}
 387 };
 388
 389 static void
 390 nvc0_graph_trap_gpc_rop(struct nvc0_graph_priv *priv, int gpc)
 391 {
 392         u32 trap[4];
 393         int i;
 394
 395         trap[0] = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
 396         trap[1] = nv_rd32(priv, GPC_UNIT(gpc, 0x0434));
 397         trap[2] = nv_rd32(priv, GPC_UNIT(gpc, 0x0438));
 398         trap[3] = nv_rd32(priv, GPC_UNIT(gpc, 0x043c));
 399
 400         nv_error(priv, "GPC%d/PROP trap:", gpc);
 401         for (i = 0; i <= 29; ++i) {
 402                 if (!(trap[0] & (1 << i)))
 403                         continue;
 404                 pr_cont(" ");
 405                 nouveau_enum_print(nvc0_gpc_rop_error, i);
 406         }
 407         pr_cont("\n");
 408
 409         nv_error(priv, "x = %u, y = %u, format = %x, storage type = %x\n",
 410                  trap[1] & 0xffff, trap[1] >> 16, (trap[2] >> 8) & 0x3f,
 411                  trap[3] & 0xff);
 412         nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 413 }
 414
 415 static const struct nouveau_enum nvc0_mp_warp_error[] = {
 416         { 0x00, "NO_ERROR" },
 417         { 0x01, "STACK_MISMATCH" },
 418         { 0x05, "MISALIGNED_PC" },
 419         { 0x08, "MISALIGNED_GPR" },
 420         { 0x09, "INVALID_OPCODE" },
 421         { 0x0d, "GPR_OUT_OF_BOUNDS" },
 422         { 0x0e, "MEM_OUT_OF_BOUNDS" },
 423         { 0x0f, "UNALIGNED_MEM_ACCESS" },
 424         { 0x11, "INVALID_PARAM" },
 425         {}
 426 };
 427
 428 static const struct nouveau_bitfield nvc0_mp_global_error[] = {
 429         { 0x00000004, "MULTIPLE_WARP_ERRORS" },
 430         { 0x00000008, "OUT_OF_STACK_SPACE" },
 431         {}
 432 };
 433
 434 static void
 435 nvc0_graph_trap_mp(struct nvc0_graph_priv *priv, int gpc, int tpc)
 436 {
 437         u32 werr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x648));
 438         u32 gerr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x650));
 439
 440         nv_error(priv, "GPC%i/TPC%i/MP trap:", gpc, tpc);
 441         nouveau_bitfield_print(nvc0_mp_global_error, gerr);
 442         if (werr) {
 443                 pr_cont(" ");
 444                 nouveau_enum_print(nvc0_mp_warp_error, werr & 0xffff);
 445         }
 446         pr_cont("\n");
 447
 448         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);
 449         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x650), gerr);
 450 }
 451
 452 static void
 453 nvc0_graph_trap_tpc(struct nvc0_graph_priv *priv, int gpc, int tpc)
 454 {
 455         u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0508));
 456
 457         if (stat & 0x00000001) {
 458                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0224));
 459                 nv_error(priv, "GPC%d/TPC%d/TEX: 0x%08x\n", gpc, tpc, trap);
 460                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
 461                 stat &= ~0x00000001;
 462         }
 463
 464         if (stat & 0x00000002) {
 465                 nvc0_graph_trap_mp(priv, gpc, tpc);
 466                 stat &= ~0x00000002;
 467         }
 468
 469         if (stat & 0x00000004) {
 470                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0084));
 471                 nv_error(priv, "GPC%d/TPC%d/POLY: 0x%08x\n", gpc, tpc, trap);
 472                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
 473                 stat &= ~0x00000004;
 474         }
 475
 476         if (stat & 0x00000008) {
 477                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x048c));
 478                 nv_error(priv, "GPC%d/TPC%d/L1C: 0x%08x\n", gpc, tpc, trap);
 479                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
 480                 stat &= ~0x00000008;
 481         }
 482
 483         if (stat) {
 484                 nv_error(priv, "GPC%d/TPC%d/0x%08x: unknown\n", gpc, tpc, stat);
 485         }
 486 }
 487
 488 static void
 489 nvc0_graph_trap_gpc(struct nvc0_graph_priv *priv, int gpc)
 490 {
 491         u32 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
 492         int tpc;
 493
 494         if (stat & 0x00000001) {
 495                 nvc0_graph_trap_gpc_rop(priv, gpc);
 496                 stat &= ~0x00000001;
 497         }
 498
 499         if (stat & 0x00000002) {
 500                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
 501                 nv_error(priv, "GPC%d/ZCULL: 0x%08x\n", gpc, trap);
 502                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 503                 stat &= ~0x00000002;
 504         }
 505
 506         if (stat & 0x00000004) {
 507                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
 508                 nv_error(priv, "GPC%d/CCACHE: 0x%08x\n", gpc, trap);
 509                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 510                 stat &= ~0x00000004;
 511         }
 512
 513         if (stat & 0x00000008) {
 514                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
 515                 nv_error(priv, "GPC%d/ESETUP: 0x%08x\n", gpc, trap);
 516                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 517                 stat &= ~0x00000009;
 518         }
 519
 520         for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
 521                 u32 mask = 0x00010000 << tpc;
 522                 if (stat & mask) {
 523                         nvc0_graph_trap_tpc(priv, gpc, tpc);
 524                         nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), mask);
 525                         stat &= ~mask;
 526                 }
 527         }
 528
 529         if (stat) {
 530                 nv_error(priv, "GPC%d/0x%08x: unknown\n", gpc, stat);
 531         }
 532 }
 533
 534 static void
 535 nvc0_graph_trap_intr(struct nvc0_graph_priv *priv)
 536 {
 537         u32 trap = nv_rd32(priv, 0x400108);
 538         int rop, gpc, i;
 539
 540         if (trap & 0x00000001) {
 541                 u32 stat = nv_rd32(priv, 0x404000);
 542                 nv_error(priv, "DISPATCH 0x%08x\n", stat);
 543                 nv_wr32(priv, 0x404000, 0xc0000000);
 544                 nv_wr32(priv, 0x400108, 0x00000001);
 545                 trap &= ~0x00000001;
 546         }
 547
 548         if (trap & 0x00000002) {
 549                 u32 stat = nv_rd32(priv, 0x404600);
 550                 nv_error(priv, "M2MF 0x%08x\n", stat);
 551                 nv_wr32(priv, 0x404600, 0xc0000000);
 552                 nv_wr32(priv, 0x400108, 0x00000002);
 553                 trap &= ~0x00000002;
 554         }
 555
 556         if (trap & 0x00000008) {
 557                 u32 stat = nv_rd32(priv, 0x408030);
 558                 nv_error(priv, "CCACHE 0x%08x\n", stat);
 559                 nv_wr32(priv, 0x408030, 0xc0000000);
 560                 nv_wr32(priv, 0x400108, 0x00000008);
 561                 trap &= ~0x00000008;
 562         }
 563
 564         if (trap & 0x00000010) {
 565                 u32 stat = nv_rd32(priv, 0x405840);
 566                 nv_error(priv, "SHADER 0x%08x\n", stat);
 567                 nv_wr32(priv, 0x405840, 0xc0000000);
 568                 nv_wr32(priv, 0x400108, 0x00000010);
 569                 trap &= ~0x00000010;
 570         }
 571
 572         if (trap & 0x00000040) {
 573                 u32 stat = nv_rd32(priv, 0x40601c);
 574                 nv_error(priv, "UNK6 0x%08x\n", stat);
 575                 nv_wr32(priv, 0x40601c, 0xc0000000);
 576                 nv_wr32(priv, 0x400108, 0x00000040);
 577                 trap &= ~0x00000040;
 578         }
 579
 580         if (trap & 0x00000080) {
 581                 u32 stat = nv_rd32(priv, 0x404490);
 582                 nv_error(priv, "MACRO 0x%08x\n", stat);
 583                 nv_wr32(priv, 0x404490, 0xc0000000);
 584                 nv_wr32(priv, 0x400108, 0x00000080);
 585                 trap &= ~0x00000080;
 586         }
 587
 588         if (trap & 0x00000100) {
 589                 u32 stat = nv_rd32(priv, 0x407020);
 590
 591                 nv_error(priv, "SKED:");
 592                 for (i = 0; i <= 29; ++i) {
 593                         if (!(stat & (1 << i)))
 594                                 continue;
 595                         pr_cont(" ");
 596                         nouveau_enum_print(nve0_sked_error, i);
 597                 }
 598                 pr_cont("\n");
 599
 600                 if (stat & 0x3fffffff)
 601                         nv_wr32(priv, 0x407020, 0x40000000);
 602                 nv_wr32(priv, 0x400108, 0x00000100);
 603                 trap &= ~0x00000100;
 604         }
 605
 606         if (trap & 0x01000000) {
 607                 u32 stat = nv_rd32(priv, 0x400118);
 608                 for (gpc = 0; stat && gpc < priv->gpc_nr; gpc++) {
 609                         u32 mask = 0x00000001 << gpc;
 610                         if (stat & mask) {
 611                                 nvc0_graph_trap_gpc(priv, gpc);
 612                                 nv_wr32(priv, 0x400118, mask);
 613                                 stat &= ~mask;
 614                         }
 615                 }
 616                 nv_wr32(priv, 0x400108, 0x01000000);
 617                 trap &= ~0x01000000;
 618         }
 619
 620         if (trap & 0x02000000) {
 621                 for (rop = 0; rop < priv->rop_nr; rop++) {
 622                         u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
 623                         u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
 624                         nv_error(priv, "ROP%d 0x%08x 0x%08x\n",
 625                                  rop, statz, statc);
 626                         nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
 627                         nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
 628                 }
 629                 nv_wr32(priv, 0x400108, 0x02000000);
 630                 trap &= ~0x02000000;
 631         }
 632
 633         if (trap) {
 634                 nv_error(priv, "TRAP UNHANDLED 0x%08x\n", trap);
 635                 nv_wr32(priv, 0x400108, trap);
 636         }
 637 }
 638
 639 static void
 640 nvc0_graph_ctxctl_debug_unit(struct nvc0_graph_priv *priv, u32 base)
 641 {
 642         nv_error(priv, "%06x - done 0x%08x\n", base,
 643                  nv_rd32(priv, base + 0x400));
 644         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
 645                  nv_rd32(priv, base + 0x800), nv_rd32(priv, base + 0x804),
 646                  nv_rd32(priv, base + 0x808), nv_rd32(priv, base + 0x80c));
 647         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
 648                  nv_rd32(priv, base + 0x810), nv_rd32(priv, base + 0x814),
 649                  nv_rd32(priv, base + 0x818), nv_rd32(priv, base + 0x81c));
 650 }
 651
 652 void
 653 nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *priv)
 654 {
 655         u32 gpcnr = nv_rd32(priv, 0x409604) & 0xffff;
 656         u32 gpc;
 657
 658         nvc0_graph_ctxctl_debug_unit(priv, 0x409000);
 659         for (gpc = 0; gpc < gpcnr; gpc++)
 660                 nvc0_graph_ctxctl_debug_unit(priv, 0x502000 + (gpc * 0x8000));
 661 }
 662
 663 static void
 664 nvc0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
 665 {
 666         u32 ustat = nv_rd32(priv, 0x409c18);
 667
 668         if (ustat & 0x00000001)
 669                 nv_error(priv, "CTXCTL ucode error\n");
 670         if (ustat & 0x00080000)
 671                 nv_error(priv, "CTXCTL watchdog timeout\n");
 672         if (ustat & ~0x00080001)
 673                 nv_error(priv, "CTXCTL 0x%08x\n", ustat);
 674
 675         nvc0_graph_ctxctl_debug(priv);
 676         nv_wr32(priv, 0x409c20, ustat);
 677 }
 678
 679 static void
 680 nvc0_graph_intr(struct nouveau_subdev *subdev)
 681 {
 682         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
 683         struct nouveau_engine *engine = nv_engine(subdev);
 684         struct nouveau_object *engctx;
 685         struct nouveau_handle *handle;
 686         struct nvc0_graph_priv *priv = (void *)subdev;
 687         u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
 688         u32 stat = nv_rd32(priv, 0x400100);
 689         u32 addr = nv_rd32(priv, 0x400704);
 690         u32 mthd = (addr & 0x00003ffc);
 691         u32 subc = (addr & 0x00070000) >> 16;
 692         u32 data = nv_rd32(priv, 0x400708);
 693         u32 code = nv_rd32(priv, 0x400110);
 694         u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
 695         int chid;
 696
 697         engctx = nouveau_engctx_get(engine, inst);
 698         chid   = pfifo->chid(pfifo, engctx);
 699
 700         if (stat & 0x00000010) {
 701                 handle = nouveau_handle_get_class(engctx, class);
 702                 if (!handle || nv_call(handle->object, mthd, data)) {
 703                         nv_error(priv,
 704                                  "ILLEGAL_MTHD ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 705                                  chid, inst << 12, nouveau_client_name(engctx),
 706                                  subc, class, mthd, data);
 707                 }
 708                 nouveau_handle_put(handle);
 709                 nv_wr32(priv, 0x400100, 0x00000010);
 710                 stat &= ~0x00000010;
 711         }
 712
 713         if (stat & 0x00000020) {
 714                 nv_error(priv,
 715                          "ILLEGAL_CLASS ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 716                          chid, inst << 12, nouveau_client_name(engctx), subc,
 717                          class, mthd, data);
 718                 nv_wr32(priv, 0x400100, 0x00000020);
 719                 stat &= ~0x00000020;
 720         }
 721
 722         if (stat & 0x00100000) {
 723                 nv_error(priv, "DATA_ERROR [");
 724                 nouveau_enum_print(nv50_data_error_names, code);
 725                 pr_cont("] ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 726                         chid, inst << 12, nouveau_client_name(engctx), subc,
 727                         class, mthd, data);
 728                 nv_wr32(priv, 0x400100, 0x00100000);
 729                 stat &= ~0x00100000;
 730         }
 731
 732         if (stat & 0x00200000) {
 733                 nv_error(priv, "TRAP ch %d [0x%010llx %s]\n", chid, inst << 12,
 734                          nouveau_client_name(engctx));
 735                 nvc0_graph_trap_intr(priv);
 736                 nv_wr32(priv, 0x400100, 0x00200000);
 737                 stat &= ~0x00200000;
 738         }
 739
 740         if (stat & 0x00080000) {
 741                 nvc0_graph_ctxctl_isr(priv);
 742                 nv_wr32(priv, 0x400100, 0x00080000);
 743                 stat &= ~0x00080000;
 744         }
 745
 746         if (stat) {
 747                 nv_error(priv, "unknown stat 0x%08x\n", stat);
 748                 nv_wr32(priv, 0x400100, stat);
 749         }
 750
 751         nv_wr32(priv, 0x400500, 0x00010001);
 752         nouveau_engctx_put(engctx);
 753 }
 754
 755 void
 756 nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
 757                    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
 758 {
 759         int i;
 760
 761         nv_wr32(priv, fuc_base + 0x01c0, 0x01000000);
 762         for (i = 0; i < data->size / 4; i++)
 763                 nv_wr32(priv, fuc_base + 0x01c4, data->data[i]);
 764
 765         nv_wr32(priv, fuc_base + 0x0180, 0x01000000);
 766         for (i = 0; i < code->size / 4; i++) {
 767                 if ((i & 0x3f) == 0)
 768                         nv_wr32(priv, fuc_base + 0x0188, i >> 6);
 769                 nv_wr32(priv, fuc_base + 0x0184, code->data[i]);
 770         }
 771 }
 772
 773 static void
 774 nvc0_graph_init_csdata(struct nvc0_graph_priv *priv,
 775                        struct nvc0_graph_init *init,
 776                        u32 falcon, u32 starstar, u32 base)
 777 {
 778         u32 addr = init->addr;
 779         u32 next = addr;
 780         u32 star, temp;
 781
 782         nv_wr32(priv, falcon + 0x01c0, 0x02000000 + starstar);
 783         star = nv_rd32(priv, falcon + 0x01c4);
 784         temp = nv_rd32(priv, falcon + 0x01c4);
 785         if (temp > star)
 786                 star = temp;
 787         nv_wr32(priv, falcon + 0x01c0, 0x01000000 + star);
 788
 789         do {
 790                 if (init->addr != next) {
 791                         while (addr < next) {
 792                                 u32 nr = min((int)(next - addr) / 4, 32);
 793                                 nv_wr32(priv, falcon + 0x01c4,
 794                                         ((nr - 1) << 26) | (addr - base));
 795                                 addr += nr * 4;
 796                                 star += 4;
 797                         }
 798                         addr = next = init->addr;
 799                 }
 800                 next += init->count * 4;
 801         } while ((init++)->count);
 802
 803         nv_wr32(priv, falcon + 0x01c0, 0x01000004 + starstar);
 804         nv_wr32(priv, falcon + 0x01c4, star);
 805 }
 806
 807 int
 808 nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
 809 {
 810         struct nvc0_graph_oclass *oclass = (void *)nv_object(priv)->oclass;
 811         struct nvc0_grctx_oclass *cclass = (void *)nv_engine(priv)->cclass;
 812         struct nvc0_graph_init *init;
 813         u32 r000260;
 814         int i;
 815
 816         if (priv->firmware) {
 817                 /* load fuc microcode */
 818                 r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 819                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
 820                                                    &priv->fuc409d);
 821                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
 822                                                    &priv->fuc41ad);
 823                 nv_wr32(priv, 0x000260, r000260);
 824
 825                 /* start both of them running */
 826                 nv_wr32(priv, 0x409840, 0xffffffff);
 827                 nv_wr32(priv, 0x41a10c, 0x00000000);
 828                 nv_wr32(priv, 0x40910c, 0x00000000);
 829                 nv_wr32(priv, 0x41a100, 0x00000002);
 830                 nv_wr32(priv, 0x409100, 0x00000002);
 831                 if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
 832                         nv_warn(priv, "0x409800 wait failed\n");
 833
 834                 nv_wr32(priv, 0x409840, 0xffffffff);
 835                 nv_wr32(priv, 0x409500, 0x7fffffff);
 836                 nv_wr32(priv, 0x409504, 0x00000021);
 837
 838                 nv_wr32(priv, 0x409840, 0xffffffff);
 839                 nv_wr32(priv, 0x409500, 0x00000000);
 840                 nv_wr32(priv, 0x409504, 0x00000010);
 841                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 842                         nv_error(priv, "fuc09 req 0x10 timeout\n");
 843                         return -EBUSY;
 844                 }
 845                 priv->size = nv_rd32(priv, 0x409800);
 846
 847                 nv_wr32(priv, 0x409840, 0xffffffff);
 848                 nv_wr32(priv, 0x409500, 0x00000000);
 849                 nv_wr32(priv, 0x409504, 0x00000016);
 850                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 851                         nv_error(priv, "fuc09 req 0x16 timeout\n");
 852                         return -EBUSY;
 853                 }
 854
 855                 nv_wr32(priv, 0x409840, 0xffffffff);
 856                 nv_wr32(priv, 0x409500, 0x00000000);
 857                 nv_wr32(priv, 0x409504, 0x00000025);
 858                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 859                         nv_error(priv, "fuc09 req 0x25 timeout\n");
 860                         return -EBUSY;
 861                 }
 862
 863                 if (nv_device(priv)->chipset >= 0xe0) {
 864                         nv_wr32(priv, 0x409800, 0x00000000);
 865                         nv_wr32(priv, 0x409500, 0x00000001);
 866                         nv_wr32(priv, 0x409504, 0x00000030);
 867                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 868                                 nv_error(priv, "fuc09 req 0x30 timeout\n");
 869                                 return -EBUSY;
 870                         }
 871
 872                         nv_wr32(priv, 0x409810, 0xb00095c8);
 873                         nv_wr32(priv, 0x409800, 0x00000000);
 874                         nv_wr32(priv, 0x409500, 0x00000001);
 875                         nv_wr32(priv, 0x409504, 0x00000031);
 876                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 877                                 nv_error(priv, "fuc09 req 0x31 timeout\n");
 878                                 return -EBUSY;
 879                         }
 880
 881                         nv_wr32(priv, 0x409810, 0x00080420);
 882                         nv_wr32(priv, 0x409800, 0x00000000);
 883                         nv_wr32(priv, 0x409500, 0x00000001);
 884                         nv_wr32(priv, 0x409504, 0x00000032);
 885                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 886                                 nv_error(priv, "fuc09 req 0x32 timeout\n");
 887                                 return -EBUSY;
 888                         }
 889
 890                         nv_wr32(priv, 0x409614, 0x00000070);
 891                         nv_wr32(priv, 0x409614, 0x00000770);
 892                         nv_wr32(priv, 0x40802c, 0x00000001);
 893                 }
 894
 895                 if (priv->data == NULL) {
 896                         int ret = nvc0_grctx_generate(priv);
 897                         if (ret) {
 898                                 nv_error(priv, "failed to construct context\n");
 899                                 return ret;
 900                         }
 901                 }
 902
 903                 return 0;
 904         }
 905
 906         /* load HUB microcode */
 907         r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 908         nv_wr32(priv, 0x4091c0, 0x01000000);
 909         for (i = 0; i < oclass->fecs.ucode->data.size / 4; i++)
 910                 nv_wr32(priv, 0x4091c4, oclass->fecs.ucode->data.data[i]);
 911
 912         nv_wr32(priv, 0x409180, 0x01000000);
 913         for (i = 0; i < oclass->fecs.ucode->code.size / 4; i++) {
 914                 if ((i & 0x3f) == 0)
 915                         nv_wr32(priv, 0x409188, i >> 6);
 916                 nv_wr32(priv, 0x409184, oclass->fecs.ucode->code.data[i]);
 917         }
 918
 919         for (i = 0; (init = cclass->hub[i]); i++) {
 920                 nvc0_graph_init_csdata(priv, init, 0x409000, 0x000, 0x000000);
 921         }
 922
 923         /* load GPC microcode */
 924         nv_wr32(priv, 0x41a1c0, 0x01000000);
 925         for (i = 0; i < oclass->gpccs.ucode->data.size / 4; i++)
 926                 nv_wr32(priv, 0x41a1c4, oclass->gpccs.ucode->data.data[i]);
 927
 928         nv_wr32(priv, 0x41a180, 0x01000000);
 929         for (i = 0; i < oclass->gpccs.ucode->code.size / 4; i++) {
 930                 if ((i & 0x3f) == 0)
 931                         nv_wr32(priv, 0x41a188, i >> 6);
 932                 nv_wr32(priv, 0x41a184, oclass->gpccs.ucode->code.data[i]);
 933         }
 934         nv_wr32(priv, 0x000260, r000260);
 935
 936         if ((init = cclass->gpc[0]))
 937                 nvc0_graph_init_csdata(priv, init, 0x41a000, 0x000, 0x418000);
 938         if ((init = cclass->gpc[2]))
 939                 nvc0_graph_init_csdata(priv, init, 0x41a000, 0x004, 0x419800);
 940         if ((init = cclass->gpc[3]))
 941                 nvc0_graph_init_csdata(priv, init, 0x41a000, 0x008, 0x41be00);
 942
 943         /* start HUB ucode running, it'll init the GPCs */
 944         nv_wr32(priv, 0x40910c, 0x00000000);
 945         nv_wr32(priv, 0x409100, 0x00000002);
 946         if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
 947                 nv_error(priv, "HUB_INIT timed out\n");
 948                 nvc0_graph_ctxctl_debug(priv);
 949                 return -EBUSY;
 950         }
 951
 952         priv->size = nv_rd32(priv, 0x409804);
 953         if (priv->data == NULL) {
 954                 int ret = nvc0_grctx_generate(priv);
 955                 if (ret) {
 956                         nv_error(priv, "failed to construct context\n");
 957                         return ret;
 958                 }
 959         }
 960
 961         return 0;
 962 }
 963
 964 int
 965 nvc0_graph_init(struct nouveau_object *object)
 966 {
 967         struct nvc0_graph_oclass *oclass = (void *)object->oclass;
 968         struct nvc0_graph_priv *priv = (void *)object;
 969         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
 970         u32 data[TPC_MAX / 8] = {};
 971         u8  tpcnr[GPC_MAX];
 972         int gpc, tpc, rop;
 973         int ret, i;
 974
 975         ret = nouveau_graph_init(&priv->base);
 976         if (ret)
 977                 return ret;
 978
 979         nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
 980         nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
 981         nv_wr32(priv, GPC_BCAST(0x0888), 0x00000000);
 982         nv_wr32(priv, GPC_BCAST(0x088c), 0x00000000);
 983         nv_wr32(priv, GPC_BCAST(0x0890), 0x00000000);
 984         nv_wr32(priv, GPC_BCAST(0x0894), 0x00000000);
 985         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
 986         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
 987
 988         for (i = 0; oclass->mmio[i]; i++)
 989                 nvc0_graph_mmio(priv, oclass->mmio[i]);
 990
 991         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
 992         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
 993                 do {
 994                         gpc = (gpc + 1) % priv->gpc_nr;
 995                 } while (!tpcnr[gpc]);
 996                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
 997
 998                 data[i / 8] |= tpc << ((i % 8) * 4);
 999         }
1000
1001         nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
1002         nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
1003         nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
1004         nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
1005
1006         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1007                 nv_wr32(priv, GPC_UNIT(gpc, 0x0914),
1008                         priv->magic_not_rop_nr << 8 | priv->tpc_nr[gpc]);
1009                 nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 |
1010                         priv->tpc_total);
1011                 nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
1012         }
1013
1014         if (nv_device(priv)->chipset != 0xd7)
1015                 nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
1016         else
1017                 nv_wr32(priv, GPC_BCAST(0x3fd4), magicgpc918);
1018
1019         nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
1020
1021         nv_wr32(priv, 0x400500, 0x00010001);
1022
1023         nv_wr32(priv, 0x400100, 0xffffffff);
1024         nv_wr32(priv, 0x40013c, 0xffffffff);
1025
1026         nv_wr32(priv, 0x409c24, 0x000f0000);
1027         nv_wr32(priv, 0x404000, 0xc0000000);
1028         nv_wr32(priv, 0x404600, 0xc0000000);
1029         nv_wr32(priv, 0x408030, 0xc0000000);
1030         nv_wr32(priv, 0x40601c, 0xc0000000);
1031         nv_wr32(priv, 0x404490, 0xc0000000);
1032         nv_wr32(priv, 0x406018, 0xc0000000);
1033         nv_wr32(priv, 0x405840, 0xc0000000);
1034         nv_wr32(priv, 0x405844, 0x00ffffff);
1035         nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
1036         nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
1037
1038         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1039                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
1040                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
1041                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
1042                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
1043                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
1044                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
1045                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
1046                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
1047                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
1048                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
1049                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
1050                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
1051                 }
1052                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
1053                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
1054         }
1055
1056         for (rop = 0; rop < priv->rop_nr; rop++) {
1057                 nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
1058                 nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
1059                 nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
1060                 nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
1061         }
1062
1063         nv_wr32(priv, 0x400108, 0xffffffff);
1064         nv_wr32(priv, 0x400138, 0xffffffff);
1065         nv_wr32(priv, 0x400118, 0xffffffff);
1066         nv_wr32(priv, 0x400130, 0xffffffff);
1067         nv_wr32(priv, 0x40011c, 0xffffffff);
1068         nv_wr32(priv, 0x400134, 0xffffffff);
1069
1070         nv_wr32(priv, 0x400054, 0x34ce3464);
1071         return nvc0_graph_init_ctxctl(priv);
1072 }
1073
1074 static void
1075 nvc0_graph_dtor_fw(struct nvc0_graph_fuc *fuc)
1076 {
1077         kfree(fuc->data);
1078         fuc->data = NULL;
1079 }
1080
1081 int
1082 nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
1083                    struct nvc0_graph_fuc *fuc)
1084 {
1085         struct nouveau_device *device = nv_device(priv);
1086         const struct firmware *fw;
1087         char f[32];
1088         int ret;
1089
1090         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
1091         ret = request_firmware(&fw, f, &device->pdev->dev);
1092         if (ret) {
1093                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
1094                 ret = request_firmware(&fw, f, &device->pdev->dev);
1095                 if (ret) {
1096                         nv_error(priv, "failed to load %s\n", fwname);
1097                         return ret;
1098                 }
1099         }
1100
1101         fuc->size = fw->size;
1102         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
1103         release_firmware(fw);
1104         return (fuc->data != NULL) ? 0 : -ENOMEM;
1105 }
1106
1107 void
1108 nvc0_graph_dtor(struct nouveau_object *object)
1109 {
1110         struct nvc0_graph_priv *priv = (void *)object;
1111
1112         kfree(priv->data);
1113
1114         nvc0_graph_dtor_fw(&priv->fuc409c);
1115         nvc0_graph_dtor_fw(&priv->fuc409d);
1116         nvc0_graph_dtor_fw(&priv->fuc41ac);
1117         nvc0_graph_dtor_fw(&priv->fuc41ad);
1118
1119         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
1120         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
1121
1122         nouveau_graph_destroy(&priv->base);
1123 }
1124
1125 int
1126 nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
1127                 struct nouveau_oclass *bclass, void *data, u32 size,
1128                 struct nouveau_object **pobject)
1129 {
1130         struct nvc0_graph_oclass *oclass = (void *)bclass;
1131         struct nouveau_device *device = nv_device(parent);
1132         struct nvc0_graph_priv *priv;
1133         int ret, i;
1134
1135         ret = nouveau_graph_create(parent, engine, bclass,
1136                                    (oclass->fecs.ucode != NULL), &priv);
1137         *pobject = nv_object(priv);
1138         if (ret)
1139                 return ret;
1140
1141         nv_subdev(priv)->unit = 0x18001000;
1142         nv_subdev(priv)->intr = nvc0_graph_intr;
1143
1144         priv->base.units = nvc0_graph_units;
1145
1146         if (nouveau_boolopt(device->cfgopt, "NvGrUseFW", false)) {
1147                 nv_info(priv, "using external firmware\n");
1148                 if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
1149                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
1150                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
1151                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
1152                         return -EINVAL;
1153                 priv->firmware = true;
1154         }
1155
1156         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
1157                                 &priv->unk4188b4);
1158         if (ret)
1159                 return ret;
1160
1161         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
1162                                 &priv->unk4188b8);
1163         if (ret)
1164                 return ret;
1165
1166         for (i = 0; i < 0x1000; i += 4) {
1167                 nv_wo32(priv->unk4188b4, i, 0x00000010);
1168                 nv_wo32(priv->unk4188b8, i, 0x00000010);
1169         }
1170
1171         priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
1172         priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
1173         for (i = 0; i < priv->gpc_nr; i++) {
1174                 priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
1175                 priv->tpc_total += priv->tpc_nr[i];
1176         }
1177
1178         /*XXX: these need figuring out... though it might not even matter */
1179         switch (nv_device(priv)->chipset) {
1180         case 0xc0:
1181                 if (priv->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
1182                         priv->magic_not_rop_nr = 0x07;
1183                 } else
1184                 if (priv->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
1185                         priv->magic_not_rop_nr = 0x05;
1186                 } else
1187                 if (priv->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
1188                         priv->magic_not_rop_nr = 0x06;
1189                 }
1190                 break;
1191         case 0xc3: /* 450, 4/0/0/0, 2 */
1192                 priv->magic_not_rop_nr = 0x03;
1193                 break;
1194         case 0xc4: /* 460, 3/4/0/0, 4 */
1195                 priv->magic_not_rop_nr = 0x01;
1196                 break;
1197         case 0xc1: /* 2/0/0/0, 1 */
1198                 priv->magic_not_rop_nr = 0x01;
1199                 break;
1200         case 0xc8: /* 4/4/3/4, 5 */
1201                 priv->magic_not_rop_nr = 0x06;
1202                 break;
1203         case 0xce: /* 4/4/0/0, 4 */
1204                 priv->magic_not_rop_nr = 0x03;
1205                 break;
1206         case 0xcf: /* 4/0/0/0, 3 */
1207                 priv->magic_not_rop_nr = 0x03;
1208                 break;
1209         case 0xd7:
1210         case 0xd9: /* 1/0/0/0, 1 */
1211                 priv->magic_not_rop_nr = 0x01;
1212                 break;
1213         }
1214
1215         nv_engine(priv)->cclass = *oclass->cclass;
1216         nv_engine(priv)->sclass =  oclass->sclass;
1217         return 0;
1218 }
1219
1220 struct nvc0_graph_init *
1221 nvc0_graph_init_mmio[] = {
1222         nvc0_graph_init_regs,
1223         nvc0_graph_init_unk40xx,
1224         nvc0_graph_init_unk44xx,
1225         nvc0_graph_init_unk78xx,
1226         nvc0_graph_init_unk60xx,
1227         nvc0_graph_init_unk58xx,
1228         nvc0_graph_init_unk80xx,
1229         nvc0_graph_init_gpc,
1230         nvc0_graph_init_tpc,
1231         nvc0_graph_init_unk88xx,
1232         nvc0_graph_tpc_0,
1233         NULL
1234 };
1235
1236 #include "fuc/hubnvc0.fuc.h"
1237
1238 struct nvc0_graph_ucode
1239 nvc0_graph_fecs_ucode = {
1240         .code.data = nvc0_grhub_code,
1241         .code.size = sizeof(nvc0_grhub_code),
1242         .data.data = nvc0_grhub_data,
1243         .data.size = sizeof(nvc0_grhub_data),
1244 };
1245
1246 #include "fuc/gpcnvc0.fuc.h"
1247
1248 struct nvc0_graph_ucode
1249 nvc0_graph_gpccs_ucode = {
1250         .code.data = nvc0_grgpc_code,
1251         .code.size = sizeof(nvc0_grgpc_code),
1252         .data.data = nvc0_grgpc_data,
1253         .data.size = sizeof(nvc0_grgpc_data),
1254 };
1255
1256 struct nouveau_oclass *
1257 nvc0_graph_oclass = &(struct nvc0_graph_oclass) {
1258         .base.handle = NV_ENGINE(GR, 0xc0),
1259         .base.ofuncs = &(struct nouveau_ofuncs) {
1260                 .ctor = nvc0_graph_ctor,
1261                 .dtor = nvc0_graph_dtor,
1262                 .init = nvc0_graph_init,
1263                 .fini = _nouveau_graph_fini,
1264         },
1265         .cclass = &nvc0_grctx_oclass,
1266         .sclass =  nvc0_graph_sclass,
1267         .mmio = nvc0_graph_init_mmio,
1268         .fecs.ucode = &nvc0_graph_fecs_ucode,
1269         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
1270 }.base;