drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include "nvc0.h"
  26 #include "ctxnvc0.h"
  27
  28 /*******************************************************************************
  29  * Graphics object classes
  30  ******************************************************************************/
  31
  32 struct nouveau_oclass
  33 nvc0_graph_sclass[] = {
  34         { 0x902d, &nouveau_object_ofuncs },
  35         { 0x9039, &nouveau_object_ofuncs },
  36         { 0x9097, &nouveau_object_ofuncs },
  37         { 0x90c0, &nouveau_object_ofuncs },
  38         {}
  39 };
  40
  41 /*******************************************************************************
  42  * PGRAPH context
  43  ******************************************************************************/
  44
  45 int
  46 nvc0_graph_context_ctor(struct nouveau_object *parent,
  47                         struct nouveau_object *engine,
  48                         struct nouveau_oclass *oclass, void *args, u32 size,
  49                         struct nouveau_object **pobject)
  50 {
  51         struct nouveau_vm *vm = nouveau_client(parent)->vm;
  52         struct nvc0_graph_priv *priv = (void *)engine;
  53         struct nvc0_graph_data *data = priv->mmio_data;
  54         struct nvc0_graph_mmio *mmio = priv->mmio_list;
  55         struct nvc0_graph_chan *chan;
  56         int ret, i;
  57
  58         /* allocate memory for context, and fill with default values */
  59         ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
  60                                            priv->size, 0x100,
  61                                            NVOBJ_FLAG_ZERO_ALLOC, &chan);
  62         *pobject = nv_object(chan);
  63         if (ret)
  64                 return ret;
  65
  66         /* allocate memory for a "mmio list" buffer that's used by the HUB
  67          * fuc to modify some per-context register settings on first load
  68          * of the context.
  69          */
  70         ret = nouveau_gpuobj_new(nv_object(chan), NULL, 0x1000, 0x100, 0,
  71                                 &chan->mmio);
  72         if (ret)
  73                 return ret;
  74
  75         ret = nouveau_gpuobj_map_vm(nv_gpuobj(chan->mmio), vm,
  76                                     NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
  77                                     &chan->mmio_vma);
  78         if (ret)
  79                 return ret;
  80
  81         /* allocate buffers referenced by mmio list */
  82         for (i = 0; data->size && i < ARRAY_SIZE(priv->mmio_data); i++) {
  83                 ret = nouveau_gpuobj_new(nv_object(chan), NULL, data->size,
  84                                          data->align, 0, &chan->data[i].mem);
  85                 if (ret)
  86                         return ret;
  87
  88                 ret = nouveau_gpuobj_map_vm(chan->data[i].mem, vm, data->access,
  89                                            &chan->data[i].vma);
  90                 if (ret)
  91                         return ret;
  92
  93                 data++;
  94         }
  95
  96         /* finally, fill in the mmio list and point the context at it */
  97         for (i = 0; mmio->addr && i < ARRAY_SIZE(priv->mmio_list); i++) {
  98                 u32 addr = mmio->addr;
  99                 u32 data = mmio->data;
 100
 101                 if (mmio->shift) {
 102                         u64 info = chan->data[mmio->buffer].vma.offset;
 103                         data |= info >> mmio->shift;
 104                 }
 105
 106                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
 107                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
 108                 mmio++;
 109         }
 110
 111         for (i = 0; i < priv->size; i += 4)
 112                 nv_wo32(chan, i, priv->data[i / 4]);
 113
 114         if (!priv->firmware) {
 115                 nv_wo32(chan, 0x00, chan->mmio_nr / 2);
 116                 nv_wo32(chan, 0x04, chan->mmio_vma.offset >> 8);
 117         } else {
 118                 nv_wo32(chan, 0xf4, 0);
 119                 nv_wo32(chan, 0xf8, 0);
 120                 nv_wo32(chan, 0x10, chan->mmio_nr / 2);
 121                 nv_wo32(chan, 0x14, lower_32_bits(chan->mmio_vma.offset));
 122                 nv_wo32(chan, 0x18, upper_32_bits(chan->mmio_vma.offset));
 123                 nv_wo32(chan, 0x1c, 1);
 124                 nv_wo32(chan, 0x20, 0);
 125                 nv_wo32(chan, 0x28, 0);
 126                 nv_wo32(chan, 0x2c, 0);
 127         }
 128
 129         return 0;
 130 }
 131
 132 void
 133 nvc0_graph_context_dtor(struct nouveau_object *object)
 134 {
 135         struct nvc0_graph_chan *chan = (void *)object;
 136         int i;
 137
 138         for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
 139                 nouveau_gpuobj_unmap(&chan->data[i].vma);
 140                 nouveau_gpuobj_ref(NULL, &chan->data[i].mem);
 141         }
 142
 143         nouveau_gpuobj_unmap(&chan->mmio_vma);
 144         nouveau_gpuobj_ref(NULL, &chan->mmio);
 145
 146         nouveau_graph_context_destroy(&chan->base);
 147 }
 148
 149 /*******************************************************************************
 150  * PGRAPH register lists
 151  ******************************************************************************/
 152
 153 const struct nvc0_graph_init
 154 nvc0_graph_init_main_0[] = {
 155         { 0x400080,   1, 0x04, 0x003083c2 },
 156         { 0x400088,   1, 0x04, 0x00006fe7 },
 157         { 0x40008c,   1, 0x04, 0x00000000 },
 158         { 0x400090,   1, 0x04, 0x00000030 },
 159         { 0x40013c,   1, 0x04, 0x013901f7 },
 160         { 0x400140,   1, 0x04, 0x00000100 },
 161         { 0x400144,   1, 0x04, 0x00000000 },
 162         { 0x400148,   1, 0x04, 0x00000110 },
 163         { 0x400138,   1, 0x04, 0x00000000 },
 164         { 0x400130,   2, 0x04, 0x00000000 },
 165         { 0x400124,   1, 0x04, 0x00000002 },
 166         {}
 167 };
 168
 169 const struct nvc0_graph_init
 170 nvc0_graph_init_fe_0[] = {
 171         { 0x40415c,   1, 0x04, 0x00000000 },
 172         { 0x404170,   1, 0x04, 0x00000000 },
 173         {}
 174 };
 175
 176 const struct nvc0_graph_init
 177 nvc0_graph_init_pri_0[] = {
 178         { 0x404488,   2, 0x04, 0x00000000 },
 179         {}
 180 };
 181
 182 const struct nvc0_graph_init
 183 nvc0_graph_init_rstr2d_0[] = {
 184         { 0x407808,   1, 0x04, 0x00000000 },
 185         {}
 186 };
 187
 188 const struct nvc0_graph_init
 189 nvc0_graph_init_pd_0[] = {
 190         { 0x406024,   1, 0x04, 0x00000000 },
 191         {}
 192 };
 193
 194 const struct nvc0_graph_init
 195 nvc0_graph_init_ds_0[] = {
 196         { 0x405844,   1, 0x04, 0x00ffffff },
 197         { 0x405850,   1, 0x04, 0x00000000 },
 198         { 0x405908,   1, 0x04, 0x00000000 },
 199         {}
 200 };
 201
 202 const struct nvc0_graph_init
 203 nvc0_graph_init_scc_0[] = {
 204         { 0x40803c,   1, 0x04, 0x00000000 },
 205         {}
 206 };
 207
 208 const struct nvc0_graph_init
 209 nvc0_graph_init_prop_0[] = {
 210         { 0x4184a0,   1, 0x04, 0x00000000 },
 211         {}
 212 };
 213
 214 const struct nvc0_graph_init
 215 nvc0_graph_init_gpc_unk_0[] = {
 216         { 0x418604,   1, 0x04, 0x00000000 },
 217         { 0x418680,   1, 0x04, 0x00000000 },
 218         { 0x418714,   1, 0x04, 0x80000000 },
 219         { 0x418384,   1, 0x04, 0x00000000 },
 220         {}
 221 };
 222
 223 const struct nvc0_graph_init
 224 nvc0_graph_init_setup_0[] = {
 225         { 0x418814,   3, 0x04, 0x00000000 },
 226         {}
 227 };
 228
 229 const struct nvc0_graph_init
 230 nvc0_graph_init_crstr_0[] = {
 231         { 0x418b04,   1, 0x04, 0x00000000 },
 232         {}
 233 };
 234
 235 const struct nvc0_graph_init
 236 nvc0_graph_init_setup_1[] = {
 237         { 0x4188c8,   1, 0x04, 0x80000000 },
 238         { 0x4188cc,   1, 0x04, 0x00000000 },
 239         { 0x4188d0,   1, 0x04, 0x00010000 },
 240         { 0x4188d4,   1, 0x04, 0x00000001 },
 241         {}
 242 };
 243
 244 const struct nvc0_graph_init
 245 nvc0_graph_init_zcull_0[] = {
 246         { 0x418910,   1, 0x04, 0x00010001 },
 247         { 0x418914,   1, 0x04, 0x00000301 },
 248         { 0x418918,   1, 0x04, 0x00800000 },
 249         { 0x418980,   1, 0x04, 0x77777770 },
 250         { 0x418984,   3, 0x04, 0x77777777 },
 251         {}
 252 };
 253
 254 const struct nvc0_graph_init
 255 nvc0_graph_init_gpm_0[] = {
 256         { 0x418c04,   1, 0x04, 0x00000000 },
 257         { 0x418c88,   1, 0x04, 0x00000000 },
 258         {}
 259 };
 260
 261 const struct nvc0_graph_init
 262 nvc0_graph_init_gpc_unk_1[] = {
 263         { 0x418d00,   1, 0x04, 0x00000000 },
 264         { 0x418f08,   1, 0x04, 0x00000000 },
 265         { 0x418e00,   1, 0x04, 0x00000050 },
 266         { 0x418e08,   1, 0x04, 0x00000000 },
 267         {}
 268 };
 269
 270 const struct nvc0_graph_init
 271 nvc0_graph_init_gcc_0[] = {
 272         { 0x41900c,   1, 0x04, 0x00000000 },
 273         { 0x419018,   1, 0x04, 0x00000000 },
 274         {}
 275 };
 276
 277 const struct nvc0_graph_init
 278 nvc0_graph_init_tpccs_0[] = {
 279         { 0x419d08,   2, 0x04, 0x00000000 },
 280         { 0x419d10,   1, 0x04, 0x00000014 },
 281         {}
 282 };
 283
 284 const struct nvc0_graph_init
 285 nvc0_graph_init_tex_0[] = {
 286         { 0x419ab0,   1, 0x04, 0x00000000 },
 287         { 0x419ab8,   1, 0x04, 0x000000e7 },
 288         { 0x419abc,   2, 0x04, 0x00000000 },
 289         {}
 290 };
 291
 292 const struct nvc0_graph_init
 293 nvc0_graph_init_pe_0[] = {
 294         { 0x41980c,   3, 0x04, 0x00000000 },
 295         { 0x419844,   1, 0x04, 0x00000000 },
 296         { 0x41984c,   1, 0x04, 0x00005bc5 },
 297         { 0x419850,   4, 0x04, 0x00000000 },
 298         {}
 299 };
 300
 301 const struct nvc0_graph_init
 302 nvc0_graph_init_l1c_0[] = {
 303         { 0x419c98,   1, 0x04, 0x00000000 },
 304         { 0x419ca8,   1, 0x04, 0x80000000 },
 305         { 0x419cb4,   1, 0x04, 0x00000000 },
 306         { 0x419cb8,   1, 0x04, 0x00008bf4 },
 307         { 0x419cbc,   1, 0x04, 0x28137606 },
 308         { 0x419cc0,   2, 0x04, 0x00000000 },
 309         {}
 310 };
 311
 312 const struct nvc0_graph_init
 313 nvc0_graph_init_wwdx_0[] = {
 314         { 0x419bd4,   1, 0x04, 0x00800000 },
 315         { 0x419bdc,   1, 0x04, 0x00000000 },
 316         {}
 317 };
 318
 319 const struct nvc0_graph_init
 320 nvc0_graph_init_tpccs_1[] = {
 321         { 0x419d2c,   1, 0x04, 0x00000000 },
 322         {}
 323 };
 324
 325 const struct nvc0_graph_init
 326 nvc0_graph_init_mpc_0[] = {
 327         { 0x419c0c,   1, 0x04, 0x00000000 },
 328         {}
 329 };
 330
 331 static const struct nvc0_graph_init
 332 nvc0_graph_init_sm_0[] = {
 333         { 0x419e00,   1, 0x04, 0x00000000 },
 334         { 0x419ea0,   1, 0x04, 0x00000000 },
 335         { 0x419ea4,   1, 0x04, 0x00000100 },
 336         { 0x419ea8,   1, 0x04, 0x00001100 },
 337         { 0x419eac,   1, 0x04, 0x11100702 },
 338         { 0x419eb0,   1, 0x04, 0x00000003 },
 339         { 0x419eb4,   4, 0x04, 0x00000000 },
 340         { 0x419ec8,   1, 0x04, 0x06060618 },
 341         { 0x419ed0,   1, 0x04, 0x0eff0e38 },
 342         { 0x419ed4,   1, 0x04, 0x011104f1 },
 343         { 0x419edc,   1, 0x04, 0x00000000 },
 344         { 0x419f00,   1, 0x04, 0x00000000 },
 345         { 0x419f2c,   1, 0x04, 0x00000000 },
 346         {}
 347 };
 348
 349 const struct nvc0_graph_init
 350 nvc0_graph_init_be_0[] = {
 351         { 0x40880c,   1, 0x04, 0x00000000 },
 352         { 0x408910,   9, 0x04, 0x00000000 },
 353         { 0x408950,   1, 0x04, 0x00000000 },
 354         { 0x408954,   1, 0x04, 0x0000ffff },
 355         { 0x408984,   1, 0x04, 0x00000000 },
 356         { 0x408988,   1, 0x04, 0x08040201 },
 357         { 0x40898c,   1, 0x04, 0x80402010 },
 358         {}
 359 };
 360
 361 const struct nvc0_graph_init
 362 nvc0_graph_init_fe_1[] = {
 363         { 0x4040f0,   1, 0x04, 0x00000000 },
 364         {}
 365 };
 366
 367 const struct nvc0_graph_init
 368 nvc0_graph_init_pe_1[] = {
 369         { 0x419880,   1, 0x04, 0x00000002 },
 370         {}
 371 };
 372
 373 static const struct nvc0_graph_pack
 374 nvc0_graph_pack_mmio[] = {
 375         { nvc0_graph_init_main_0 },
 376         { nvc0_graph_init_fe_0 },
 377         { nvc0_graph_init_pri_0 },
 378         { nvc0_graph_init_rstr2d_0 },
 379         { nvc0_graph_init_pd_0 },
 380         { nvc0_graph_init_ds_0 },
 381         { nvc0_graph_init_scc_0 },
 382         { nvc0_graph_init_prop_0 },
 383         { nvc0_graph_init_gpc_unk_0 },
 384         { nvc0_graph_init_setup_0 },
 385         { nvc0_graph_init_crstr_0 },
 386         { nvc0_graph_init_setup_1 },
 387         { nvc0_graph_init_zcull_0 },
 388         { nvc0_graph_init_gpm_0 },
 389         { nvc0_graph_init_gpc_unk_1 },
 390         { nvc0_graph_init_gcc_0 },
 391         { nvc0_graph_init_tpccs_0 },
 392         { nvc0_graph_init_tex_0 },
 393         { nvc0_graph_init_pe_0 },
 394         { nvc0_graph_init_l1c_0 },
 395         { nvc0_graph_init_wwdx_0 },
 396         { nvc0_graph_init_tpccs_1 },
 397         { nvc0_graph_init_mpc_0 },
 398         { nvc0_graph_init_sm_0 },
 399         { nvc0_graph_init_be_0 },
 400         { nvc0_graph_init_fe_1 },
 401         { nvc0_graph_init_pe_1 },
 402         {}
 403 };
 404
 405 /*******************************************************************************
 406  * PGRAPH engine/subdev functions
 407  ******************************************************************************/
 408
 409 void
 410 nvc0_graph_mmio(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 411 {
 412         const struct nvc0_graph_pack *pack;
 413         const struct nvc0_graph_init *init;
 414
 415         pack_for_each_init(init, pack, p) {
 416                 u32 next = init->addr + init->count * init->pitch;
 417                 u32 addr = init->addr;
 418                 while (addr < next) {
 419                         nv_wr32(priv, addr, init->data);
 420                         addr += init->pitch;
 421                 }
 422         }
 423 }
 424
 425 void
 426 nvc0_graph_icmd(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 427 {
 428         const struct nvc0_graph_pack *pack;
 429         const struct nvc0_graph_init *init;
 430         u32 data = 0;
 431
 432         nv_wr32(priv, 0x400208, 0x80000000);
 433
 434         pack_for_each_init(init, pack, p) {
 435                 u32 next = init->addr + init->count * init->pitch;
 436                 u32 addr = init->addr;
 437
 438                 if ((pack == p && init == p->init) || data != init->data) {
 439                         nv_wr32(priv, 0x400204, init->data);
 440                         data = init->data;
 441                 }
 442
 443                 while (addr < next) {
 444                         nv_wr32(priv, 0x400200, addr);
 445                         nv_wait(priv, 0x400700, 0x00000002, 0x00000000);
 446                         addr += init->pitch;
 447                 }
 448         }
 449
 450         nv_wr32(priv, 0x400208, 0x00000000);
 451 }
 452
 453 void
 454 nvc0_graph_mthd(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 455 {
 456         const struct nvc0_graph_pack *pack;
 457         const struct nvc0_graph_init *init;
 458         u32 data = 0;
 459
 460         pack_for_each_init(init, pack, p) {
 461                 u32 ctrl = 0x80000000 | pack->type;
 462                 u32 next = init->addr + init->count * init->pitch;
 463                 u32 addr = init->addr;
 464
 465                 if ((pack == p && init == p->init) || data != init->data) {
 466                         nv_wr32(priv, 0x40448c, init->data);
 467                         data = init->data;
 468                 }
 469
 470                 while (addr < next) {
 471                         nv_wr32(priv, 0x404488, ctrl | (addr << 14));
 472                         addr += init->pitch;
 473                 }
 474         }
 475 }
 476
 477 u64
 478 nvc0_graph_units(struct nouveau_graph *graph)
 479 {
 480         struct nvc0_graph_priv *priv = (void *)graph;
 481         u64 cfg;
 482
 483         cfg  = (u32)priv->gpc_nr;
 484         cfg |= (u32)priv->tpc_total << 8;
 485         cfg |= (u64)priv->rop_nr << 32;
 486
 487         return cfg;
 488 }
 489
 490 static const struct nouveau_enum nve0_sked_error[] = {
 491         { 7, "CONSTANT_BUFFER_SIZE" },
 492         { 9, "LOCAL_MEMORY_SIZE_POS" },
 493         { 10, "LOCAL_MEMORY_SIZE_NEG" },
 494         { 11, "WARP_CSTACK_SIZE" },
 495         { 12, "TOTAL_TEMP_SIZE" },
 496         { 13, "REGISTER_COUNT" },
 497         { 18, "TOTAL_THREADS" },
 498         { 20, "PROGRAM_OFFSET" },
 499         { 21, "SHARED_MEMORY_SIZE" },
 500         { 25, "SHARED_CONFIG_TOO_SMALL" },
 501         { 26, "TOTAL_REGISTER_COUNT" },
 502         {}
 503 };
 504
 505 static const struct nouveau_enum nvc0_gpc_rop_error[] = {
 506         { 1, "RT_PITCH_OVERRUN" },
 507         { 4, "RT_WIDTH_OVERRUN" },
 508         { 5, "RT_HEIGHT_OVERRUN" },
 509         { 7, "ZETA_STORAGE_TYPE_MISMATCH" },
 510         { 8, "RT_STORAGE_TYPE_MISMATCH" },
 511         { 10, "RT_LINEAR_MISMATCH" },
 512         {}
 513 };
 514
 515 static void
 516 nvc0_graph_trap_gpc_rop(struct nvc0_graph_priv *priv, int gpc)
 517 {
 518         u32 trap[4];
 519         int i;
 520
 521         trap[0] = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
 522         trap[1] = nv_rd32(priv, GPC_UNIT(gpc, 0x0434));
 523         trap[2] = nv_rd32(priv, GPC_UNIT(gpc, 0x0438));
 524         trap[3] = nv_rd32(priv, GPC_UNIT(gpc, 0x043c));
 525
 526         nv_error(priv, "GPC%d/PROP trap:", gpc);
 527         for (i = 0; i <= 29; ++i) {
 528                 if (!(trap[0] & (1 << i)))
 529                         continue;
 530                 pr_cont(" ");
 531                 nouveau_enum_print(nvc0_gpc_rop_error, i);
 532         }
 533         pr_cont("\n");
 534
 535         nv_error(priv, "x = %u, y = %u, format = %x, storage type = %x\n",
 536                  trap[1] & 0xffff, trap[1] >> 16, (trap[2] >> 8) & 0x3f,
 537                  trap[3] & 0xff);
 538         nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 539 }
 540
 541 static const struct nouveau_enum nvc0_mp_warp_error[] = {
 542         { 0x00, "NO_ERROR" },
 543         { 0x01, "STACK_MISMATCH" },
 544         { 0x05, "MISALIGNED_PC" },
 545         { 0x08, "MISALIGNED_GPR" },
 546         { 0x09, "INVALID_OPCODE" },
 547         { 0x0d, "GPR_OUT_OF_BOUNDS" },
 548         { 0x0e, "MEM_OUT_OF_BOUNDS" },
 549         { 0x0f, "UNALIGNED_MEM_ACCESS" },
 550         { 0x11, "INVALID_PARAM" },
 551         {}
 552 };
 553
 554 static const struct nouveau_bitfield nvc0_mp_global_error[] = {
 555         { 0x00000004, "MULTIPLE_WARP_ERRORS" },
 556         { 0x00000008, "OUT_OF_STACK_SPACE" },
 557         {}
 558 };
 559
 560 static void
 561 nvc0_graph_trap_mp(struct nvc0_graph_priv *priv, int gpc, int tpc)
 562 {
 563         u32 werr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x648));
 564         u32 gerr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x650));
 565
 566         nv_error(priv, "GPC%i/TPC%i/MP trap:", gpc, tpc);
 567         nouveau_bitfield_print(nvc0_mp_global_error, gerr);
 568         if (werr) {
 569                 pr_cont(" ");
 570                 nouveau_enum_print(nvc0_mp_warp_error, werr & 0xffff);
 571         }
 572         pr_cont("\n");
 573
 574         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);
 575         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x650), gerr);
 576 }
 577
 578 static void
 579 nvc0_graph_trap_tpc(struct nvc0_graph_priv *priv, int gpc, int tpc)
 580 {
 581         u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0508));
 582
 583         if (stat & 0x00000001) {
 584                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0224));
 585                 nv_error(priv, "GPC%d/TPC%d/TEX: 0x%08x\n", gpc, tpc, trap);
 586                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
 587                 stat &= ~0x00000001;
 588         }
 589
 590         if (stat & 0x00000002) {
 591                 nvc0_graph_trap_mp(priv, gpc, tpc);
 592                 stat &= ~0x00000002;
 593         }
 594
 595         if (stat & 0x00000004) {
 596                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0084));
 597                 nv_error(priv, "GPC%d/TPC%d/POLY: 0x%08x\n", gpc, tpc, trap);
 598                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
 599                 stat &= ~0x00000004;
 600         }
 601
 602         if (stat & 0x00000008) {
 603                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x048c));
 604                 nv_error(priv, "GPC%d/TPC%d/L1C: 0x%08x\n", gpc, tpc, trap);
 605                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
 606                 stat &= ~0x00000008;
 607         }
 608
 609         if (stat) {
 610                 nv_error(priv, "GPC%d/TPC%d/0x%08x: unknown\n", gpc, tpc, stat);
 611         }
 612 }
 613
 614 static void
 615 nvc0_graph_trap_gpc(struct nvc0_graph_priv *priv, int gpc)
 616 {
 617         u32 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
 618         int tpc;
 619
 620         if (stat & 0x00000001) {
 621                 nvc0_graph_trap_gpc_rop(priv, gpc);
 622                 stat &= ~0x00000001;
 623         }
 624
 625         if (stat & 0x00000002) {
 626                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
 627                 nv_error(priv, "GPC%d/ZCULL: 0x%08x\n", gpc, trap);
 628                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 629                 stat &= ~0x00000002;
 630         }
 631
 632         if (stat & 0x00000004) {
 633                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
 634                 nv_error(priv, "GPC%d/CCACHE: 0x%08x\n", gpc, trap);
 635                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 636                 stat &= ~0x00000004;
 637         }
 638
 639         if (stat & 0x00000008) {
 640                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
 641                 nv_error(priv, "GPC%d/ESETUP: 0x%08x\n", gpc, trap);
 642                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 643                 stat &= ~0x00000009;
 644         }
 645
 646         for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
 647                 u32 mask = 0x00010000 << tpc;
 648                 if (stat & mask) {
 649                         nvc0_graph_trap_tpc(priv, gpc, tpc);
 650                         nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), mask);
 651                         stat &= ~mask;
 652                 }
 653         }
 654
 655         if (stat) {
 656                 nv_error(priv, "GPC%d/0x%08x: unknown\n", gpc, stat);
 657         }
 658 }
 659
 660 static void
 661 nvc0_graph_trap_intr(struct nvc0_graph_priv *priv)
 662 {
 663         u32 trap = nv_rd32(priv, 0x400108);
 664         int rop, gpc, i;
 665
 666         if (trap & 0x00000001) {
 667                 u32 stat = nv_rd32(priv, 0x404000);
 668                 nv_error(priv, "DISPATCH 0x%08x\n", stat);
 669                 nv_wr32(priv, 0x404000, 0xc0000000);
 670                 nv_wr32(priv, 0x400108, 0x00000001);
 671                 trap &= ~0x00000001;
 672         }
 673
 674         if (trap & 0x00000002) {
 675                 u32 stat = nv_rd32(priv, 0x404600);
 676                 nv_error(priv, "M2MF 0x%08x\n", stat);
 677                 nv_wr32(priv, 0x404600, 0xc0000000);
 678                 nv_wr32(priv, 0x400108, 0x00000002);
 679                 trap &= ~0x00000002;
 680         }
 681
 682         if (trap & 0x00000008) {
 683                 u32 stat = nv_rd32(priv, 0x408030);
 684                 nv_error(priv, "CCACHE 0x%08x\n", stat);
 685                 nv_wr32(priv, 0x408030, 0xc0000000);
 686                 nv_wr32(priv, 0x400108, 0x00000008);
 687                 trap &= ~0x00000008;
 688         }
 689
 690         if (trap & 0x00000010) {
 691                 u32 stat = nv_rd32(priv, 0x405840);
 692                 nv_error(priv, "SHADER 0x%08x\n", stat);
 693                 nv_wr32(priv, 0x405840, 0xc0000000);
 694                 nv_wr32(priv, 0x400108, 0x00000010);
 695                 trap &= ~0x00000010;
 696         }
 697
 698         if (trap & 0x00000040) {
 699                 u32 stat = nv_rd32(priv, 0x40601c);
 700                 nv_error(priv, "UNK6 0x%08x\n", stat);
 701                 nv_wr32(priv, 0x40601c, 0xc0000000);
 702                 nv_wr32(priv, 0x400108, 0x00000040);
 703                 trap &= ~0x00000040;
 704         }
 705
 706         if (trap & 0x00000080) {
 707                 u32 stat = nv_rd32(priv, 0x404490);
 708                 nv_error(priv, "MACRO 0x%08x\n", stat);
 709                 nv_wr32(priv, 0x404490, 0xc0000000);
 710                 nv_wr32(priv, 0x400108, 0x00000080);
 711                 trap &= ~0x00000080;
 712         }
 713
 714         if (trap & 0x00000100) {
 715                 u32 stat = nv_rd32(priv, 0x407020);
 716
 717                 nv_error(priv, "SKED:");
 718                 for (i = 0; i <= 29; ++i) {
 719                         if (!(stat & (1 << i)))
 720                                 continue;
 721                         pr_cont(" ");
 722                         nouveau_enum_print(nve0_sked_error, i);
 723                 }
 724                 pr_cont("\n");
 725
 726                 if (stat & 0x3fffffff)
 727                         nv_wr32(priv, 0x407020, 0x40000000);
 728                 nv_wr32(priv, 0x400108, 0x00000100);
 729                 trap &= ~0x00000100;
 730         }
 731
 732         if (trap & 0x01000000) {
 733                 u32 stat = nv_rd32(priv, 0x400118);
 734                 for (gpc = 0; stat && gpc < priv->gpc_nr; gpc++) {
 735                         u32 mask = 0x00000001 << gpc;
 736                         if (stat & mask) {
 737                                 nvc0_graph_trap_gpc(priv, gpc);
 738                                 nv_wr32(priv, 0x400118, mask);
 739                                 stat &= ~mask;
 740                         }
 741                 }
 742                 nv_wr32(priv, 0x400108, 0x01000000);
 743                 trap &= ~0x01000000;
 744         }
 745
 746         if (trap & 0x02000000) {
 747                 for (rop = 0; rop < priv->rop_nr; rop++) {
 748                         u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
 749                         u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
 750                         nv_error(priv, "ROP%d 0x%08x 0x%08x\n",
 751                                  rop, statz, statc);
 752                         nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
 753                         nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
 754                 }
 755                 nv_wr32(priv, 0x400108, 0x02000000);
 756                 trap &= ~0x02000000;
 757         }
 758
 759         if (trap) {
 760                 nv_error(priv, "TRAP UNHANDLED 0x%08x\n", trap);
 761                 nv_wr32(priv, 0x400108, trap);
 762         }
 763 }
 764
 765 static void
 766 nvc0_graph_ctxctl_debug_unit(struct nvc0_graph_priv *priv, u32 base)
 767 {
 768         nv_error(priv, "%06x - done 0x%08x\n", base,
 769                  nv_rd32(priv, base + 0x400));
 770         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
 771                  nv_rd32(priv, base + 0x800), nv_rd32(priv, base + 0x804),
 772                  nv_rd32(priv, base + 0x808), nv_rd32(priv, base + 0x80c));
 773         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
 774                  nv_rd32(priv, base + 0x810), nv_rd32(priv, base + 0x814),
 775                  nv_rd32(priv, base + 0x818), nv_rd32(priv, base + 0x81c));
 776 }
 777
 778 void
 779 nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *priv)
 780 {
 781         u32 gpcnr = nv_rd32(priv, 0x409604) & 0xffff;
 782         u32 gpc;
 783
 784         nvc0_graph_ctxctl_debug_unit(priv, 0x409000);
 785         for (gpc = 0; gpc < gpcnr; gpc++)
 786                 nvc0_graph_ctxctl_debug_unit(priv, 0x502000 + (gpc * 0x8000));
 787 }
 788
 789 static void
 790 nvc0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
 791 {
 792         u32 ustat = nv_rd32(priv, 0x409c18);
 793
 794         if (ustat & 0x00000001)
 795                 nv_error(priv, "CTXCTL ucode error\n");
 796         if (ustat & 0x00080000)
 797                 nv_error(priv, "CTXCTL watchdog timeout\n");
 798         if (ustat & ~0x00080001)
 799                 nv_error(priv, "CTXCTL 0x%08x\n", ustat);
 800
 801         nvc0_graph_ctxctl_debug(priv);
 802         nv_wr32(priv, 0x409c20, ustat);
 803 }
 804
 805 static void
 806 nvc0_graph_intr(struct nouveau_subdev *subdev)
 807 {
 808         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
 809         struct nouveau_engine *engine = nv_engine(subdev);
 810         struct nouveau_object *engctx;
 811         struct nouveau_handle *handle;
 812         struct nvc0_graph_priv *priv = (void *)subdev;
 813         u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
 814         u32 stat = nv_rd32(priv, 0x400100);
 815         u32 addr = nv_rd32(priv, 0x400704);
 816         u32 mthd = (addr & 0x00003ffc);
 817         u32 subc = (addr & 0x00070000) >> 16;
 818         u32 data = nv_rd32(priv, 0x400708);
 819         u32 code = nv_rd32(priv, 0x400110);
 820         u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
 821         int chid;
 822
 823         engctx = nouveau_engctx_get(engine, inst);
 824         chid   = pfifo->chid(pfifo, engctx);
 825
 826         if (stat & 0x00000010) {
 827                 handle = nouveau_handle_get_class(engctx, class);
 828                 if (!handle || nv_call(handle->object, mthd, data)) {
 829                         nv_error(priv,
 830                                  "ILLEGAL_MTHD ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 831                                  chid, inst << 12, nouveau_client_name(engctx),
 832                                  subc, class, mthd, data);
 833                 }
 834                 nouveau_handle_put(handle);
 835                 nv_wr32(priv, 0x400100, 0x00000010);
 836                 stat &= ~0x00000010;
 837         }
 838
 839         if (stat & 0x00000020) {
 840                 nv_error(priv,
 841                          "ILLEGAL_CLASS ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 842                          chid, inst << 12, nouveau_client_name(engctx), subc,
 843                          class, mthd, data);
 844                 nv_wr32(priv, 0x400100, 0x00000020);
 845                 stat &= ~0x00000020;
 846         }
 847
 848         if (stat & 0x00100000) {
 849                 nv_error(priv, "DATA_ERROR [");
 850                 nouveau_enum_print(nv50_data_error_names, code);
 851                 pr_cont("] ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 852                         chid, inst << 12, nouveau_client_name(engctx), subc,
 853                         class, mthd, data);
 854                 nv_wr32(priv, 0x400100, 0x00100000);
 855                 stat &= ~0x00100000;
 856         }
 857
 858         if (stat & 0x00200000) {
 859                 nv_error(priv, "TRAP ch %d [0x%010llx %s]\n", chid, inst << 12,
 860                          nouveau_client_name(engctx));
 861                 nvc0_graph_trap_intr(priv);
 862                 nv_wr32(priv, 0x400100, 0x00200000);
 863                 stat &= ~0x00200000;
 864         }
 865
 866         if (stat & 0x00080000) {
 867                 nvc0_graph_ctxctl_isr(priv);
 868                 nv_wr32(priv, 0x400100, 0x00080000);
 869                 stat &= ~0x00080000;
 870         }
 871
 872         if (stat) {
 873                 nv_error(priv, "unknown stat 0x%08x\n", stat);
 874                 nv_wr32(priv, 0x400100, stat);
 875         }
 876
 877         nv_wr32(priv, 0x400500, 0x00010001);
 878         nouveau_engctx_put(engctx);
 879 }
 880
 881 void
 882 nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
 883                    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
 884 {
 885         int i;
 886
 887         nv_wr32(priv, fuc_base + 0x01c0, 0x01000000);
 888         for (i = 0; i < data->size / 4; i++)
 889                 nv_wr32(priv, fuc_base + 0x01c4, data->data[i]);
 890
 891         nv_wr32(priv, fuc_base + 0x0180, 0x01000000);
 892         for (i = 0; i < code->size / 4; i++) {
 893                 if ((i & 0x3f) == 0)
 894                         nv_wr32(priv, fuc_base + 0x0188, i >> 6);
 895                 nv_wr32(priv, fuc_base + 0x0184, code->data[i]);
 896         }
 897
 898         /* code must be padded to 0x40 words */
 899         for (; i & 0x3f; i++)
 900                 nv_wr32(priv, fuc_base + 0x0184, 0);
 901 }
 902
 903 static void
 904 nvc0_graph_init_csdata(struct nvc0_graph_priv *priv,
 905                        const struct nvc0_graph_pack *pack,
 906                        u32 falcon, u32 starstar, u32 base)
 907 {
 908         const struct nvc0_graph_pack *iter;
 909         const struct nvc0_graph_init *init;
 910         u32 addr = ~0, prev = ~0, xfer = 0;
 911         u32 star, temp;
 912
 913         nv_wr32(priv, falcon + 0x01c0, 0x02000000 + starstar);
 914         star = nv_rd32(priv, falcon + 0x01c4);
 915         temp = nv_rd32(priv, falcon + 0x01c4);
 916         if (temp > star)
 917                 star = temp;
 918         nv_wr32(priv, falcon + 0x01c0, 0x01000000 + star);
 919
 920         pack_for_each_init(init, iter, pack) {
 921                 u32 head = init->addr - base;
 922                 u32 tail = head + init->count * init->pitch;
 923                 while (head < tail) {
 924                         if (head != prev + 4 || xfer >= 32) {
 925                                 if (xfer) {
 926                                         u32 data = ((--xfer << 26) | addr);
 927                                         nv_wr32(priv, falcon + 0x01c4, data);
 928                                         star += 4;
 929                                 }
 930                                 addr = head;
 931                                 xfer = 0;
 932                         }
 933                         prev = head;
 934                         xfer = xfer + 1;
 935                         head = head + init->pitch;
 936                 }
 937         }
 938
 939         nv_wr32(priv, falcon + 0x01c4, (--xfer << 26) | addr);
 940         nv_wr32(priv, falcon + 0x01c0, 0x01000004 + starstar);
 941         nv_wr32(priv, falcon + 0x01c4, star + 4);
 942 }
 943
 944 int
 945 nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
 946 {
 947         struct nvc0_graph_oclass *oclass = (void *)nv_object(priv)->oclass;
 948         struct nvc0_grctx_oclass *cclass = (void *)nv_engine(priv)->cclass;
 949         u32 r000260;
 950         int i;
 951
 952         if (priv->firmware) {
 953                 /* load fuc microcode */
 954                 r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 955                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
 956                                                    &priv->fuc409d);
 957                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
 958                                                    &priv->fuc41ad);
 959                 nv_wr32(priv, 0x000260, r000260);
 960
 961                 /* start both of them running */
 962                 nv_wr32(priv, 0x409840, 0xffffffff);
 963                 nv_wr32(priv, 0x41a10c, 0x00000000);
 964                 nv_wr32(priv, 0x40910c, 0x00000000);
 965                 nv_wr32(priv, 0x41a100, 0x00000002);
 966                 nv_wr32(priv, 0x409100, 0x00000002);
 967                 if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
 968                         nv_warn(priv, "0x409800 wait failed\n");
 969
 970                 nv_wr32(priv, 0x409840, 0xffffffff);
 971                 nv_wr32(priv, 0x409500, 0x7fffffff);
 972                 nv_wr32(priv, 0x409504, 0x00000021);
 973
 974                 nv_wr32(priv, 0x409840, 0xffffffff);
 975                 nv_wr32(priv, 0x409500, 0x00000000);
 976                 nv_wr32(priv, 0x409504, 0x00000010);
 977                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 978                         nv_error(priv, "fuc09 req 0x10 timeout\n");
 979                         return -EBUSY;
 980                 }
 981                 priv->size = nv_rd32(priv, 0x409800);
 982
 983                 nv_wr32(priv, 0x409840, 0xffffffff);
 984                 nv_wr32(priv, 0x409500, 0x00000000);
 985                 nv_wr32(priv, 0x409504, 0x00000016);
 986                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 987                         nv_error(priv, "fuc09 req 0x16 timeout\n");
 988                         return -EBUSY;
 989                 }
 990
 991                 nv_wr32(priv, 0x409840, 0xffffffff);
 992                 nv_wr32(priv, 0x409500, 0x00000000);
 993                 nv_wr32(priv, 0x409504, 0x00000025);
 994                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 995                         nv_error(priv, "fuc09 req 0x25 timeout\n");
 996                         return -EBUSY;
 997                 }
 998
 999                 if (nv_device(priv)->chipset >= 0xe0) {
1000                         nv_wr32(priv, 0x409800, 0x00000000);
1001                         nv_wr32(priv, 0x409500, 0x00000001);
1002                         nv_wr32(priv, 0x409504, 0x00000030);
1003                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1004                                 nv_error(priv, "fuc09 req 0x30 timeout\n");
1005                                 return -EBUSY;
1006                         }
1007
1008                         nv_wr32(priv, 0x409810, 0xb00095c8);
1009                         nv_wr32(priv, 0x409800, 0x00000000);
1010                         nv_wr32(priv, 0x409500, 0x00000001);
1011                         nv_wr32(priv, 0x409504, 0x00000031);
1012                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1013                                 nv_error(priv, "fuc09 req 0x31 timeout\n");
1014                                 return -EBUSY;
1015                         }
1016
1017                         nv_wr32(priv, 0x409810, 0x00080420);
1018                         nv_wr32(priv, 0x409800, 0x00000000);
1019                         nv_wr32(priv, 0x409500, 0x00000001);
1020                         nv_wr32(priv, 0x409504, 0x00000032);
1021                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1022                                 nv_error(priv, "fuc09 req 0x32 timeout\n");
1023                                 return -EBUSY;
1024                         }
1025
1026                         nv_wr32(priv, 0x409614, 0x00000070);
1027                         nv_wr32(priv, 0x409614, 0x00000770);
1028                         nv_wr32(priv, 0x40802c, 0x00000001);
1029                 }
1030
1031                 if (priv->data == NULL) {
1032                         int ret = nvc0_grctx_generate(priv);
1033                         if (ret) {
1034                                 nv_error(priv, "failed to construct context\n");
1035                                 return ret;
1036                         }
1037                 }
1038
1039                 return 0;
1040         } else
1041         if (!oclass->fecs.ucode) {
1042                 return -ENOSYS;
1043         }
1044
1045         /* load HUB microcode */
1046         r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
1047         nv_wr32(priv, 0x4091c0, 0x01000000);
1048         for (i = 0; i < oclass->fecs.ucode->data.size / 4; i++)
1049                 nv_wr32(priv, 0x4091c4, oclass->fecs.ucode->data.data[i]);
1050
1051         nv_wr32(priv, 0x409180, 0x01000000);
1052         for (i = 0; i < oclass->fecs.ucode->code.size / 4; i++) {
1053                 if ((i & 0x3f) == 0)
1054                         nv_wr32(priv, 0x409188, i >> 6);
1055                 nv_wr32(priv, 0x409184, oclass->fecs.ucode->code.data[i]);
1056         }
1057
1058         /* load GPC microcode */
1059         nv_wr32(priv, 0x41a1c0, 0x01000000);
1060         for (i = 0; i < oclass->gpccs.ucode->data.size / 4; i++)
1061                 nv_wr32(priv, 0x41a1c4, oclass->gpccs.ucode->data.data[i]);
1062
1063         nv_wr32(priv, 0x41a180, 0x01000000);
1064         for (i = 0; i < oclass->gpccs.ucode->code.size / 4; i++) {
1065                 if ((i & 0x3f) == 0)
1066                         nv_wr32(priv, 0x41a188, i >> 6);
1067                 nv_wr32(priv, 0x41a184, oclass->gpccs.ucode->code.data[i]);
1068         }
1069         nv_wr32(priv, 0x000260, r000260);
1070
1071         /* load register lists */
1072         nvc0_graph_init_csdata(priv, cclass->hub, 0x409000, 0x000, 0x000000);
1073         nvc0_graph_init_csdata(priv, cclass->gpc, 0x41a000, 0x000, 0x418000);
1074         nvc0_graph_init_csdata(priv, cclass->tpc, 0x41a000, 0x004, 0x419800);
1075         nvc0_graph_init_csdata(priv, cclass->ppc, 0x41a000, 0x008, 0x41be00);
1076
1077         /* start HUB ucode running, it'll init the GPCs */
1078         nv_wr32(priv, 0x40910c, 0x00000000);
1079         nv_wr32(priv, 0x409100, 0x00000002);
1080         if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
1081                 nv_error(priv, "HUB_INIT timed out\n");
1082                 nvc0_graph_ctxctl_debug(priv);
1083                 return -EBUSY;
1084         }
1085
1086         priv->size = nv_rd32(priv, 0x409804);
1087         if (priv->data == NULL) {
1088                 int ret = nvc0_grctx_generate(priv);
1089                 if (ret) {
1090                         nv_error(priv, "failed to construct context\n");
1091                         return ret;
1092                 }
1093         }
1094
1095         return 0;
1096 }
1097
1098 int
1099 nvc0_graph_init(struct nouveau_object *object)
1100 {
1101         struct nvc0_graph_oclass *oclass = (void *)object->oclass;
1102         struct nvc0_graph_priv *priv = (void *)object;
1103         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
1104         u32 data[TPC_MAX / 8] = {};
1105         u8  tpcnr[GPC_MAX];
1106         int gpc, tpc, rop;
1107         int ret, i;
1108
1109         ret = nouveau_graph_init(&priv->base);
1110         if (ret)
1111                 return ret;
1112
1113         nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
1114         nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
1115         nv_wr32(priv, GPC_BCAST(0x0888), 0x00000000);
1116         nv_wr32(priv, GPC_BCAST(0x088c), 0x00000000);
1117         nv_wr32(priv, GPC_BCAST(0x0890), 0x00000000);
1118         nv_wr32(priv, GPC_BCAST(0x0894), 0x00000000);
1119         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
1120         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
1121
1122         nvc0_graph_mmio(priv, oclass->mmio);
1123
1124         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
1125         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
1126                 do {
1127                         gpc = (gpc + 1) % priv->gpc_nr;
1128                 } while (!tpcnr[gpc]);
1129                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
1130
1131                 data[i / 8] |= tpc << ((i % 8) * 4);
1132         }
1133
1134         nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
1135         nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
1136         nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
1137         nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
1138
1139         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1140                 nv_wr32(priv, GPC_UNIT(gpc, 0x0914),
1141                         priv->magic_not_rop_nr << 8 | priv->tpc_nr[gpc]);
1142                 nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 |
1143                         priv->tpc_total);
1144                 nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
1145         }
1146
1147         if (nv_device(priv)->chipset != 0xd7)
1148                 nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
1149         else
1150                 nv_wr32(priv, GPC_BCAST(0x3fd4), magicgpc918);
1151
1152         nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
1153
1154         nv_wr32(priv, 0x400500, 0x00010001);
1155
1156         nv_wr32(priv, 0x400100, 0xffffffff);
1157         nv_wr32(priv, 0x40013c, 0xffffffff);
1158
1159         nv_wr32(priv, 0x409c24, 0x000f0000);
1160         nv_wr32(priv, 0x404000, 0xc0000000);
1161         nv_wr32(priv, 0x404600, 0xc0000000);
1162         nv_wr32(priv, 0x408030, 0xc0000000);
1163         nv_wr32(priv, 0x40601c, 0xc0000000);
1164         nv_wr32(priv, 0x404490, 0xc0000000);
1165         nv_wr32(priv, 0x406018, 0xc0000000);
1166         nv_wr32(priv, 0x405840, 0xc0000000);
1167         nv_wr32(priv, 0x405844, 0x00ffffff);
1168         nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
1169         nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
1170
1171         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1172                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
1173                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
1174                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
1175                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
1176                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
1177                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
1178                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
1179                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
1180                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
1181                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
1182                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
1183                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
1184                 }
1185                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
1186                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
1187         }
1188
1189         for (rop = 0; rop < priv->rop_nr; rop++) {
1190                 nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
1191                 nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
1192                 nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
1193                 nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
1194         }
1195
1196         nv_wr32(priv, 0x400108, 0xffffffff);
1197         nv_wr32(priv, 0x400138, 0xffffffff);
1198         nv_wr32(priv, 0x400118, 0xffffffff);
1199         nv_wr32(priv, 0x400130, 0xffffffff);
1200         nv_wr32(priv, 0x40011c, 0xffffffff);
1201         nv_wr32(priv, 0x400134, 0xffffffff);
1202
1203         nv_wr32(priv, 0x400054, 0x34ce3464);
1204         return nvc0_graph_init_ctxctl(priv);
1205 }
1206
1207 static void
1208 nvc0_graph_dtor_fw(struct nvc0_graph_fuc *fuc)
1209 {
1210         kfree(fuc->data);
1211         fuc->data = NULL;
1212 }
1213
1214 int
1215 nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
1216                    struct nvc0_graph_fuc *fuc)
1217 {
1218         struct nouveau_device *device = nv_device(priv);
1219         const struct firmware *fw;
1220         char f[32];
1221         int ret;
1222
1223         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
1224         ret = request_firmware(&fw, f, nv_device_base(device));
1225         if (ret) {
1226                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
1227                 ret = request_firmware(&fw, f, nv_device_base(device));
1228                 if (ret) {
1229                         nv_error(priv, "failed to load %s\n", fwname);
1230                         return ret;
1231                 }
1232         }
1233
1234         fuc->size = fw->size;
1235         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
1236         release_firmware(fw);
1237         return (fuc->data != NULL) ? 0 : -ENOMEM;
1238 }
1239
1240 void
1241 nvc0_graph_dtor(struct nouveau_object *object)
1242 {
1243         struct nvc0_graph_priv *priv = (void *)object;
1244
1245         kfree(priv->data);
1246
1247         nvc0_graph_dtor_fw(&priv->fuc409c);
1248         nvc0_graph_dtor_fw(&priv->fuc409d);
1249         nvc0_graph_dtor_fw(&priv->fuc41ac);
1250         nvc0_graph_dtor_fw(&priv->fuc41ad);
1251
1252         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
1253         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
1254
1255         nouveau_graph_destroy(&priv->base);
1256 }
1257
1258 int
1259 nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
1260                 struct nouveau_oclass *bclass, void *data, u32 size,
1261                 struct nouveau_object **pobject)
1262 {
1263         struct nvc0_graph_oclass *oclass = (void *)bclass;
1264         struct nouveau_device *device = nv_device(parent);
1265         struct nvc0_graph_priv *priv;
1266         bool use_ext_fw, enable;
1267         int ret, i;
1268
1269         use_ext_fw = nouveau_boolopt(device->cfgopt, "NvGrUseFW",
1270                                      oclass->fecs.ucode == NULL);
1271         enable = use_ext_fw || oclass->fecs.ucode != NULL;
1272
1273         ret = nouveau_graph_create(parent, engine, bclass, enable, &priv);
1274         *pobject = nv_object(priv);
1275         if (ret)
1276                 return ret;
1277
1278         nv_subdev(priv)->unit = 0x08001000;
1279         nv_subdev(priv)->intr = nvc0_graph_intr;
1280
1281         priv->base.units = nvc0_graph_units;
1282
1283         if (use_ext_fw) {
1284                 nv_info(priv, "using external firmware\n");
1285                 if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
1286                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
1287                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
1288                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
1289                         return -EINVAL;
1290                 priv->firmware = true;
1291         }
1292
1293         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
1294                                 &priv->unk4188b4);
1295         if (ret)
1296                 return ret;
1297
1298         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
1299                                 &priv->unk4188b8);
1300         if (ret)
1301                 return ret;
1302
1303         for (i = 0; i < 0x1000; i += 4) {
1304                 nv_wo32(priv->unk4188b4, i, 0x00000010);
1305                 nv_wo32(priv->unk4188b8, i, 0x00000010);
1306         }
1307
1308         priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
1309         priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
1310         for (i = 0; i < priv->gpc_nr; i++) {
1311                 priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
1312                 priv->tpc_total += priv->tpc_nr[i];
1313         }
1314
1315         /*XXX: these need figuring out... though it might not even matter */
1316         switch (nv_device(priv)->chipset) {
1317         case 0xc0:
1318                 if (priv->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
1319                         priv->magic_not_rop_nr = 0x07;
1320                 } else
1321                 if (priv->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
1322                         priv->magic_not_rop_nr = 0x05;
1323                 } else
1324                 if (priv->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
1325                         priv->magic_not_rop_nr = 0x06;
1326                 }
1327                 break;
1328         case 0xc3: /* 450, 4/0/0/0, 2 */
1329                 priv->magic_not_rop_nr = 0x03;
1330                 break;
1331         case 0xc4: /* 460, 3/4/0/0, 4 */
1332                 priv->magic_not_rop_nr = 0x01;
1333                 break;
1334         case 0xc1: /* 2/0/0/0, 1 */
1335                 priv->magic_not_rop_nr = 0x01;
1336                 break;
1337         case 0xc8: /* 4/4/3/4, 5 */
1338                 priv->magic_not_rop_nr = 0x06;
1339                 break;
1340         case 0xce: /* 4/4/0/0, 4 */
1341                 priv->magic_not_rop_nr = 0x03;
1342                 break;
1343         case 0xcf: /* 4/0/0/0, 3 */
1344                 priv->magic_not_rop_nr = 0x03;
1345                 break;
1346         case 0xd7:
1347         case 0xd9: /* 1/0/0/0, 1 */
1348                 priv->magic_not_rop_nr = 0x01;
1349                 break;
1350         }
1351
1352         nv_engine(priv)->cclass = *oclass->cclass;
1353         nv_engine(priv)->sclass =  oclass->sclass;
1354         return 0;
1355 }
1356
1357 #include "fuc/hubnvc0.fuc.h"
1358
1359 struct nvc0_graph_ucode
1360 nvc0_graph_fecs_ucode = {
1361         .code.data = nvc0_grhub_code,
1362         .code.size = sizeof(nvc0_grhub_code),
1363         .data.data = nvc0_grhub_data,
1364         .data.size = sizeof(nvc0_grhub_data),
1365 };
1366
1367 #include "fuc/gpcnvc0.fuc.h"
1368
1369 struct nvc0_graph_ucode
1370 nvc0_graph_gpccs_ucode = {
1371         .code.data = nvc0_grgpc_code,
1372         .code.size = sizeof(nvc0_grgpc_code),
1373         .data.data = nvc0_grgpc_data,
1374         .data.size = sizeof(nvc0_grgpc_data),
1375 };
1376
1377 struct nouveau_oclass *
1378 nvc0_graph_oclass = &(struct nvc0_graph_oclass) {
1379         .base.handle = NV_ENGINE(GR, 0xc0),
1380         .base.ofuncs = &(struct nouveau_ofuncs) {
1381                 .ctor = nvc0_graph_ctor,
1382                 .dtor = nvc0_graph_dtor,
1383                 .init = nvc0_graph_init,
1384                 .fini = _nouveau_graph_fini,
1385         },
1386         .cclass = &nvc0_grctx_oclass,
1387         .sclass =  nvc0_graph_sclass,
1388         .mmio = nvc0_graph_pack_mmio,
1389         .fecs.ucode = &nvc0_graph_fecs_ucode,
1390         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
1391 }.base;