Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cascardo/linux.git] / drivers / gpu / drm / nouveau / core / engine / graph / nvc0.c
1 /*
2  * Copyright 2012 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24
25 #include "nvc0.h"
26 #include "ctxnvc0.h"
27
28 /*******************************************************************************
29  * Graphics object classes
30  ******************************************************************************/
31
32 struct nouveau_oclass
33 nvc0_graph_sclass[] = {
34         { 0x902d, &nouveau_object_ofuncs },
35         { 0x9039, &nouveau_object_ofuncs },
36         { 0x9097, &nouveau_object_ofuncs },
37         { 0x90c0, &nouveau_object_ofuncs },
38         {}
39 };
40
41 /*******************************************************************************
42  * PGRAPH context
43  ******************************************************************************/
44
45 int
46 nvc0_graph_context_ctor(struct nouveau_object *parent,
47                         struct nouveau_object *engine,
48                         struct nouveau_oclass *oclass, void *args, u32 size,
49                         struct nouveau_object **pobject)
50 {
51         struct nouveau_vm *vm = nouveau_client(parent)->vm;
52         struct nvc0_graph_priv *priv = (void *)engine;
53         struct nvc0_graph_data *data = priv->mmio_data;
54         struct nvc0_graph_mmio *mmio = priv->mmio_list;
55         struct nvc0_graph_chan *chan;
56         int ret, i;
57
58         /* allocate memory for context, and fill with default values */
59         ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
60                                            priv->size, 0x100,
61                                            NVOBJ_FLAG_ZERO_ALLOC, &chan);
62         *pobject = nv_object(chan);
63         if (ret)
64                 return ret;
65
66         /* allocate memory for a "mmio list" buffer that's used by the HUB
67          * fuc to modify some per-context register settings on first load
68          * of the context.
69          */
70         ret = nouveau_gpuobj_new(nv_object(chan), NULL, 0x1000, 0x100, 0,
71                                 &chan->mmio);
72         if (ret)
73                 return ret;
74
75         ret = nouveau_gpuobj_map_vm(nv_gpuobj(chan->mmio), vm,
76                                     NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
77                                     &chan->mmio_vma);
78         if (ret)
79                 return ret;
80
81         /* allocate buffers referenced by mmio list */
82         for (i = 0; data->size && i < ARRAY_SIZE(priv->mmio_data); i++) {
83                 ret = nouveau_gpuobj_new(nv_object(chan), NULL, data->size,
84                                          data->align, 0, &chan->data[i].mem);
85                 if (ret)
86                         return ret;
87
88                 ret = nouveau_gpuobj_map_vm(chan->data[i].mem, vm, data->access,
89                                            &chan->data[i].vma);
90                 if (ret)
91                         return ret;
92
93                 data++;
94         }
95
96         /* finally, fill in the mmio list and point the context at it */
97         for (i = 0; mmio->addr && i < ARRAY_SIZE(priv->mmio_list); i++) {
98                 u32 addr = mmio->addr;
99                 u32 data = mmio->data;
100
101                 if (mmio->shift) {
102                         u64 info = chan->data[mmio->buffer].vma.offset;
103                         data |= info >> mmio->shift;
104                 }
105
106                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
107                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
108                 mmio++;
109         }
110
111         for (i = 0; i < priv->size; i += 4)
112                 nv_wo32(chan, i, priv->data[i / 4]);
113
114         if (!priv->firmware) {
115                 nv_wo32(chan, 0x00, chan->mmio_nr / 2);
116                 nv_wo32(chan, 0x04, chan->mmio_vma.offset >> 8);
117         } else {
118                 nv_wo32(chan, 0xf4, 0);
119                 nv_wo32(chan, 0xf8, 0);
120                 nv_wo32(chan, 0x10, chan->mmio_nr / 2);
121                 nv_wo32(chan, 0x14, lower_32_bits(chan->mmio_vma.offset));
122                 nv_wo32(chan, 0x18, upper_32_bits(chan->mmio_vma.offset));
123                 nv_wo32(chan, 0x1c, 1);
124                 nv_wo32(chan, 0x20, 0);
125                 nv_wo32(chan, 0x28, 0);
126                 nv_wo32(chan, 0x2c, 0);
127         }
128
129         return 0;
130 }
131
132 void
133 nvc0_graph_context_dtor(struct nouveau_object *object)
134 {
135         struct nvc0_graph_chan *chan = (void *)object;
136         int i;
137
138         for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
139                 nouveau_gpuobj_unmap(&chan->data[i].vma);
140                 nouveau_gpuobj_ref(NULL, &chan->data[i].mem);
141         }
142
143         nouveau_gpuobj_unmap(&chan->mmio_vma);
144         nouveau_gpuobj_ref(NULL, &chan->mmio);
145
146         nouveau_graph_context_destroy(&chan->base);
147 }
148
149 /*******************************************************************************
150  * PGRAPH register lists
151  ******************************************************************************/
152
153 const struct nvc0_graph_init
154 nvc0_graph_init_main_0[] = {
155         { 0x400080,   1, 0x04, 0x003083c2 },
156         { 0x400088,   1, 0x04, 0x00006fe7 },
157         { 0x40008c,   1, 0x04, 0x00000000 },
158         { 0x400090,   1, 0x04, 0x00000030 },
159         { 0x40013c,   1, 0x04, 0x013901f7 },
160         { 0x400140,   1, 0x04, 0x00000100 },
161         { 0x400144,   1, 0x04, 0x00000000 },
162         { 0x400148,   1, 0x04, 0x00000110 },
163         { 0x400138,   1, 0x04, 0x00000000 },
164         { 0x400130,   2, 0x04, 0x00000000 },
165         { 0x400124,   1, 0x04, 0x00000002 },
166         {}
167 };
168
169 const struct nvc0_graph_init
170 nvc0_graph_init_fe_0[] = {
171         { 0x40415c,   1, 0x04, 0x00000000 },
172         { 0x404170,   1, 0x04, 0x00000000 },
173         {}
174 };
175
176 const struct nvc0_graph_init
177 nvc0_graph_init_pri_0[] = {
178         { 0x404488,   2, 0x04, 0x00000000 },
179         {}
180 };
181
182 const struct nvc0_graph_init
183 nvc0_graph_init_rstr2d_0[] = {
184         { 0x407808,   1, 0x04, 0x00000000 },
185         {}
186 };
187
188 const struct nvc0_graph_init
189 nvc0_graph_init_pd_0[] = {
190         { 0x406024,   1, 0x04, 0x00000000 },
191         {}
192 };
193
194 const struct nvc0_graph_init
195 nvc0_graph_init_ds_0[] = {
196         { 0x405844,   1, 0x04, 0x00ffffff },
197         { 0x405850,   1, 0x04, 0x00000000 },
198         { 0x405908,   1, 0x04, 0x00000000 },
199         {}
200 };
201
202 const struct nvc0_graph_init
203 nvc0_graph_init_scc_0[] = {
204         { 0x40803c,   1, 0x04, 0x00000000 },
205         {}
206 };
207
208 const struct nvc0_graph_init
209 nvc0_graph_init_prop_0[] = {
210         { 0x4184a0,   1, 0x04, 0x00000000 },
211         {}
212 };
213
214 const struct nvc0_graph_init
215 nvc0_graph_init_gpc_unk_0[] = {
216         { 0x418604,   1, 0x04, 0x00000000 },
217         { 0x418680,   1, 0x04, 0x00000000 },
218         { 0x418714,   1, 0x04, 0x80000000 },
219         { 0x418384,   1, 0x04, 0x00000000 },
220         {}
221 };
222
223 const struct nvc0_graph_init
224 nvc0_graph_init_setup_0[] = {
225         { 0x418814,   3, 0x04, 0x00000000 },
226         {}
227 };
228
229 const struct nvc0_graph_init
230 nvc0_graph_init_crstr_0[] = {
231         { 0x418b04,   1, 0x04, 0x00000000 },
232         {}
233 };
234
235 const struct nvc0_graph_init
236 nvc0_graph_init_setup_1[] = {
237         { 0x4188c8,   1, 0x04, 0x80000000 },
238         { 0x4188cc,   1, 0x04, 0x00000000 },
239         { 0x4188d0,   1, 0x04, 0x00010000 },
240         { 0x4188d4,   1, 0x04, 0x00000001 },
241         {}
242 };
243
244 const struct nvc0_graph_init
245 nvc0_graph_init_zcull_0[] = {
246         { 0x418910,   1, 0x04, 0x00010001 },
247         { 0x418914,   1, 0x04, 0x00000301 },
248         { 0x418918,   1, 0x04, 0x00800000 },
249         { 0x418980,   1, 0x04, 0x77777770 },
250         { 0x418984,   3, 0x04, 0x77777777 },
251         {}
252 };
253
254 const struct nvc0_graph_init
255 nvc0_graph_init_gpm_0[] = {
256         { 0x418c04,   1, 0x04, 0x00000000 },
257         { 0x418c88,   1, 0x04, 0x00000000 },
258         {}
259 };
260
261 const struct nvc0_graph_init
262 nvc0_graph_init_gpc_unk_1[] = {
263         { 0x418d00,   1, 0x04, 0x00000000 },
264         { 0x418f08,   1, 0x04, 0x00000000 },
265         { 0x418e00,   1, 0x04, 0x00000050 },
266         { 0x418e08,   1, 0x04, 0x00000000 },
267         {}
268 };
269
270 const struct nvc0_graph_init
271 nvc0_graph_init_gcc_0[] = {
272         { 0x41900c,   1, 0x04, 0x00000000 },
273         { 0x419018,   1, 0x04, 0x00000000 },
274         {}
275 };
276
277 const struct nvc0_graph_init
278 nvc0_graph_init_tpccs_0[] = {
279         { 0x419d08,   2, 0x04, 0x00000000 },
280         { 0x419d10,   1, 0x04, 0x00000014 },
281         {}
282 };
283
284 const struct nvc0_graph_init
285 nvc0_graph_init_tex_0[] = {
286         { 0x419ab0,   1, 0x04, 0x00000000 },
287         { 0x419ab8,   1, 0x04, 0x000000e7 },
288         { 0x419abc,   2, 0x04, 0x00000000 },
289         {}
290 };
291
292 const struct nvc0_graph_init
293 nvc0_graph_init_pe_0[] = {
294         { 0x41980c,   3, 0x04, 0x00000000 },
295         { 0x419844,   1, 0x04, 0x00000000 },
296         { 0x41984c,   1, 0x04, 0x00005bc5 },
297         { 0x419850,   4, 0x04, 0x00000000 },
298         {}
299 };
300
301 const struct nvc0_graph_init
302 nvc0_graph_init_l1c_0[] = {
303         { 0x419c98,   1, 0x04, 0x00000000 },
304         { 0x419ca8,   1, 0x04, 0x80000000 },
305         { 0x419cb4,   1, 0x04, 0x00000000 },
306         { 0x419cb8,   1, 0x04, 0x00008bf4 },
307         { 0x419cbc,   1, 0x04, 0x28137606 },
308         { 0x419cc0,   2, 0x04, 0x00000000 },
309         {}
310 };
311
312 const struct nvc0_graph_init
313 nvc0_graph_init_wwdx_0[] = {
314         { 0x419bd4,   1, 0x04, 0x00800000 },
315         { 0x419bdc,   1, 0x04, 0x00000000 },
316         {}
317 };
318
319 const struct nvc0_graph_init
320 nvc0_graph_init_tpccs_1[] = {
321         { 0x419d2c,   1, 0x04, 0x00000000 },
322         {}
323 };
324
325 const struct nvc0_graph_init
326 nvc0_graph_init_mpc_0[] = {
327         { 0x419c0c,   1, 0x04, 0x00000000 },
328         {}
329 };
330
331 static const struct nvc0_graph_init
332 nvc0_graph_init_sm_0[] = {
333         { 0x419e00,   1, 0x04, 0x00000000 },
334         { 0x419ea0,   1, 0x04, 0x00000000 },
335         { 0x419ea4,   1, 0x04, 0x00000100 },
336         { 0x419ea8,   1, 0x04, 0x00001100 },
337         { 0x419eac,   1, 0x04, 0x11100702 },
338         { 0x419eb0,   1, 0x04, 0x00000003 },
339         { 0x419eb4,   4, 0x04, 0x00000000 },
340         { 0x419ec8,   1, 0x04, 0x06060618 },
341         { 0x419ed0,   1, 0x04, 0x0eff0e38 },
342         { 0x419ed4,   1, 0x04, 0x011104f1 },
343         { 0x419edc,   1, 0x04, 0x00000000 },
344         { 0x419f00,   1, 0x04, 0x00000000 },
345         { 0x419f2c,   1, 0x04, 0x00000000 },
346         {}
347 };
348
349 const struct nvc0_graph_init
350 nvc0_graph_init_be_0[] = {
351         { 0x40880c,   1, 0x04, 0x00000000 },
352         { 0x408910,   9, 0x04, 0x00000000 },
353         { 0x408950,   1, 0x04, 0x00000000 },
354         { 0x408954,   1, 0x04, 0x0000ffff },
355         { 0x408984,   1, 0x04, 0x00000000 },
356         { 0x408988,   1, 0x04, 0x08040201 },
357         { 0x40898c,   1, 0x04, 0x80402010 },
358         {}
359 };
360
361 const struct nvc0_graph_init
362 nvc0_graph_init_fe_1[] = {
363         { 0x4040f0,   1, 0x04, 0x00000000 },
364         {}
365 };
366
367 const struct nvc0_graph_init
368 nvc0_graph_init_pe_1[] = {
369         { 0x419880,   1, 0x04, 0x00000002 },
370         {}
371 };
372
373 static const struct nvc0_graph_pack
374 nvc0_graph_pack_mmio[] = {
375         { nvc0_graph_init_main_0 },
376         { nvc0_graph_init_fe_0 },
377         { nvc0_graph_init_pri_0 },
378         { nvc0_graph_init_rstr2d_0 },
379         { nvc0_graph_init_pd_0 },
380         { nvc0_graph_init_ds_0 },
381         { nvc0_graph_init_scc_0 },
382         { nvc0_graph_init_prop_0 },
383         { nvc0_graph_init_gpc_unk_0 },
384         { nvc0_graph_init_setup_0 },
385         { nvc0_graph_init_crstr_0 },
386         { nvc0_graph_init_setup_1 },
387         { nvc0_graph_init_zcull_0 },
388         { nvc0_graph_init_gpm_0 },
389         { nvc0_graph_init_gpc_unk_1 },
390         { nvc0_graph_init_gcc_0 },
391         { nvc0_graph_init_tpccs_0 },
392         { nvc0_graph_init_tex_0 },
393         { nvc0_graph_init_pe_0 },
394         { nvc0_graph_init_l1c_0 },
395         { nvc0_graph_init_wwdx_0 },
396         { nvc0_graph_init_tpccs_1 },
397         { nvc0_graph_init_mpc_0 },
398         { nvc0_graph_init_sm_0 },
399         { nvc0_graph_init_be_0 },
400         { nvc0_graph_init_fe_1 },
401         { nvc0_graph_init_pe_1 },
402         {}
403 };
404
405 /*******************************************************************************
406  * PGRAPH engine/subdev functions
407  ******************************************************************************/
408
409 void
410 nvc0_graph_mmio(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
411 {
412         const struct nvc0_graph_pack *pack;
413         const struct nvc0_graph_init *init;
414
415         pack_for_each_init(init, pack, p) {
416                 u32 next = init->addr + init->count * init->pitch;
417                 u32 addr = init->addr;
418                 while (addr < next) {
419                         nv_wr32(priv, addr, init->data);
420                         addr += init->pitch;
421                 }
422         }
423 }
424
425 void
426 nvc0_graph_icmd(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
427 {
428         const struct nvc0_graph_pack *pack;
429         const struct nvc0_graph_init *init;
430         u32 data = 0;
431
432         nv_wr32(priv, 0x400208, 0x80000000);
433
434         pack_for_each_init(init, pack, p) {
435                 u32 next = init->addr + init->count * init->pitch;
436                 u32 addr = init->addr;
437
438                 if ((pack == p && init == p->init) || data != init->data) {
439                         nv_wr32(priv, 0x400204, init->data);
440                         data = init->data;
441                 }
442
443                 while (addr < next) {
444                         nv_wr32(priv, 0x400200, addr);
445                         nv_wait(priv, 0x400700, 0x00000002, 0x00000000);
446                         addr += init->pitch;
447                 }
448         }
449
450         nv_wr32(priv, 0x400208, 0x00000000);
451 }
452
453 void
454 nvc0_graph_mthd(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
455 {
456         const struct nvc0_graph_pack *pack;
457         const struct nvc0_graph_init *init;
458         u32 data = 0;
459
460         pack_for_each_init(init, pack, p) {
461                 u32 ctrl = 0x80000000 | pack->type;
462                 u32 next = init->addr + init->count * init->pitch;
463                 u32 addr = init->addr;
464
465                 if ((pack == p && init == p->init) || data != init->data) {
466                         nv_wr32(priv, 0x40448c, init->data);
467                         data = init->data;
468                 }
469
470                 while (addr < next) {
471                         nv_wr32(priv, 0x404488, ctrl | (addr << 14));
472                         addr += init->pitch;
473                 }
474         }
475 }
476
477 u64
478 nvc0_graph_units(struct nouveau_graph *graph)
479 {
480         struct nvc0_graph_priv *priv = (void *)graph;
481         u64 cfg;
482
483         cfg  = (u32)priv->gpc_nr;
484         cfg |= (u32)priv->tpc_total << 8;
485         cfg |= (u64)priv->rop_nr << 32;
486
487         return cfg;
488 }
489
490 static const struct nouveau_enum nve0_sked_error[] = {
491         { 7, "CONSTANT_BUFFER_SIZE" },
492         { 9, "LOCAL_MEMORY_SIZE_POS" },
493         { 10, "LOCAL_MEMORY_SIZE_NEG" },
494         { 11, "WARP_CSTACK_SIZE" },
495         { 12, "TOTAL_TEMP_SIZE" },
496         { 13, "REGISTER_COUNT" },
497         { 18, "TOTAL_THREADS" },
498         { 20, "PROGRAM_OFFSET" },
499         { 21, "SHARED_MEMORY_SIZE" },
500         { 25, "SHARED_CONFIG_TOO_SMALL" },
501         { 26, "TOTAL_REGISTER_COUNT" },
502         {}
503 };
504
505 static const struct nouveau_enum nvc0_gpc_rop_error[] = {
506         { 1, "RT_PITCH_OVERRUN" },
507         { 4, "RT_WIDTH_OVERRUN" },
508         { 5, "RT_HEIGHT_OVERRUN" },
509         { 7, "ZETA_STORAGE_TYPE_MISMATCH" },
510         { 8, "RT_STORAGE_TYPE_MISMATCH" },
511         { 10, "RT_LINEAR_MISMATCH" },
512         {}
513 };
514
515 static void
516 nvc0_graph_trap_gpc_rop(struct nvc0_graph_priv *priv, int gpc)
517 {
518         u32 trap[4];
519         int i;
520
521         trap[0] = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
522         trap[1] = nv_rd32(priv, GPC_UNIT(gpc, 0x0434));
523         trap[2] = nv_rd32(priv, GPC_UNIT(gpc, 0x0438));
524         trap[3] = nv_rd32(priv, GPC_UNIT(gpc, 0x043c));
525
526         nv_error(priv, "GPC%d/PROP trap:", gpc);
527         for (i = 0; i <= 29; ++i) {
528                 if (!(trap[0] & (1 << i)))
529                         continue;
530                 pr_cont(" ");
531                 nouveau_enum_print(nvc0_gpc_rop_error, i);
532         }
533         pr_cont("\n");
534
535         nv_error(priv, "x = %u, y = %u, format = %x, storage type = %x\n",
536                  trap[1] & 0xffff, trap[1] >> 16, (trap[2] >> 8) & 0x3f,
537                  trap[3] & 0xff);
538         nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
539 }
540
541 static const struct nouveau_enum nvc0_mp_warp_error[] = {
542         { 0x00, "NO_ERROR" },
543         { 0x01, "STACK_MISMATCH" },
544         { 0x05, "MISALIGNED_PC" },
545         { 0x08, "MISALIGNED_GPR" },
546         { 0x09, "INVALID_OPCODE" },
547         { 0x0d, "GPR_OUT_OF_BOUNDS" },
548         { 0x0e, "MEM_OUT_OF_BOUNDS" },
549         { 0x0f, "UNALIGNED_MEM_ACCESS" },
550         { 0x11, "INVALID_PARAM" },
551         {}
552 };
553
554 static const struct nouveau_bitfield nvc0_mp_global_error[] = {
555         { 0x00000004, "MULTIPLE_WARP_ERRORS" },
556         { 0x00000008, "OUT_OF_STACK_SPACE" },
557         {}
558 };
559
560 static void
561 nvc0_graph_trap_mp(struct nvc0_graph_priv *priv, int gpc, int tpc)
562 {
563         u32 werr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x648));
564         u32 gerr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x650));
565
566         nv_error(priv, "GPC%i/TPC%i/MP trap:", gpc, tpc);
567         nouveau_bitfield_print(nvc0_mp_global_error, gerr);
568         if (werr) {
569                 pr_cont(" ");
570                 nouveau_enum_print(nvc0_mp_warp_error, werr & 0xffff);
571         }
572         pr_cont("\n");
573
574         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);
575         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x650), gerr);
576 }
577
578 static void
579 nvc0_graph_trap_tpc(struct nvc0_graph_priv *priv, int gpc, int tpc)
580 {
581         u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0508));
582
583         if (stat & 0x00000001) {
584                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0224));
585                 nv_error(priv, "GPC%d/TPC%d/TEX: 0x%08x\n", gpc, tpc, trap);
586                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
587                 stat &= ~0x00000001;
588         }
589
590         if (stat & 0x00000002) {
591                 nvc0_graph_trap_mp(priv, gpc, tpc);
592                 stat &= ~0x00000002;
593         }
594
595         if (stat & 0x00000004) {
596                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0084));
597                 nv_error(priv, "GPC%d/TPC%d/POLY: 0x%08x\n", gpc, tpc, trap);
598                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
599                 stat &= ~0x00000004;
600         }
601
602         if (stat & 0x00000008) {
603                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x048c));
604                 nv_error(priv, "GPC%d/TPC%d/L1C: 0x%08x\n", gpc, tpc, trap);
605                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
606                 stat &= ~0x00000008;
607         }
608
609         if (stat) {
610                 nv_error(priv, "GPC%d/TPC%d/0x%08x: unknown\n", gpc, tpc, stat);
611         }
612 }
613
614 static void
615 nvc0_graph_trap_gpc(struct nvc0_graph_priv *priv, int gpc)
616 {
617         u32 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
618         int tpc;
619
620         if (stat & 0x00000001) {
621                 nvc0_graph_trap_gpc_rop(priv, gpc);
622                 stat &= ~0x00000001;
623         }
624
625         if (stat & 0x00000002) {
626                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
627                 nv_error(priv, "GPC%d/ZCULL: 0x%08x\n", gpc, trap);
628                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
629                 stat &= ~0x00000002;
630         }
631
632         if (stat & 0x00000004) {
633                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
634                 nv_error(priv, "GPC%d/CCACHE: 0x%08x\n", gpc, trap);
635                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
636                 stat &= ~0x00000004;
637         }
638
639         if (stat & 0x00000008) {
640                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
641                 nv_error(priv, "GPC%d/ESETUP: 0x%08x\n", gpc, trap);
642                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
643                 stat &= ~0x00000009;
644         }
645
646         for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
647                 u32 mask = 0x00010000 << tpc;
648                 if (stat & mask) {
649                         nvc0_graph_trap_tpc(priv, gpc, tpc);
650                         nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), mask);
651                         stat &= ~mask;
652                 }
653         }
654
655         if (stat) {
656                 nv_error(priv, "GPC%d/0x%08x: unknown\n", gpc, stat);
657         }
658 }
659
660 static void
661 nvc0_graph_trap_intr(struct nvc0_graph_priv *priv)
662 {
663         u32 trap = nv_rd32(priv, 0x400108);
664         int rop, gpc, i;
665
666         if (trap & 0x00000001) {
667                 u32 stat = nv_rd32(priv, 0x404000);
668                 nv_error(priv, "DISPATCH 0x%08x\n", stat);
669                 nv_wr32(priv, 0x404000, 0xc0000000);
670                 nv_wr32(priv, 0x400108, 0x00000001);
671                 trap &= ~0x00000001;
672         }
673
674         if (trap & 0x00000002) {
675                 u32 stat = nv_rd32(priv, 0x404600);
676                 nv_error(priv, "M2MF 0x%08x\n", stat);
677                 nv_wr32(priv, 0x404600, 0xc0000000);
678                 nv_wr32(priv, 0x400108, 0x00000002);
679                 trap &= ~0x00000002;
680         }
681
682         if (trap & 0x00000008) {
683                 u32 stat = nv_rd32(priv, 0x408030);
684                 nv_error(priv, "CCACHE 0x%08x\n", stat);
685                 nv_wr32(priv, 0x408030, 0xc0000000);
686                 nv_wr32(priv, 0x400108, 0x00000008);
687                 trap &= ~0x00000008;
688         }
689
690         if (trap & 0x00000010) {
691                 u32 stat = nv_rd32(priv, 0x405840);
692                 nv_error(priv, "SHADER 0x%08x\n", stat);
693                 nv_wr32(priv, 0x405840, 0xc0000000);
694                 nv_wr32(priv, 0x400108, 0x00000010);
695                 trap &= ~0x00000010;
696         }
697
698         if (trap & 0x00000040) {
699                 u32 stat = nv_rd32(priv, 0x40601c);
700                 nv_error(priv, "UNK6 0x%08x\n", stat);
701                 nv_wr32(priv, 0x40601c, 0xc0000000);
702                 nv_wr32(priv, 0x400108, 0x00000040);
703                 trap &= ~0x00000040;
704         }
705
706         if (trap & 0x00000080) {
707                 u32 stat = nv_rd32(priv, 0x404490);
708                 nv_error(priv, "MACRO 0x%08x\n", stat);
709                 nv_wr32(priv, 0x404490, 0xc0000000);
710                 nv_wr32(priv, 0x400108, 0x00000080);
711                 trap &= ~0x00000080;
712         }
713
714         if (trap & 0x00000100) {
715                 u32 stat = nv_rd32(priv, 0x407020);
716
717                 nv_error(priv, "SKED:");
718                 for (i = 0; i <= 29; ++i) {
719                         if (!(stat & (1 << i)))
720                                 continue;
721                         pr_cont(" ");
722                         nouveau_enum_print(nve0_sked_error, i);
723                 }
724                 pr_cont("\n");
725
726                 if (stat & 0x3fffffff)
727                         nv_wr32(priv, 0x407020, 0x40000000);
728                 nv_wr32(priv, 0x400108, 0x00000100);
729                 trap &= ~0x00000100;
730         }
731
732         if (trap & 0x01000000) {
733                 u32 stat = nv_rd32(priv, 0x400118);
734                 for (gpc = 0; stat && gpc < priv->gpc_nr; gpc++) {
735                         u32 mask = 0x00000001 << gpc;
736                         if (stat & mask) {
737                                 nvc0_graph_trap_gpc(priv, gpc);
738                                 nv_wr32(priv, 0x400118, mask);
739                                 stat &= ~mask;
740                         }
741                 }
742                 nv_wr32(priv, 0x400108, 0x01000000);
743                 trap &= ~0x01000000;
744         }
745
746         if (trap & 0x02000000) {
747                 for (rop = 0; rop < priv->rop_nr; rop++) {
748                         u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
749                         u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
750                         nv_error(priv, "ROP%d 0x%08x 0x%08x\n",
751                                  rop, statz, statc);
752                         nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
753                         nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
754                 }
755                 nv_wr32(priv, 0x400108, 0x02000000);
756                 trap &= ~0x02000000;
757         }
758
759         if (trap) {
760                 nv_error(priv, "TRAP UNHANDLED 0x%08x\n", trap);
761                 nv_wr32(priv, 0x400108, trap);
762         }
763 }
764
765 static void
766 nvc0_graph_ctxctl_debug_unit(struct nvc0_graph_priv *priv, u32 base)
767 {
768         nv_error(priv, "%06x - done 0x%08x\n", base,
769                  nv_rd32(priv, base + 0x400));
770         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
771                  nv_rd32(priv, base + 0x800), nv_rd32(priv, base + 0x804),
772                  nv_rd32(priv, base + 0x808), nv_rd32(priv, base + 0x80c));
773         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
774                  nv_rd32(priv, base + 0x810), nv_rd32(priv, base + 0x814),
775                  nv_rd32(priv, base + 0x818), nv_rd32(priv, base + 0x81c));
776 }
777
778 void
779 nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *priv)
780 {
781         u32 gpcnr = nv_rd32(priv, 0x409604) & 0xffff;
782         u32 gpc;
783
784         nvc0_graph_ctxctl_debug_unit(priv, 0x409000);
785         for (gpc = 0; gpc < gpcnr; gpc++)
786                 nvc0_graph_ctxctl_debug_unit(priv, 0x502000 + (gpc * 0x8000));
787 }
788
789 static void
790 nvc0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
791 {
792         u32 stat = nv_rd32(priv, 0x409c18);
793
794         if (stat & 0x00000001) {
795                 u32 code = nv_rd32(priv, 0x409814);
796                 if (code == E_BAD_FWMTHD) {
797                         u32 class = nv_rd32(priv, 0x409808);
798                         u32  addr = nv_rd32(priv, 0x40980c);
799                         u32  subc = (addr & 0x00070000) >> 16;
800                         u32  mthd = (addr & 0x00003ffc);
801                         u32  data = nv_rd32(priv, 0x409810);
802
803                         nv_error(priv, "FECS MTHD subc %d class 0x%04x "
804                                        "mthd 0x%04x data 0x%08x\n",
805                                  subc, class, mthd, data);
806
807                         nv_wr32(priv, 0x409c20, 0x00000001);
808                         stat &= ~0x00000001;
809                 } else {
810                         nv_error(priv, "FECS ucode error %d\n", code);
811                 }
812         }
813
814         if (stat & 0x00080000) {
815                 nv_error(priv, "FECS watchdog timeout\n");
816                 nvc0_graph_ctxctl_debug(priv);
817                 nv_wr32(priv, 0x409c20, 0x00080000);
818                 stat &= ~0x00080000;
819         }
820
821         if (stat) {
822                 nv_error(priv, "FECS 0x%08x\n", stat);
823                 nvc0_graph_ctxctl_debug(priv);
824                 nv_wr32(priv, 0x409c20, stat);
825         }
826 }
827
828 static void
829 nvc0_graph_intr(struct nouveau_subdev *subdev)
830 {
831         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
832         struct nouveau_engine *engine = nv_engine(subdev);
833         struct nouveau_object *engctx;
834         struct nouveau_handle *handle;
835         struct nvc0_graph_priv *priv = (void *)subdev;
836         u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
837         u32 stat = nv_rd32(priv, 0x400100);
838         u32 addr = nv_rd32(priv, 0x400704);
839         u32 mthd = (addr & 0x00003ffc);
840         u32 subc = (addr & 0x00070000) >> 16;
841         u32 data = nv_rd32(priv, 0x400708);
842         u32 code = nv_rd32(priv, 0x400110);
843         u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
844         int chid;
845
846         engctx = nouveau_engctx_get(engine, inst);
847         chid   = pfifo->chid(pfifo, engctx);
848
849         if (stat & 0x00000010) {
850                 handle = nouveau_handle_get_class(engctx, class);
851                 if (!handle || nv_call(handle->object, mthd, data)) {
852                         nv_error(priv,
853                                  "ILLEGAL_MTHD ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
854                                  chid, inst << 12, nouveau_client_name(engctx),
855                                  subc, class, mthd, data);
856                 }
857                 nouveau_handle_put(handle);
858                 nv_wr32(priv, 0x400100, 0x00000010);
859                 stat &= ~0x00000010;
860         }
861
862         if (stat & 0x00000020) {
863                 nv_error(priv,
864                          "ILLEGAL_CLASS ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
865                          chid, inst << 12, nouveau_client_name(engctx), subc,
866                          class, mthd, data);
867                 nv_wr32(priv, 0x400100, 0x00000020);
868                 stat &= ~0x00000020;
869         }
870
871         if (stat & 0x00100000) {
872                 nv_error(priv, "DATA_ERROR [");
873                 nouveau_enum_print(nv50_data_error_names, code);
874                 pr_cont("] ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
875                         chid, inst << 12, nouveau_client_name(engctx), subc,
876                         class, mthd, data);
877                 nv_wr32(priv, 0x400100, 0x00100000);
878                 stat &= ~0x00100000;
879         }
880
881         if (stat & 0x00200000) {
882                 nv_error(priv, "TRAP ch %d [0x%010llx %s]\n", chid, inst << 12,
883                          nouveau_client_name(engctx));
884                 nvc0_graph_trap_intr(priv);
885                 nv_wr32(priv, 0x400100, 0x00200000);
886                 stat &= ~0x00200000;
887         }
888
889         if (stat & 0x00080000) {
890                 nvc0_graph_ctxctl_isr(priv);
891                 nv_wr32(priv, 0x400100, 0x00080000);
892                 stat &= ~0x00080000;
893         }
894
895         if (stat) {
896                 nv_error(priv, "unknown stat 0x%08x\n", stat);
897                 nv_wr32(priv, 0x400100, stat);
898         }
899
900         nv_wr32(priv, 0x400500, 0x00010001);
901         nouveau_engctx_put(engctx);
902 }
903
904 void
905 nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
906                    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
907 {
908         int i;
909
910         nv_wr32(priv, fuc_base + 0x01c0, 0x01000000);
911         for (i = 0; i < data->size / 4; i++)
912                 nv_wr32(priv, fuc_base + 0x01c4, data->data[i]);
913
914         nv_wr32(priv, fuc_base + 0x0180, 0x01000000);
915         for (i = 0; i < code->size / 4; i++) {
916                 if ((i & 0x3f) == 0)
917                         nv_wr32(priv, fuc_base + 0x0188, i >> 6);
918                 nv_wr32(priv, fuc_base + 0x0184, code->data[i]);
919         }
920
921         /* code must be padded to 0x40 words */
922         for (; i & 0x3f; i++)
923                 nv_wr32(priv, fuc_base + 0x0184, 0);
924 }
925
926 static void
927 nvc0_graph_init_csdata(struct nvc0_graph_priv *priv,
928                        const struct nvc0_graph_pack *pack,
929                        u32 falcon, u32 starstar, u32 base)
930 {
931         const struct nvc0_graph_pack *iter;
932         const struct nvc0_graph_init *init;
933         u32 addr = ~0, prev = ~0, xfer = 0;
934         u32 star, temp;
935
936         nv_wr32(priv, falcon + 0x01c0, 0x02000000 + starstar);
937         star = nv_rd32(priv, falcon + 0x01c4);
938         temp = nv_rd32(priv, falcon + 0x01c4);
939         if (temp > star)
940                 star = temp;
941         nv_wr32(priv, falcon + 0x01c0, 0x01000000 + star);
942
943         pack_for_each_init(init, iter, pack) {
944                 u32 head = init->addr - base;
945                 u32 tail = head + init->count * init->pitch;
946                 while (head < tail) {
947                         if (head != prev + 4 || xfer >= 32) {
948                                 if (xfer) {
949                                         u32 data = ((--xfer << 26) | addr);
950                                         nv_wr32(priv, falcon + 0x01c4, data);
951                                         star += 4;
952                                 }
953                                 addr = head;
954                                 xfer = 0;
955                         }
956                         prev = head;
957                         xfer = xfer + 1;
958                         head = head + init->pitch;
959                 }
960         }
961
962         nv_wr32(priv, falcon + 0x01c4, (--xfer << 26) | addr);
963         nv_wr32(priv, falcon + 0x01c0, 0x01000004 + starstar);
964         nv_wr32(priv, falcon + 0x01c4, star + 4);
965 }
966
967 int
968 nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
969 {
970         struct nvc0_graph_oclass *oclass = (void *)nv_object(priv)->oclass;
971         struct nvc0_grctx_oclass *cclass = (void *)nv_engine(priv)->cclass;
972         u32 r000260;
973         int i;
974
975         if (priv->firmware) {
976                 /* load fuc microcode */
977                 r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
978                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
979                                                    &priv->fuc409d);
980                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
981                                                    &priv->fuc41ad);
982                 nv_wr32(priv, 0x000260, r000260);
983
984                 /* start both of them running */
985                 nv_wr32(priv, 0x409840, 0xffffffff);
986                 nv_wr32(priv, 0x41a10c, 0x00000000);
987                 nv_wr32(priv, 0x40910c, 0x00000000);
988                 nv_wr32(priv, 0x41a100, 0x00000002);
989                 nv_wr32(priv, 0x409100, 0x00000002);
990                 if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
991                         nv_warn(priv, "0x409800 wait failed\n");
992
993                 nv_wr32(priv, 0x409840, 0xffffffff);
994                 nv_wr32(priv, 0x409500, 0x7fffffff);
995                 nv_wr32(priv, 0x409504, 0x00000021);
996
997                 nv_wr32(priv, 0x409840, 0xffffffff);
998                 nv_wr32(priv, 0x409500, 0x00000000);
999                 nv_wr32(priv, 0x409504, 0x00000010);
1000                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1001                         nv_error(priv, "fuc09 req 0x10 timeout\n");
1002                         return -EBUSY;
1003                 }
1004                 priv->size = nv_rd32(priv, 0x409800);
1005
1006                 nv_wr32(priv, 0x409840, 0xffffffff);
1007                 nv_wr32(priv, 0x409500, 0x00000000);
1008                 nv_wr32(priv, 0x409504, 0x00000016);
1009                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1010                         nv_error(priv, "fuc09 req 0x16 timeout\n");
1011                         return -EBUSY;
1012                 }
1013
1014                 nv_wr32(priv, 0x409840, 0xffffffff);
1015                 nv_wr32(priv, 0x409500, 0x00000000);
1016                 nv_wr32(priv, 0x409504, 0x00000025);
1017                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1018                         nv_error(priv, "fuc09 req 0x25 timeout\n");
1019                         return -EBUSY;
1020                 }
1021
1022                 if (nv_device(priv)->chipset >= 0xe0) {
1023                         nv_wr32(priv, 0x409800, 0x00000000);
1024                         nv_wr32(priv, 0x409500, 0x00000001);
1025                         nv_wr32(priv, 0x409504, 0x00000030);
1026                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1027                                 nv_error(priv, "fuc09 req 0x30 timeout\n");
1028                                 return -EBUSY;
1029                         }
1030
1031                         nv_wr32(priv, 0x409810, 0xb00095c8);
1032                         nv_wr32(priv, 0x409800, 0x00000000);
1033                         nv_wr32(priv, 0x409500, 0x00000001);
1034                         nv_wr32(priv, 0x409504, 0x00000031);
1035                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1036                                 nv_error(priv, "fuc09 req 0x31 timeout\n");
1037                                 return -EBUSY;
1038                         }
1039
1040                         nv_wr32(priv, 0x409810, 0x00080420);
1041                         nv_wr32(priv, 0x409800, 0x00000000);
1042                         nv_wr32(priv, 0x409500, 0x00000001);
1043                         nv_wr32(priv, 0x409504, 0x00000032);
1044                         if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1045                                 nv_error(priv, "fuc09 req 0x32 timeout\n");
1046                                 return -EBUSY;
1047                         }
1048
1049                         nv_wr32(priv, 0x409614, 0x00000070);
1050                         nv_wr32(priv, 0x409614, 0x00000770);
1051                         nv_wr32(priv, 0x40802c, 0x00000001);
1052                 }
1053
1054                 if (priv->data == NULL) {
1055                         int ret = nvc0_grctx_generate(priv);
1056                         if (ret) {
1057                                 nv_error(priv, "failed to construct context\n");
1058                                 return ret;
1059                         }
1060                 }
1061
1062                 return 0;
1063         } else
1064         if (!oclass->fecs.ucode) {
1065                 return -ENOSYS;
1066         }
1067
1068         /* load HUB microcode */
1069         r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
1070         nv_wr32(priv, 0x4091c0, 0x01000000);
1071         for (i = 0; i < oclass->fecs.ucode->data.size / 4; i++)
1072                 nv_wr32(priv, 0x4091c4, oclass->fecs.ucode->data.data[i]);
1073
1074         nv_wr32(priv, 0x409180, 0x01000000);
1075         for (i = 0; i < oclass->fecs.ucode->code.size / 4; i++) {
1076                 if ((i & 0x3f) == 0)
1077                         nv_wr32(priv, 0x409188, i >> 6);
1078                 nv_wr32(priv, 0x409184, oclass->fecs.ucode->code.data[i]);
1079         }
1080
1081         /* load GPC microcode */
1082         nv_wr32(priv, 0x41a1c0, 0x01000000);
1083         for (i = 0; i < oclass->gpccs.ucode->data.size / 4; i++)
1084                 nv_wr32(priv, 0x41a1c4, oclass->gpccs.ucode->data.data[i]);
1085
1086         nv_wr32(priv, 0x41a180, 0x01000000);
1087         for (i = 0; i < oclass->gpccs.ucode->code.size / 4; i++) {
1088                 if ((i & 0x3f) == 0)
1089                         nv_wr32(priv, 0x41a188, i >> 6);
1090                 nv_wr32(priv, 0x41a184, oclass->gpccs.ucode->code.data[i]);
1091         }
1092         nv_wr32(priv, 0x000260, r000260);
1093
1094         /* load register lists */
1095         nvc0_graph_init_csdata(priv, cclass->hub, 0x409000, 0x000, 0x000000);
1096         nvc0_graph_init_csdata(priv, cclass->gpc, 0x41a000, 0x000, 0x418000);
1097         nvc0_graph_init_csdata(priv, cclass->tpc, 0x41a000, 0x004, 0x419800);
1098         nvc0_graph_init_csdata(priv, cclass->ppc, 0x41a000, 0x008, 0x41be00);
1099
1100         /* start HUB ucode running, it'll init the GPCs */
1101         nv_wr32(priv, 0x40910c, 0x00000000);
1102         nv_wr32(priv, 0x409100, 0x00000002);
1103         if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
1104                 nv_error(priv, "HUB_INIT timed out\n");
1105                 nvc0_graph_ctxctl_debug(priv);
1106                 return -EBUSY;
1107         }
1108
1109         priv->size = nv_rd32(priv, 0x409804);
1110         if (priv->data == NULL) {
1111                 int ret = nvc0_grctx_generate(priv);
1112                 if (ret) {
1113                         nv_error(priv, "failed to construct context\n");
1114                         return ret;
1115                 }
1116         }
1117
1118         return 0;
1119 }
1120
1121 int
1122 nvc0_graph_init(struct nouveau_object *object)
1123 {
1124         struct nvc0_graph_oclass *oclass = (void *)object->oclass;
1125         struct nvc0_graph_priv *priv = (void *)object;
1126         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
1127         u32 data[TPC_MAX / 8] = {};
1128         u8  tpcnr[GPC_MAX];
1129         int gpc, tpc, rop;
1130         int ret, i;
1131
1132         ret = nouveau_graph_init(&priv->base);
1133         if (ret)
1134                 return ret;
1135
1136         nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
1137         nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
1138         nv_wr32(priv, GPC_BCAST(0x0888), 0x00000000);
1139         nv_wr32(priv, GPC_BCAST(0x088c), 0x00000000);
1140         nv_wr32(priv, GPC_BCAST(0x0890), 0x00000000);
1141         nv_wr32(priv, GPC_BCAST(0x0894), 0x00000000);
1142         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
1143         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
1144
1145         nvc0_graph_mmio(priv, oclass->mmio);
1146
1147         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
1148         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
1149                 do {
1150                         gpc = (gpc + 1) % priv->gpc_nr;
1151                 } while (!tpcnr[gpc]);
1152                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
1153
1154                 data[i / 8] |= tpc << ((i % 8) * 4);
1155         }
1156
1157         nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
1158         nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
1159         nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
1160         nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
1161
1162         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1163                 nv_wr32(priv, GPC_UNIT(gpc, 0x0914),
1164                         priv->magic_not_rop_nr << 8 | priv->tpc_nr[gpc]);
1165                 nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 |
1166                         priv->tpc_total);
1167                 nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
1168         }
1169
1170         if (nv_device(priv)->chipset != 0xd7)
1171                 nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
1172         else
1173                 nv_wr32(priv, GPC_BCAST(0x3fd4), magicgpc918);
1174
1175         nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
1176
1177         nv_wr32(priv, 0x400500, 0x00010001);
1178
1179         nv_wr32(priv, 0x400100, 0xffffffff);
1180         nv_wr32(priv, 0x40013c, 0xffffffff);
1181
1182         nv_wr32(priv, 0x409c24, 0x000f0000);
1183         nv_wr32(priv, 0x404000, 0xc0000000);
1184         nv_wr32(priv, 0x404600, 0xc0000000);
1185         nv_wr32(priv, 0x408030, 0xc0000000);
1186         nv_wr32(priv, 0x40601c, 0xc0000000);
1187         nv_wr32(priv, 0x404490, 0xc0000000);
1188         nv_wr32(priv, 0x406018, 0xc0000000);
1189         nv_wr32(priv, 0x405840, 0xc0000000);
1190         nv_wr32(priv, 0x405844, 0x00ffffff);
1191         nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
1192         nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
1193
1194         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1195                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
1196                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
1197                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
1198                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
1199                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
1200                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
1201                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
1202                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
1203                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
1204                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
1205                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
1206                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
1207                 }
1208                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
1209                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
1210         }
1211
1212         for (rop = 0; rop < priv->rop_nr; rop++) {
1213                 nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
1214                 nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
1215                 nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
1216                 nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
1217         }
1218
1219         nv_wr32(priv, 0x400108, 0xffffffff);
1220         nv_wr32(priv, 0x400138, 0xffffffff);
1221         nv_wr32(priv, 0x400118, 0xffffffff);
1222         nv_wr32(priv, 0x400130, 0xffffffff);
1223         nv_wr32(priv, 0x40011c, 0xffffffff);
1224         nv_wr32(priv, 0x400134, 0xffffffff);
1225
1226         nv_wr32(priv, 0x400054, 0x34ce3464);
1227         return nvc0_graph_init_ctxctl(priv);
1228 }
1229
1230 static void
1231 nvc0_graph_dtor_fw(struct nvc0_graph_fuc *fuc)
1232 {
1233         kfree(fuc->data);
1234         fuc->data = NULL;
1235 }
1236
1237 int
1238 nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
1239                    struct nvc0_graph_fuc *fuc)
1240 {
1241         struct nouveau_device *device = nv_device(priv);
1242         const struct firmware *fw;
1243         char f[32];
1244         int ret;
1245
1246         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
1247         ret = request_firmware(&fw, f, nv_device_base(device));
1248         if (ret) {
1249                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
1250                 ret = request_firmware(&fw, f, nv_device_base(device));
1251                 if (ret) {
1252                         nv_error(priv, "failed to load %s\n", fwname);
1253                         return ret;
1254                 }
1255         }
1256
1257         fuc->size = fw->size;
1258         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
1259         release_firmware(fw);
1260         return (fuc->data != NULL) ? 0 : -ENOMEM;
1261 }
1262
1263 void
1264 nvc0_graph_dtor(struct nouveau_object *object)
1265 {
1266         struct nvc0_graph_priv *priv = (void *)object;
1267
1268         kfree(priv->data);
1269
1270         nvc0_graph_dtor_fw(&priv->fuc409c);
1271         nvc0_graph_dtor_fw(&priv->fuc409d);
1272         nvc0_graph_dtor_fw(&priv->fuc41ac);
1273         nvc0_graph_dtor_fw(&priv->fuc41ad);
1274
1275         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
1276         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
1277
1278         nouveau_graph_destroy(&priv->base);
1279 }
1280
1281 int
1282 nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
1283                 struct nouveau_oclass *bclass, void *data, u32 size,
1284                 struct nouveau_object **pobject)
1285 {
1286         struct nvc0_graph_oclass *oclass = (void *)bclass;
1287         struct nouveau_device *device = nv_device(parent);
1288         struct nvc0_graph_priv *priv;
1289         bool use_ext_fw, enable;
1290         int ret, i;
1291
1292         use_ext_fw = nouveau_boolopt(device->cfgopt, "NvGrUseFW",
1293                                      oclass->fecs.ucode == NULL);
1294         enable = use_ext_fw || oclass->fecs.ucode != NULL;
1295
1296         ret = nouveau_graph_create(parent, engine, bclass, enable, &priv);
1297         *pobject = nv_object(priv);
1298         if (ret)
1299                 return ret;
1300
1301         nv_subdev(priv)->unit = 0x08001000;
1302         nv_subdev(priv)->intr = nvc0_graph_intr;
1303
1304         priv->base.units = nvc0_graph_units;
1305
1306         if (use_ext_fw) {
1307                 nv_info(priv, "using external firmware\n");
1308                 if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
1309                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
1310                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
1311                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
1312                         return -EINVAL;
1313                 priv->firmware = true;
1314         }
1315
1316         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
1317                                 &priv->unk4188b4);
1318         if (ret)
1319                 return ret;
1320
1321         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
1322                                 &priv->unk4188b8);
1323         if (ret)
1324                 return ret;
1325
1326         for (i = 0; i < 0x1000; i += 4) {
1327                 nv_wo32(priv->unk4188b4, i, 0x00000010);
1328                 nv_wo32(priv->unk4188b8, i, 0x00000010);
1329         }
1330
1331         priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
1332         priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
1333         for (i = 0; i < priv->gpc_nr; i++) {
1334                 priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
1335                 priv->tpc_total += priv->tpc_nr[i];
1336         }
1337
1338         /*XXX: these need figuring out... though it might not even matter */
1339         switch (nv_device(priv)->chipset) {
1340         case 0xc0:
1341                 if (priv->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
1342                         priv->magic_not_rop_nr = 0x07;
1343                 } else
1344                 if (priv->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
1345                         priv->magic_not_rop_nr = 0x05;
1346                 } else
1347                 if (priv->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
1348                         priv->magic_not_rop_nr = 0x06;
1349                 }
1350                 break;
1351         case 0xc3: /* 450, 4/0/0/0, 2 */
1352                 priv->magic_not_rop_nr = 0x03;
1353                 break;
1354         case 0xc4: /* 460, 3/4/0/0, 4 */
1355                 priv->magic_not_rop_nr = 0x01;
1356                 break;
1357         case 0xc1: /* 2/0/0/0, 1 */
1358                 priv->magic_not_rop_nr = 0x01;
1359                 break;
1360         case 0xc8: /* 4/4/3/4, 5 */
1361                 priv->magic_not_rop_nr = 0x06;
1362                 break;
1363         case 0xce: /* 4/4/0/0, 4 */
1364                 priv->magic_not_rop_nr = 0x03;
1365                 break;
1366         case 0xcf: /* 4/0/0/0, 3 */
1367                 priv->magic_not_rop_nr = 0x03;
1368                 break;
1369         case 0xd7:
1370         case 0xd9: /* 1/0/0/0, 1 */
1371                 priv->magic_not_rop_nr = 0x01;
1372                 break;
1373         }
1374
1375         nv_engine(priv)->cclass = *oclass->cclass;
1376         nv_engine(priv)->sclass =  oclass->sclass;
1377         return 0;
1378 }
1379
1380 #include "fuc/hubnvc0.fuc.h"
1381
1382 struct nvc0_graph_ucode
1383 nvc0_graph_fecs_ucode = {
1384         .code.data = nvc0_grhub_code,
1385         .code.size = sizeof(nvc0_grhub_code),
1386         .data.data = nvc0_grhub_data,
1387         .data.size = sizeof(nvc0_grhub_data),
1388 };
1389
1390 #include "fuc/gpcnvc0.fuc.h"
1391
1392 struct nvc0_graph_ucode
1393 nvc0_graph_gpccs_ucode = {
1394         .code.data = nvc0_grgpc_code,
1395         .code.size = sizeof(nvc0_grgpc_code),
1396         .data.data = nvc0_grgpc_data,
1397         .data.size = sizeof(nvc0_grgpc_data),
1398 };
1399
1400 struct nouveau_oclass *
1401 nvc0_graph_oclass = &(struct nvc0_graph_oclass) {
1402         .base.handle = NV_ENGINE(GR, 0xc0),
1403         .base.ofuncs = &(struct nouveau_ofuncs) {
1404                 .ctor = nvc0_graph_ctor,
1405                 .dtor = nvc0_graph_dtor,
1406                 .init = nvc0_graph_init,
1407                 .fini = _nouveau_graph_fini,
1408         },
1409         .cclass = &nvc0_grctx_oclass,
1410         .sclass =  nvc0_graph_sclass,
1411         .mmio = nvc0_graph_pack_mmio,
1412         .fecs.ucode = &nvc0_graph_fecs_ucode,
1413         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
1414 }.base;