drm/nv50: implement global channel address space on new VM code
[cascardo/linux.git] / drivers / gpu / drm / nouveau / nouveau_state.c
index 989322b..e0811f9 100644 (file)
@@ -35,6 +35,8 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 #include "nouveau_fbcon.h"
+#include "nouveau_ramht.h"
+#include "nouveau_pm.h"
 #include "nv50_display.h"
 
 static void nouveau_stub_takedown(struct drm_device *dev) {}
@@ -51,10 +53,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->instmem.takedown        = nv04_instmem_takedown;
                engine->instmem.suspend         = nv04_instmem_suspend;
                engine->instmem.resume          = nv04_instmem_resume;
-               engine->instmem.populate        = nv04_instmem_populate;
-               engine->instmem.clear           = nv04_instmem_clear;
-               engine->instmem.bind            = nv04_instmem_bind;
-               engine->instmem.unbind          = nv04_instmem_unbind;
+               engine->instmem.get             = nv04_instmem_get;
+               engine->instmem.put             = nv04_instmem_put;
+               engine->instmem.map             = nv04_instmem_map;
+               engine->instmem.unmap           = nv04_instmem_unmap;
                engine->instmem.flush           = nv04_instmem_flush;
                engine->mc.init                 = nv04_mc_init;
                engine->mc.takedown             = nv04_mc_takedown;
@@ -63,7 +65,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nv04_fb_init;
                engine->fb.takedown             = nv04_fb_takedown;
-               engine->graph.grclass           = nv04_graph_grclass;
                engine->graph.init              = nv04_graph_init;
                engine->graph.takedown          = nv04_graph_takedown;
                engine->graph.fifo_access       = nv04_graph_fifo_access;
@@ -74,11 +75,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->graph.unload_context    = nv04_graph_unload_context;
                engine->fifo.channels           = 16;
                engine->fifo.init               = nv04_fifo_init;
-               engine->fifo.takedown           = nouveau_stub_takedown;
+               engine->fifo.takedown           = nv04_fifo_fini;
                engine->fifo.disable            = nv04_fifo_disable;
                engine->fifo.enable             = nv04_fifo_enable;
                engine->fifo.reassign           = nv04_fifo_reassign;
-               engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                engine->fifo.channel_id         = nv04_fifo_channel_id;
                engine->fifo.create_context     = nv04_fifo_create_context;
@@ -95,16 +95,21 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.get                = NULL;
                engine->gpio.set                = NULL;
                engine->gpio.irq_enable         = NULL;
+               engine->pm.clock_get            = nv04_pm_clock_get;
+               engine->pm.clock_pre            = nv04_pm_clock_pre;
+               engine->pm.clock_set            = nv04_pm_clock_set;
+               engine->crypt.init              = nouveau_stub_init;
+               engine->crypt.takedown          = nouveau_stub_takedown;
                break;
        case 0x10:
                engine->instmem.init            = nv04_instmem_init;
                engine->instmem.takedown        = nv04_instmem_takedown;
                engine->instmem.suspend         = nv04_instmem_suspend;
                engine->instmem.resume          = nv04_instmem_resume;
-               engine->instmem.populate        = nv04_instmem_populate;
-               engine->instmem.clear           = nv04_instmem_clear;
-               engine->instmem.bind            = nv04_instmem_bind;
-               engine->instmem.unbind          = nv04_instmem_unbind;
+               engine->instmem.get             = nv04_instmem_get;
+               engine->instmem.put             = nv04_instmem_put;
+               engine->instmem.map             = nv04_instmem_map;
+               engine->instmem.unmap           = nv04_instmem_unmap;
                engine->instmem.flush           = nv04_instmem_flush;
                engine->mc.init                 = nv04_mc_init;
                engine->mc.takedown             = nv04_mc_takedown;
@@ -113,8 +118,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nv10_fb_init;
                engine->fb.takedown             = nv10_fb_takedown;
-               engine->fb.set_region_tiling    = nv10_fb_set_region_tiling;
-               engine->graph.grclass           = nv10_graph_grclass;
+               engine->fb.init_tile_region     = nv10_fb_init_tile_region;
+               engine->fb.set_tile_region      = nv10_fb_set_tile_region;
+               engine->fb.free_tile_region     = nv10_fb_free_tile_region;
                engine->graph.init              = nv10_graph_init;
                engine->graph.takedown          = nv10_graph_takedown;
                engine->graph.channel           = nv10_graph_channel;
@@ -123,18 +129,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->graph.fifo_access       = nv04_graph_fifo_access;
                engine->graph.load_context      = nv10_graph_load_context;
                engine->graph.unload_context    = nv10_graph_unload_context;
-               engine->graph.set_region_tiling = nv10_graph_set_region_tiling;
+               engine->graph.set_tile_region   = nv10_graph_set_tile_region;
                engine->fifo.channels           = 32;
                engine->fifo.init               = nv10_fifo_init;
-               engine->fifo.takedown           = nouveau_stub_takedown;
+               engine->fifo.takedown           = nv04_fifo_fini;
                engine->fifo.disable            = nv04_fifo_disable;
                engine->fifo.enable             = nv04_fifo_enable;
                engine->fifo.reassign           = nv04_fifo_reassign;
-               engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                engine->fifo.channel_id         = nv10_fifo_channel_id;
                engine->fifo.create_context     = nv10_fifo_create_context;
-               engine->fifo.destroy_context    = nv10_fifo_destroy_context;
+               engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                engine->fifo.load_context       = nv10_fifo_load_context;
                engine->fifo.unload_context     = nv10_fifo_unload_context;
                engine->display.early_init      = nv04_display_early_init;
@@ -147,16 +152,21 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.get                = nv10_gpio_get;
                engine->gpio.set                = nv10_gpio_set;
                engine->gpio.irq_enable         = NULL;
+               engine->pm.clock_get            = nv04_pm_clock_get;
+               engine->pm.clock_pre            = nv04_pm_clock_pre;
+               engine->pm.clock_set            = nv04_pm_clock_set;
+               engine->crypt.init              = nouveau_stub_init;
+               engine->crypt.takedown          = nouveau_stub_takedown;
                break;
        case 0x20:
                engine->instmem.init            = nv04_instmem_init;
                engine->instmem.takedown        = nv04_instmem_takedown;
                engine->instmem.suspend         = nv04_instmem_suspend;
                engine->instmem.resume          = nv04_instmem_resume;
-               engine->instmem.populate        = nv04_instmem_populate;
-               engine->instmem.clear           = nv04_instmem_clear;
-               engine->instmem.bind            = nv04_instmem_bind;
-               engine->instmem.unbind          = nv04_instmem_unbind;
+               engine->instmem.get             = nv04_instmem_get;
+               engine->instmem.put             = nv04_instmem_put;
+               engine->instmem.map             = nv04_instmem_map;
+               engine->instmem.unmap           = nv04_instmem_unmap;
                engine->instmem.flush           = nv04_instmem_flush;
                engine->mc.init                 = nv04_mc_init;
                engine->mc.takedown             = nv04_mc_takedown;
@@ -165,8 +175,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nv10_fb_init;
                engine->fb.takedown             = nv10_fb_takedown;
-               engine->fb.set_region_tiling    = nv10_fb_set_region_tiling;
-               engine->graph.grclass           = nv20_graph_grclass;
+               engine->fb.init_tile_region     = nv10_fb_init_tile_region;
+               engine->fb.set_tile_region      = nv10_fb_set_tile_region;
+               engine->fb.free_tile_region     = nv10_fb_free_tile_region;
                engine->graph.init              = nv20_graph_init;
                engine->graph.takedown          = nv20_graph_takedown;
                engine->graph.channel           = nv10_graph_channel;
@@ -175,18 +186,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->graph.fifo_access       = nv04_graph_fifo_access;
                engine->graph.load_context      = nv20_graph_load_context;
                engine->graph.unload_context    = nv20_graph_unload_context;
-               engine->graph.set_region_tiling = nv20_graph_set_region_tiling;
+               engine->graph.set_tile_region   = nv20_graph_set_tile_region;
                engine->fifo.channels           = 32;
                engine->fifo.init               = nv10_fifo_init;
-               engine->fifo.takedown           = nouveau_stub_takedown;
+               engine->fifo.takedown           = nv04_fifo_fini;
                engine->fifo.disable            = nv04_fifo_disable;
                engine->fifo.enable             = nv04_fifo_enable;
                engine->fifo.reassign           = nv04_fifo_reassign;
-               engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                engine->fifo.channel_id         = nv10_fifo_channel_id;
                engine->fifo.create_context     = nv10_fifo_create_context;
-               engine->fifo.destroy_context    = nv10_fifo_destroy_context;
+               engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                engine->fifo.load_context       = nv10_fifo_load_context;
                engine->fifo.unload_context     = nv10_fifo_unload_context;
                engine->display.early_init      = nv04_display_early_init;
@@ -199,16 +209,21 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.get                = nv10_gpio_get;
                engine->gpio.set                = nv10_gpio_set;
                engine->gpio.irq_enable         = NULL;
+               engine->pm.clock_get            = nv04_pm_clock_get;
+               engine->pm.clock_pre            = nv04_pm_clock_pre;
+               engine->pm.clock_set            = nv04_pm_clock_set;
+               engine->crypt.init              = nouveau_stub_init;
+               engine->crypt.takedown          = nouveau_stub_takedown;
                break;
        case 0x30:
                engine->instmem.init            = nv04_instmem_init;
                engine->instmem.takedown        = nv04_instmem_takedown;
                engine->instmem.suspend         = nv04_instmem_suspend;
                engine->instmem.resume          = nv04_instmem_resume;
-               engine->instmem.populate        = nv04_instmem_populate;
-               engine->instmem.clear           = nv04_instmem_clear;
-               engine->instmem.bind            = nv04_instmem_bind;
-               engine->instmem.unbind          = nv04_instmem_unbind;
+               engine->instmem.get             = nv04_instmem_get;
+               engine->instmem.put             = nv04_instmem_put;
+               engine->instmem.map             = nv04_instmem_map;
+               engine->instmem.unmap           = nv04_instmem_unmap;
                engine->instmem.flush           = nv04_instmem_flush;
                engine->mc.init                 = nv04_mc_init;
                engine->mc.takedown             = nv04_mc_takedown;
@@ -217,8 +232,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nv30_fb_init;
                engine->fb.takedown             = nv30_fb_takedown;
-               engine->fb.set_region_tiling    = nv10_fb_set_region_tiling;
-               engine->graph.grclass           = nv30_graph_grclass;
+               engine->fb.init_tile_region     = nv30_fb_init_tile_region;
+               engine->fb.set_tile_region      = nv10_fb_set_tile_region;
+               engine->fb.free_tile_region     = nv30_fb_free_tile_region;
                engine->graph.init              = nv30_graph_init;
                engine->graph.takedown          = nv20_graph_takedown;
                engine->graph.fifo_access       = nv04_graph_fifo_access;
@@ -227,18 +243,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->graph.destroy_context   = nv20_graph_destroy_context;
                engine->graph.load_context      = nv20_graph_load_context;
                engine->graph.unload_context    = nv20_graph_unload_context;
-               engine->graph.set_region_tiling = nv20_graph_set_region_tiling;
+               engine->graph.set_tile_region   = nv20_graph_set_tile_region;
                engine->fifo.channels           = 32;
                engine->fifo.init               = nv10_fifo_init;
-               engine->fifo.takedown           = nouveau_stub_takedown;
+               engine->fifo.takedown           = nv04_fifo_fini;
                engine->fifo.disable            = nv04_fifo_disable;
                engine->fifo.enable             = nv04_fifo_enable;
                engine->fifo.reassign           = nv04_fifo_reassign;
-               engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                engine->fifo.channel_id         = nv10_fifo_channel_id;
                engine->fifo.create_context     = nv10_fifo_create_context;
-               engine->fifo.destroy_context    = nv10_fifo_destroy_context;
+               engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                engine->fifo.load_context       = nv10_fifo_load_context;
                engine->fifo.unload_context     = nv10_fifo_unload_context;
                engine->display.early_init      = nv04_display_early_init;
@@ -251,6 +266,13 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.get                = nv10_gpio_get;
                engine->gpio.set                = nv10_gpio_set;
                engine->gpio.irq_enable         = NULL;
+               engine->pm.clock_get            = nv04_pm_clock_get;
+               engine->pm.clock_pre            = nv04_pm_clock_pre;
+               engine->pm.clock_set            = nv04_pm_clock_set;
+               engine->pm.voltage_get          = nouveau_voltage_gpio_get;
+               engine->pm.voltage_set          = nouveau_voltage_gpio_set;
+               engine->crypt.init              = nouveau_stub_init;
+               engine->crypt.takedown          = nouveau_stub_takedown;
                break;
        case 0x40:
        case 0x60:
@@ -258,10 +280,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->instmem.takedown        = nv04_instmem_takedown;
                engine->instmem.suspend         = nv04_instmem_suspend;
                engine->instmem.resume          = nv04_instmem_resume;
-               engine->instmem.populate        = nv04_instmem_populate;
-               engine->instmem.clear           = nv04_instmem_clear;
-               engine->instmem.bind            = nv04_instmem_bind;
-               engine->instmem.unbind          = nv04_instmem_unbind;
+               engine->instmem.get             = nv04_instmem_get;
+               engine->instmem.put             = nv04_instmem_put;
+               engine->instmem.map             = nv04_instmem_map;
+               engine->instmem.unmap           = nv04_instmem_unmap;
                engine->instmem.flush           = nv04_instmem_flush;
                engine->mc.init                 = nv40_mc_init;
                engine->mc.takedown             = nv40_mc_takedown;
@@ -270,8 +292,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nv40_fb_init;
                engine->fb.takedown             = nv40_fb_takedown;
-               engine->fb.set_region_tiling    = nv40_fb_set_region_tiling;
-               engine->graph.grclass           = nv40_graph_grclass;
+               engine->fb.init_tile_region     = nv30_fb_init_tile_region;
+               engine->fb.set_tile_region      = nv40_fb_set_tile_region;
+               engine->fb.free_tile_region     = nv30_fb_free_tile_region;
                engine->graph.init              = nv40_graph_init;
                engine->graph.takedown          = nv40_graph_takedown;
                engine->graph.fifo_access       = nv04_graph_fifo_access;
@@ -280,18 +303,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->graph.destroy_context   = nv40_graph_destroy_context;
                engine->graph.load_context      = nv40_graph_load_context;
                engine->graph.unload_context    = nv40_graph_unload_context;
-               engine->graph.set_region_tiling = nv40_graph_set_region_tiling;
+               engine->graph.set_tile_region   = nv40_graph_set_tile_region;
                engine->fifo.channels           = 32;
                engine->fifo.init               = nv40_fifo_init;
-               engine->fifo.takedown           = nouveau_stub_takedown;
+               engine->fifo.takedown           = nv04_fifo_fini;
                engine->fifo.disable            = nv04_fifo_disable;
                engine->fifo.enable             = nv04_fifo_enable;
                engine->fifo.reassign           = nv04_fifo_reassign;
-               engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                engine->fifo.channel_id         = nv10_fifo_channel_id;
                engine->fifo.create_context     = nv40_fifo_create_context;
-               engine->fifo.destroy_context    = nv40_fifo_destroy_context;
+               engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                engine->fifo.load_context       = nv40_fifo_load_context;
                engine->fifo.unload_context     = nv40_fifo_unload_context;
                engine->display.early_init      = nv04_display_early_init;
@@ -304,6 +326,14 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.get                = nv10_gpio_get;
                engine->gpio.set                = nv10_gpio_set;
                engine->gpio.irq_enable         = NULL;
+               engine->pm.clock_get            = nv04_pm_clock_get;
+               engine->pm.clock_pre            = nv04_pm_clock_pre;
+               engine->pm.clock_set            = nv04_pm_clock_set;
+               engine->pm.voltage_get          = nouveau_voltage_gpio_get;
+               engine->pm.voltage_set          = nouveau_voltage_gpio_set;
+               engine->pm.temp_get             = nv40_temp_get;
+               engine->crypt.init              = nouveau_stub_init;
+               engine->crypt.takedown          = nouveau_stub_takedown;
                break;
        case 0x50:
        case 0x80: /* gotta love NVIDIA's consistency.. */
@@ -313,10 +343,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->instmem.takedown        = nv50_instmem_takedown;
                engine->instmem.suspend         = nv50_instmem_suspend;
                engine->instmem.resume          = nv50_instmem_resume;
-               engine->instmem.populate        = nv50_instmem_populate;
-               engine->instmem.clear           = nv50_instmem_clear;
-               engine->instmem.bind            = nv50_instmem_bind;
-               engine->instmem.unbind          = nv50_instmem_unbind;
+               engine->instmem.get             = nv50_instmem_get;
+               engine->instmem.put             = nv50_instmem_put;
+               engine->instmem.map             = nv50_instmem_map;
+               engine->instmem.unmap           = nv50_instmem_unmap;
                if (dev_priv->chipset == 0x50)
                        engine->instmem.flush   = nv50_instmem_flush;
                else
@@ -328,7 +358,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nv50_fb_init;
                engine->fb.takedown             = nv50_fb_takedown;
-               engine->graph.grclass           = nv50_graph_grclass;
                engine->graph.init              = nv50_graph_init;
                engine->graph.takedown          = nv50_graph_takedown;
                engine->graph.fifo_access       = nv50_graph_fifo_access;
@@ -337,6 +366,15 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->graph.destroy_context   = nv50_graph_destroy_context;
                engine->graph.load_context      = nv50_graph_load_context;
                engine->graph.unload_context    = nv50_graph_unload_context;
+               if (dev_priv->chipset != 0x86)
+                       engine->graph.tlb_flush = nv50_graph_tlb_flush;
+               else {
+                       /* from what i can see nvidia do this on every
+                        * pre-NVA3 board except NVAC, but, we've only
+                        * ever seen problems on NV86
+                        */
+                       engine->graph.tlb_flush = nv86_graph_tlb_flush;
+               }
                engine->fifo.channels           = 128;
                engine->fifo.init               = nv50_fifo_init;
                engine->fifo.takedown           = nv50_fifo_takedown;
@@ -348,26 +386,74 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->fifo.destroy_context    = nv50_fifo_destroy_context;
                engine->fifo.load_context       = nv50_fifo_load_context;
                engine->fifo.unload_context     = nv50_fifo_unload_context;
+               engine->fifo.tlb_flush          = nv50_fifo_tlb_flush;
                engine->display.early_init      = nv50_display_early_init;
                engine->display.late_takedown   = nv50_display_late_takedown;
                engine->display.create          = nv50_display_create;
                engine->display.init            = nv50_display_init;
                engine->display.destroy         = nv50_display_destroy;
                engine->gpio.init               = nv50_gpio_init;
-               engine->gpio.takedown           = nouveau_stub_takedown;
+               engine->gpio.takedown           = nv50_gpio_fini;
                engine->gpio.get                = nv50_gpio_get;
                engine->gpio.set                = nv50_gpio_set;
+               engine->gpio.irq_register       = nv50_gpio_irq_register;
+               engine->gpio.irq_unregister     = nv50_gpio_irq_unregister;
                engine->gpio.irq_enable         = nv50_gpio_irq_enable;
+               switch (dev_priv->chipset) {
+               case 0x84:
+               case 0x86:
+               case 0x92:
+               case 0x94:
+               case 0x96:
+               case 0x98:
+               case 0xa0:
+               case 0xaa:
+               case 0xac:
+               case 0x50:
+                       engine->pm.clock_get    = nv50_pm_clock_get;
+                       engine->pm.clock_pre    = nv50_pm_clock_pre;
+                       engine->pm.clock_set    = nv50_pm_clock_set;
+                       break;
+               default:
+                       engine->pm.clock_get    = nva3_pm_clock_get;
+                       engine->pm.clock_pre    = nva3_pm_clock_pre;
+                       engine->pm.clock_set    = nva3_pm_clock_set;
+                       break;
+               }
+               engine->pm.voltage_get          = nouveau_voltage_gpio_get;
+               engine->pm.voltage_set          = nouveau_voltage_gpio_set;
+               if (dev_priv->chipset >= 0x84)
+                       engine->pm.temp_get     = nv84_temp_get;
+               else
+                       engine->pm.temp_get     = nv40_temp_get;
+               switch (dev_priv->chipset) {
+               case 0x84:
+               case 0x86:
+               case 0x92:
+               case 0x94:
+               case 0x96:
+               case 0xa0:
+                       engine->crypt.init      = nv84_crypt_init;
+                       engine->crypt.takedown  = nv84_crypt_fini;
+                       engine->crypt.create_context = nv84_crypt_create_context;
+                       engine->crypt.destroy_context = nv84_crypt_destroy_context;
+                       engine->crypt.tlb_flush = nv84_crypt_tlb_flush;
+                       break;
+               default:
+                       engine->crypt.init      = nouveau_stub_init;
+                       engine->crypt.takedown  = nouveau_stub_takedown;
+                       break;
+               }
                break;
        case 0xC0:
                engine->instmem.init            = nvc0_instmem_init;
                engine->instmem.takedown        = nvc0_instmem_takedown;
                engine->instmem.suspend         = nvc0_instmem_suspend;
                engine->instmem.resume          = nvc0_instmem_resume;
-               engine->instmem.populate        = nvc0_instmem_populate;
-               engine->instmem.clear           = nvc0_instmem_clear;
-               engine->instmem.bind            = nvc0_instmem_bind;
-               engine->instmem.unbind          = nvc0_instmem_unbind;
+               engine->instmem.get             = nvc0_instmem_get;
+               engine->instmem.put             = nvc0_instmem_put;
+               engine->instmem.map             = nvc0_instmem_map;
+               engine->instmem.unmap           = nvc0_instmem_unmap;
                engine->instmem.flush           = nvc0_instmem_flush;
                engine->mc.init                 = nv50_mc_init;
                engine->mc.takedown             = nv50_mc_takedown;
@@ -376,7 +462,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->timer.takedown          = nv04_timer_takedown;
                engine->fb.init                 = nvc0_fb_init;
                engine->fb.takedown             = nvc0_fb_takedown;
-               engine->graph.grclass           = NULL;  //nvc0_graph_grclass;
                engine->graph.init              = nvc0_graph_init;
                engine->graph.takedown          = nvc0_graph_takedown;
                engine->graph.fifo_access       = nvc0_graph_fifo_access;
@@ -405,7 +490,11 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.takedown           = nouveau_stub_takedown;
                engine->gpio.get                = nv50_gpio_get;
                engine->gpio.set                = nv50_gpio_set;
+               engine->gpio.irq_register       = nv50_gpio_irq_register;
+               engine->gpio.irq_unregister     = nv50_gpio_irq_unregister;
                engine->gpio.irq_enable         = nv50_gpio_irq_enable;
+               engine->crypt.init              = nouveau_stub_init;
+               engine->crypt.takedown          = nouveau_stub_takedown;
                break;
        default:
                NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset);
@@ -437,45 +526,43 @@ static int
 nouveau_card_init_channel(struct drm_device *dev)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
-       struct nouveau_gpuobj *gpuobj;
+       struct nouveau_gpuobj *gpuobj = NULL;
        int ret;
 
        ret = nouveau_channel_alloc(dev, &dev_priv->channel,
-                                   (struct drm_file *)-2,
-                                   NvDmaFB, NvDmaTT);
+                                   (struct drm_file *)-2, NvDmaFB, NvDmaTT);
        if (ret)
                return ret;
 
-       gpuobj = NULL;
        ret = nouveau_gpuobj_dma_new(dev_priv->channel, NV_CLASS_DMA_IN_MEMORY,
                                     0, dev_priv->vram_size,
-                                    NV_DMA_ACCESS_RW, NV_DMA_TARGET_VIDMEM,
+                                    NV_MEM_ACCESS_RW, NV_MEM_TARGET_VRAM,
                                     &gpuobj);
        if (ret)
                goto out_err;
 
-       ret = nouveau_gpuobj_ref_add(dev, dev_priv->channel, NvDmaVRAM,
-                                    gpuobj, NULL);
+       ret = nouveau_ramht_insert(dev_priv->channel, NvDmaVRAM, gpuobj);
+       nouveau_gpuobj_ref(NULL, &gpuobj);
        if (ret)
                goto out_err;
 
-       gpuobj = NULL;
-       ret = nouveau_gpuobj_gart_dma_new(dev_priv->channel, 0,
-                                         dev_priv->gart_info.aper_size,
-                                         NV_DMA_ACCESS_RW, &gpuobj, NULL);
+       ret = nouveau_gpuobj_dma_new(dev_priv->channel, NV_CLASS_DMA_IN_MEMORY,
+                                    0, dev_priv->gart_info.aper_size,
+                                    NV_MEM_ACCESS_RW, NV_MEM_TARGET_GART,
+                                    &gpuobj);
        if (ret)
                goto out_err;
 
-       ret = nouveau_gpuobj_ref_add(dev, dev_priv->channel, NvDmaGART,
-                                    gpuobj, NULL);
+       ret = nouveau_ramht_insert(dev_priv->channel, NvDmaGART, gpuobj);
+       nouveau_gpuobj_ref(NULL, &gpuobj);
        if (ret)
                goto out_err;
 
+       mutex_unlock(&dev_priv->channel->mutex);
        return 0;
+
 out_err:
-       nouveau_gpuobj_del(dev, &gpuobj);
-       nouveau_channel_free(dev_priv->channel);
-       dev_priv->channel = NULL;
+       nouveau_channel_put(&dev_priv->channel);
        return ret;
 }
 
@@ -522,6 +609,8 @@ nouveau_card_init(struct drm_device *dev)
        if (ret)
                goto out;
        engine = &dev_priv->engine;
+       spin_lock_init(&dev_priv->channels.lock);
+       spin_lock_init(&dev_priv->tile.lock);
        spin_lock_init(&dev_priv->context_switch_lock);
 
        /* Make the CRTCs and I2C buses accessible */
@@ -534,35 +623,28 @@ nouveau_card_init(struct drm_device *dev)
        if (ret)
                goto out_display_early;
 
-       ret = nouveau_mem_detect(dev);
+       nouveau_pm_init(dev);
+
+       ret = nouveau_mem_vram_init(dev);
        if (ret)
                goto out_bios;
 
-       ret = nouveau_gpuobj_early_init(dev);
+       ret = nouveau_gpuobj_init(dev);
        if (ret)
-               goto out_bios;
+               goto out_vram;
 
-       /* Initialise instance memory, must happen before mem_init so we
-        * know exactly how much VRAM we're able to use for "normal"
-        * purposes.
-        */
        ret = engine->instmem.init(dev);
        if (ret)
-               goto out_gpuobj_early;
+               goto out_gpuobj;
 
-       /* Setup the memory manager */
-       ret = nouveau_mem_init(dev);
+       ret = nouveau_mem_gart_init(dev);
        if (ret)
                goto out_instmem;
 
-       ret = nouveau_gpuobj_init(dev);
-       if (ret)
-               goto out_mem;
-
        /* PMC */
        ret = engine->mc.init(dev);
        if (ret)
-               goto out_gpuobj;
+               goto out_gart;
 
        /* PGPIO */
        ret = engine->gpio.init(dev);
@@ -587,33 +669,39 @@ nouveau_card_init(struct drm_device *dev)
                if (ret)
                        goto out_fb;
 
+               /* PCRYPT */
+               ret = engine->crypt.init(dev);
+               if (ret)
+                       goto out_graph;
+
                /* PFIFO */
                ret = engine->fifo.init(dev);
                if (ret)
-                       goto out_graph;
+                       goto out_crypt;
        }
 
        ret = engine->display.create(dev);
        if (ret)
                goto out_fifo;
 
-       /* this call irq_preinstall, register irq handler and
-        * call irq_postinstall
-        */
-       ret = drm_irq_install(dev);
+       ret = drm_vblank_init(dev, nv_two_heads(dev) ? 2 : 1);
        if (ret)
-               goto out_display;
+               goto out_vblank;
 
-       ret = drm_vblank_init(dev, 0);
+       ret = nouveau_irq_init(dev);
        if (ret)
-               goto out_irq;
+               goto out_vblank;
 
        /* what about PVIDEO/PCRTC/PRAMDAC etc? */
 
        if (!engine->graph.accel_blocked) {
-               ret = nouveau_card_init_channel(dev);
+               ret = nouveau_fence_init(dev);
                if (ret)
                        goto out_irq;
+
+               ret = nouveau_card_init_channel(dev);
+               if (ret)
+                       goto out_fence;
        }
 
        ret = nouveau_backlight_init(dev);
@@ -624,13 +712,19 @@ nouveau_card_init(struct drm_device *dev)
        drm_kms_helper_poll_init(dev);
        return 0;
 
+out_fence:
+       nouveau_fence_fini(dev);
 out_irq:
-       drm_irq_uninstall(dev);
-out_display:
+       nouveau_irq_fini(dev);
+out_vblank:
+       drm_vblank_cleanup(dev);
        engine->display.destroy(dev);
 out_fifo:
        if (!nouveau_noaccel)
                engine->fifo.takedown(dev);
+out_crypt:
+       if (!nouveau_noaccel)
+               engine->crypt.takedown(dev);
 out_graph:
        if (!nouveau_noaccel)
                engine->graph.takedown(dev);
@@ -642,16 +736,16 @@ out_gpio:
        engine->gpio.takedown(dev);
 out_mc:
        engine->mc.takedown(dev);
-out_gpuobj:
-       nouveau_gpuobj_takedown(dev);
-out_mem:
-       nouveau_sgdma_takedown(dev);
-       nouveau_mem_close(dev);
+out_gart:
+       nouveau_mem_gart_fini(dev);
 out_instmem:
        engine->instmem.takedown(dev);
-out_gpuobj_early:
-       nouveau_gpuobj_late_takedown(dev);
+out_gpuobj:
+       nouveau_gpuobj_takedown(dev);
+out_vram:
+       nouveau_mem_vram_fini(dev);
 out_bios:
+       nouveau_pm_fini(dev);
        nouveau_bios_takedown(dev);
 out_display_early:
        engine->display.late_takedown(dev);
@@ -667,13 +761,14 @@ static void nouveau_card_takedown(struct drm_device *dev)
 
        nouveau_backlight_exit(dev);
 
-       if (dev_priv->channel) {
-               nouveau_channel_free(dev_priv->channel);
-               dev_priv->channel = NULL;
+       if (!engine->graph.accel_blocked) {
+               nouveau_fence_fini(dev);
+               nouveau_channel_put_unlocked(&dev_priv->channel);
        }
 
        if (!nouveau_noaccel) {
                engine->fifo.takedown(dev);
+               engine->crypt.takedown(dev);
                engine->graph.takedown(dev);
        }
        engine->fb.takedown(dev);
@@ -686,15 +781,16 @@ static void nouveau_card_takedown(struct drm_device *dev)
        ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_VRAM);
        ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_TT);
        mutex_unlock(&dev->struct_mutex);
-       nouveau_sgdma_takedown(dev);
+       nouveau_mem_gart_fini(dev);
 
-       nouveau_gpuobj_takedown(dev);
-       nouveau_mem_close(dev);
        engine->instmem.takedown(dev);
+       nouveau_gpuobj_takedown(dev);
+       nouveau_mem_vram_fini(dev);
 
-       drm_irq_uninstall(dev);
+       nouveau_irq_fini(dev);
+       drm_vblank_cleanup(dev);
 
-       nouveau_gpuobj_late_takedown(dev);
+       nouveau_pm_fini(dev);
        nouveau_bios_takedown(dev);
 
        vga_client_register(dev->pdev, NULL, NULL, NULL);
@@ -979,21 +1075,6 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                else
                        getparam->value = NV_PCI;
                break;
-       case NOUVEAU_GETPARAM_FB_PHYSICAL:
-               getparam->value = dev_priv->fb_phys;
-               break;
-       case NOUVEAU_GETPARAM_AGP_PHYSICAL:
-               getparam->value = dev_priv->gart_info.aper_base;
-               break;
-       case NOUVEAU_GETPARAM_PCI_PHYSICAL:
-               if (dev->sg) {
-                       getparam->value = (unsigned long)dev->sg->virtual;
-               } else {
-                       NV_ERROR(dev, "Requested PCIGART address, "
-                                       "while no PCIGART was created\n");
-                       return -EINVAL;
-               }
-               break;
        case NOUVEAU_GETPARAM_FB_SIZE:
                getparam->value = dev_priv->fb_available_size;
                break;
@@ -1001,11 +1082,17 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                getparam->value = dev_priv->gart_info.aper_size;
                break;
        case NOUVEAU_GETPARAM_VM_VRAM_BASE:
-               getparam->value = dev_priv->vm_vram_base;
+               getparam->value = 0; /* deprecated */
                break;
        case NOUVEAU_GETPARAM_PTIMER_TIME:
                getparam->value = dev_priv->engine.timer.read(dev);
                break;
+       case NOUVEAU_GETPARAM_HAS_BO_USAGE:
+               getparam->value = 1;
+               break;
+       case NOUVEAU_GETPARAM_HAS_PAGEFLIP:
+               getparam->value = (dev_priv->card_type < NV_50);
+               break;
        case NOUVEAU_GETPARAM_GRAPH_UNITS:
                /* NV40 and NV50 versions are quite different, but register
                 * address is the same. User is supposed to know the card
@@ -1016,7 +1103,7 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                }
                /* FALLTHRU */
        default:
-               NV_ERROR(dev, "unknown parameter %lld\n", getparam->param);
+               NV_DEBUG(dev, "unknown parameter %lld\n", getparam->param);
                return -EINVAL;
        }
 
@@ -1031,7 +1118,7 @@ nouveau_ioctl_setparam(struct drm_device *dev, void *data,
 
        switch (setparam->param) {
        default:
-               NV_ERROR(dev, "unknown parameter %lld\n", setparam->param);
+               NV_DEBUG(dev, "unknown parameter %lld\n", setparam->param);
                return -EINVAL;
        }
 
@@ -1039,8 +1126,9 @@ nouveau_ioctl_setparam(struct drm_device *dev, void *data,
 }
 
 /* Wait until (value(reg) & mask) == val, up until timeout has hit */
-bool nouveau_wait_until(struct drm_device *dev, uint64_t timeout,
-                       uint32_t reg, uint32_t mask, uint32_t val)
+bool
+nouveau_wait_eq(struct drm_device *dev, uint64_t timeout,
+               uint32_t reg, uint32_t mask, uint32_t val)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
@@ -1054,10 +1142,33 @@ bool nouveau_wait_until(struct drm_device *dev, uint64_t timeout,
        return false;
 }
 
+/* Wait until (value(reg) & mask) != val, up until timeout has hit */
+bool
+nouveau_wait_ne(struct drm_device *dev, uint64_t timeout,
+               uint32_t reg, uint32_t mask, uint32_t val)
+{
+       struct drm_nouveau_private *dev_priv = dev->dev_private;
+       struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
+       uint64_t start = ptimer->read(dev);
+
+       do {
+               if ((nv_rd32(dev, reg) & mask) != val)
+                       return true;
+       } while (ptimer->read(dev) - start < timeout);
+
+       return false;
+}
+
 /* Waits for PGRAPH to go completely idle */
 bool nouveau_wait_for_idle(struct drm_device *dev)
 {
-       if (!nv_wait(NV04_PGRAPH_STATUS, 0xffffffff, 0x00000000)) {
+       struct drm_nouveau_private *dev_priv = dev->dev_private;
+       uint32_t mask = ~0;
+
+       if (dev_priv->card_type == NV_40)
+               mask &= ~NV40_PGRAPH_STATUS_SYNC_STALL;
+
+       if (!nv_wait(dev, NV04_PGRAPH_STATUS, mask, 0)) {
                NV_ERROR(dev, "PGRAPH idle timed out with status 0x%08x\n",
                         nv_rd32(dev, NV04_PGRAPH_STATUS));
                return false;