drivers/gpu/drm/vc4/vc4_gem.c

   1 /*
   2  * Copyright © 2014 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/device.h>
  27 #include <linux/io.h>
  28
  29 #include "uapi/drm/vc4_drm.h"
  30 #include "vc4_drv.h"
  31 #include "vc4_regs.h"
  32 #include "vc4_trace.h"
  33
  34 static void
  35 vc4_queue_hangcheck(struct drm_device *dev)
  36 {
  37         struct vc4_dev *vc4 = to_vc4_dev(dev);
  38
  39         mod_timer(&vc4->hangcheck.timer,
  40                   round_jiffies_up(jiffies + msecs_to_jiffies(100)));
  41 }
  42
  43 struct vc4_hang_state {
  44         struct drm_vc4_get_hang_state user_state;
  45
  46         u32 bo_count;
  47         struct drm_gem_object **bo;
  48 };
  49
  50 static void
  51 vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state)
  52 {
  53         unsigned int i;
  54
  55         mutex_lock(&dev->struct_mutex);
  56         for (i = 0; i < state->user_state.bo_count; i++)
  57                 drm_gem_object_unreference(state->bo[i]);
  58         mutex_unlock(&dev->struct_mutex);
  59
  60         kfree(state);
  61 }
  62
  63 int
  64 vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
  65                          struct drm_file *file_priv)
  66 {
  67         struct drm_vc4_get_hang_state *get_state = data;
  68         struct drm_vc4_get_hang_state_bo *bo_state;
  69         struct vc4_hang_state *kernel_state;
  70         struct drm_vc4_get_hang_state *state;
  71         struct vc4_dev *vc4 = to_vc4_dev(dev);
  72         unsigned long irqflags;
  73         u32 i;
  74         int ret = 0;
  75
  76         spin_lock_irqsave(&vc4->job_lock, irqflags);
  77         kernel_state = vc4->hang_state;
  78         if (!kernel_state) {
  79                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
  80                 return -ENOENT;
  81         }
  82         state = &kernel_state->user_state;
  83
  84         /* If the user's array isn't big enough, just return the
  85          * required array size.
  86          */
  87         if (get_state->bo_count < state->bo_count) {
  88                 get_state->bo_count = state->bo_count;
  89                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
  90                 return 0;
  91         }
  92
  93         vc4->hang_state = NULL;
  94         spin_unlock_irqrestore(&vc4->job_lock, irqflags);
  95
  96         /* Save the user's BO pointer, so we don't stomp it with the memcpy. */
  97         state->bo = get_state->bo;
  98         memcpy(get_state, state, sizeof(*state));
  99
 100         bo_state = kcalloc(state->bo_count, sizeof(*bo_state), GFP_KERNEL);
 101         if (!bo_state) {
 102                 ret = -ENOMEM;
 103                 goto err_free;
 104         }
 105
 106         for (i = 0; i < state->bo_count; i++) {
 107                 struct vc4_bo *vc4_bo = to_vc4_bo(kernel_state->bo[i]);
 108                 u32 handle;
 109
 110                 ret = drm_gem_handle_create(file_priv, kernel_state->bo[i],
 111                                             &handle);
 112
 113                 if (ret) {
 114                         state->bo_count = i - 1;
 115                         goto err;
 116                 }
 117                 bo_state[i].handle = handle;
 118                 bo_state[i].paddr = vc4_bo->base.paddr;
 119                 bo_state[i].size = vc4_bo->base.base.size;
 120         }
 121
 122         if (copy_to_user((void __user *)(uintptr_t)get_state->bo,
 123                          bo_state,
 124                          state->bo_count * sizeof(*bo_state)))
 125                 ret = -EFAULT;
 126
 127         kfree(bo_state);
 128
 129 err_free:
 130
 131         vc4_free_hang_state(dev, kernel_state);
 132
 133 err:
 134         return ret;
 135 }
 136
 137 static void
 138 vc4_save_hang_state(struct drm_device *dev)
 139 {
 140         struct vc4_dev *vc4 = to_vc4_dev(dev);
 141         struct drm_vc4_get_hang_state *state;
 142         struct vc4_hang_state *kernel_state;
 143         struct vc4_exec_info *exec;
 144         struct vc4_bo *bo;
 145         unsigned long irqflags;
 146         unsigned int i, unref_list_count;
 147
 148         kernel_state = kcalloc(1, sizeof(*kernel_state), GFP_KERNEL);
 149         if (!kernel_state)
 150                 return;
 151
 152         state = &kernel_state->user_state;
 153
 154         spin_lock_irqsave(&vc4->job_lock, irqflags);
 155         exec = vc4_first_job(vc4);
 156         if (!exec) {
 157                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 158                 return;
 159         }
 160
 161         unref_list_count = 0;
 162         list_for_each_entry(bo, &exec->unref_list, unref_head)
 163                 unref_list_count++;
 164
 165         state->bo_count = exec->bo_count + unref_list_count;
 166         kernel_state->bo = kcalloc(state->bo_count, sizeof(*kernel_state->bo),
 167                                    GFP_ATOMIC);
 168         if (!kernel_state->bo) {
 169                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 170                 return;
 171         }
 172
 173         for (i = 0; i < exec->bo_count; i++) {
 174                 drm_gem_object_reference(&exec->bo[i]->base);
 175                 kernel_state->bo[i] = &exec->bo[i]->base;
 176         }
 177
 178         list_for_each_entry(bo, &exec->unref_list, unref_head) {
 179                 drm_gem_object_reference(&bo->base.base);
 180                 kernel_state->bo[i] = &bo->base.base;
 181                 i++;
 182         }
 183
 184         state->start_bin = exec->ct0ca;
 185         state->start_render = exec->ct1ca;
 186
 187         spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 188
 189         state->ct0ca = V3D_READ(V3D_CTNCA(0));
 190         state->ct0ea = V3D_READ(V3D_CTNEA(0));
 191
 192         state->ct1ca = V3D_READ(V3D_CTNCA(1));
 193         state->ct1ea = V3D_READ(V3D_CTNEA(1));
 194
 195         state->ct0cs = V3D_READ(V3D_CTNCS(0));
 196         state->ct1cs = V3D_READ(V3D_CTNCS(1));
 197
 198         state->ct0ra0 = V3D_READ(V3D_CT00RA0);
 199         state->ct1ra0 = V3D_READ(V3D_CT01RA0);
 200
 201         state->bpca = V3D_READ(V3D_BPCA);
 202         state->bpcs = V3D_READ(V3D_BPCS);
 203         state->bpoa = V3D_READ(V3D_BPOA);
 204         state->bpos = V3D_READ(V3D_BPOS);
 205
 206         state->vpmbase = V3D_READ(V3D_VPMBASE);
 207
 208         state->dbge = V3D_READ(V3D_DBGE);
 209         state->fdbgo = V3D_READ(V3D_FDBGO);
 210         state->fdbgb = V3D_READ(V3D_FDBGB);
 211         state->fdbgr = V3D_READ(V3D_FDBGR);
 212         state->fdbgs = V3D_READ(V3D_FDBGS);
 213         state->errstat = V3D_READ(V3D_ERRSTAT);
 214
 215         spin_lock_irqsave(&vc4->job_lock, irqflags);
 216         if (vc4->hang_state) {
 217                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 218                 vc4_free_hang_state(dev, kernel_state);
 219         } else {
 220                 vc4->hang_state = kernel_state;
 221                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 222         }
 223 }
 224
 225 static void
 226 vc4_reset(struct drm_device *dev)
 227 {
 228         struct vc4_dev *vc4 = to_vc4_dev(dev);
 229
 230         DRM_INFO("Resetting GPU.\n");
 231         vc4_v3d_set_power(vc4, false);
 232         vc4_v3d_set_power(vc4, true);
 233
 234         vc4_irq_reset(dev);
 235
 236         /* Rearm the hangcheck -- another job might have been waiting
 237          * for our hung one to get kicked off, and vc4_irq_reset()
 238          * would have started it.
 239          */
 240         vc4_queue_hangcheck(dev);
 241 }
 242
 243 static void
 244 vc4_reset_work(struct work_struct *work)
 245 {
 246         struct vc4_dev *vc4 =
 247                 container_of(work, struct vc4_dev, hangcheck.reset_work);
 248
 249         vc4_save_hang_state(vc4->dev);
 250
 251         vc4_reset(vc4->dev);
 252 }
 253
 254 static void
 255 vc4_hangcheck_elapsed(unsigned long data)
 256 {
 257         struct drm_device *dev = (struct drm_device *)data;
 258         struct vc4_dev *vc4 = to_vc4_dev(dev);
 259         uint32_t ct0ca, ct1ca;
 260
 261         /* If idle, we can stop watching for hangs. */
 262         if (list_empty(&vc4->job_list))
 263                 return;
 264
 265         ct0ca = V3D_READ(V3D_CTNCA(0));
 266         ct1ca = V3D_READ(V3D_CTNCA(1));
 267
 268         /* If we've made any progress in execution, rearm the timer
 269          * and wait.
 270          */
 271         if (ct0ca != vc4->hangcheck.last_ct0ca ||
 272             ct1ca != vc4->hangcheck.last_ct1ca) {
 273                 vc4->hangcheck.last_ct0ca = ct0ca;
 274                 vc4->hangcheck.last_ct1ca = ct1ca;
 275                 vc4_queue_hangcheck(dev);
 276                 return;
 277         }
 278
 279         /* We've gone too long with no progress, reset.  This has to
 280          * be done from a work struct, since resetting can sleep and
 281          * this timer hook isn't allowed to.
 282          */
 283         schedule_work(&vc4->hangcheck.reset_work);
 284 }
 285
 286 static void
 287 submit_cl(struct drm_device *dev, uint32_t thread, uint32_t start, uint32_t end)
 288 {
 289         struct vc4_dev *vc4 = to_vc4_dev(dev);
 290
 291         /* Set the current and end address of the control list.
 292          * Writing the end register is what starts the job.
 293          */
 294         V3D_WRITE(V3D_CTNCA(thread), start);
 295         V3D_WRITE(V3D_CTNEA(thread), end);
 296 }
 297
 298 int
 299 vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, uint64_t timeout_ns,
 300                    bool interruptible)
 301 {
 302         struct vc4_dev *vc4 = to_vc4_dev(dev);
 303         int ret = 0;
 304         unsigned long timeout_expire;
 305         DEFINE_WAIT(wait);
 306
 307         if (vc4->finished_seqno >= seqno)
 308                 return 0;
 309
 310         if (timeout_ns == 0)
 311                 return -ETIME;
 312
 313         timeout_expire = jiffies + nsecs_to_jiffies(timeout_ns);
 314
 315         trace_vc4_wait_for_seqno_begin(dev, seqno, timeout_ns);
 316         for (;;) {
 317                 prepare_to_wait(&vc4->job_wait_queue, &wait,
 318                                 interruptible ? TASK_INTERRUPTIBLE :
 319                                 TASK_UNINTERRUPTIBLE);
 320
 321                 if (interruptible && signal_pending(current)) {
 322                         ret = -ERESTARTSYS;
 323                         break;
 324                 }
 325
 326                 if (vc4->finished_seqno >= seqno)
 327                         break;
 328
 329                 if (timeout_ns != ~0ull) {
 330                         if (time_after_eq(jiffies, timeout_expire)) {
 331                                 ret = -ETIME;
 332                                 break;
 333                         }
 334                         schedule_timeout(timeout_expire - jiffies);
 335                 } else {
 336                         schedule();
 337                 }
 338         }
 339
 340         finish_wait(&vc4->job_wait_queue, &wait);
 341         trace_vc4_wait_for_seqno_end(dev, seqno);
 342
 343         if (ret && ret != -ERESTARTSYS)
 344                 DRM_ERROR("timeout waiting for render thread idle\n");
 345
 346         return ret;
 347 }
 348
 349 static void
 350 vc4_flush_caches(struct drm_device *dev)
 351 {
 352         struct vc4_dev *vc4 = to_vc4_dev(dev);
 353
 354         /* Flush the GPU L2 caches.  These caches sit on top of system
 355          * L3 (the 128kb or so shared with the CPU), and are
 356          * non-allocating in the L3.
 357          */
 358         V3D_WRITE(V3D_L2CACTL,
 359                   V3D_L2CACTL_L2CCLR);
 360
 361         V3D_WRITE(V3D_SLCACTL,
 362                   VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |
 363                   VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC) |
 364                   VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC) |
 365                   VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC));
 366 }
 367
 368 /* Sets the registers for the next job to be actually be executed in
 369  * the hardware.
 370  *
 371  * The job_lock should be held during this.
 372  */
 373 void
 374 vc4_submit_next_job(struct drm_device *dev)
 375 {
 376         struct vc4_dev *vc4 = to_vc4_dev(dev);
 377         struct vc4_exec_info *exec = vc4_first_job(vc4);
 378
 379         if (!exec)
 380                 return;
 381
 382         vc4_flush_caches(dev);
 383
 384         /* Disable the binner's pre-loaded overflow memory address */
 385         V3D_WRITE(V3D_BPOA, 0);
 386         V3D_WRITE(V3D_BPOS, 0);
 387
 388         if (exec->ct0ca != exec->ct0ea)
 389                 submit_cl(dev, 0, exec->ct0ca, exec->ct0ea);
 390         submit_cl(dev, 1, exec->ct1ca, exec->ct1ea);
 391 }
 392
 393 static void
 394 vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
 395 {
 396         struct vc4_bo *bo;
 397         unsigned i;
 398
 399         for (i = 0; i < exec->bo_count; i++) {
 400                 bo = to_vc4_bo(&exec->bo[i]->base);
 401                 bo->seqno = seqno;
 402         }
 403
 404         list_for_each_entry(bo, &exec->unref_list, unref_head) {
 405                 bo->seqno = seqno;
 406         }
 407 }
 408
 409 /* Queues a struct vc4_exec_info for execution.  If no job is
 410  * currently executing, then submits it.
 411  *
 412  * Unlike most GPUs, our hardware only handles one command list at a
 413  * time.  To queue multiple jobs at once, we'd need to edit the
 414  * previous command list to have a jump to the new one at the end, and
 415  * then bump the end address.  That's a change for a later date,
 416  * though.
 417  */
 418 static void
 419 vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec)
 420 {
 421         struct vc4_dev *vc4 = to_vc4_dev(dev);
 422         uint64_t seqno;
 423         unsigned long irqflags;
 424
 425         spin_lock_irqsave(&vc4->job_lock, irqflags);
 426
 427         seqno = ++vc4->emit_seqno;
 428         exec->seqno = seqno;
 429         vc4_update_bo_seqnos(exec, seqno);
 430
 431         list_add_tail(&exec->head, &vc4->job_list);
 432
 433         /* If no job was executing, kick ours off.  Otherwise, it'll
 434          * get started when the previous job's frame done interrupt
 435          * occurs.
 436          */
 437         if (vc4_first_job(vc4) == exec) {
 438                 vc4_submit_next_job(dev);
 439                 vc4_queue_hangcheck(dev);
 440         }
 441
 442         spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 443 }
 444
 445 /**
 446  * Looks up a bunch of GEM handles for BOs and stores the array for
 447  * use in the command validator that actually writes relocated
 448  * addresses pointing to them.
 449  */
 450 static int
 451 vc4_cl_lookup_bos(struct drm_device *dev,
 452                   struct drm_file *file_priv,
 453                   struct vc4_exec_info *exec)
 454 {
 455         struct drm_vc4_submit_cl *args = exec->args;
 456         uint32_t *handles;
 457         int ret = 0;
 458         int i;
 459
 460         exec->bo_count = args->bo_handle_count;
 461
 462         if (!exec->bo_count) {
 463                 /* See comment on bo_index for why we have to check
 464                  * this.
 465                  */
 466                 DRM_ERROR("Rendering requires BOs to validate\n");
 467                 return -EINVAL;
 468         }
 469
 470         exec->bo = kcalloc(exec->bo_count, sizeof(struct drm_gem_cma_object *),
 471                            GFP_KERNEL);
 472         if (!exec->bo) {
 473                 DRM_ERROR("Failed to allocate validated BO pointers\n");
 474                 return -ENOMEM;
 475         }
 476
 477         handles = drm_malloc_ab(exec->bo_count, sizeof(uint32_t));
 478         if (!handles) {
 479                 DRM_ERROR("Failed to allocate incoming GEM handles\n");
 480                 goto fail;
 481         }
 482
 483         ret = copy_from_user(handles,
 484                              (void __user *)(uintptr_t)args->bo_handles,
 485                              exec->bo_count * sizeof(uint32_t));
 486         if (ret) {
 487                 DRM_ERROR("Failed to copy in GEM handles\n");
 488                 goto fail;
 489         }
 490
 491         spin_lock(&file_priv->table_lock);
 492         for (i = 0; i < exec->bo_count; i++) {
 493                 struct drm_gem_object *bo = idr_find(&file_priv->object_idr,
 494                                                      handles[i]);
 495                 if (!bo) {
 496                         DRM_ERROR("Failed to look up GEM BO %d: %d\n",
 497                                   i, handles[i]);
 498                         ret = -EINVAL;
 499                         spin_unlock(&file_priv->table_lock);
 500                         goto fail;
 501                 }
 502                 drm_gem_object_reference(bo);
 503                 exec->bo[i] = (struct drm_gem_cma_object *)bo;
 504         }
 505         spin_unlock(&file_priv->table_lock);
 506
 507 fail:
 508         kfree(handles);
 509         return 0;
 510 }
 511
 512 static int
 513 vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 514 {
 515         struct drm_vc4_submit_cl *args = exec->args;
 516         void *temp = NULL;
 517         void *bin;
 518         int ret = 0;
 519         uint32_t bin_offset = 0;
 520         uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size,
 521                                              16);
 522         uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size;
 523         uint32_t exec_size = uniforms_offset + args->uniforms_size;
 524         uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) *
 525                                           args->shader_rec_count);
 526         struct vc4_bo *bo;
 527
 528         if (uniforms_offset < shader_rec_offset ||
 529             exec_size < uniforms_offset ||
 530             args->shader_rec_count >= (UINT_MAX /
 531                                           sizeof(struct vc4_shader_state)) ||
 532             temp_size < exec_size) {
 533                 DRM_ERROR("overflow in exec arguments\n");
 534                 goto fail;
 535         }
 536
 537         /* Allocate space where we'll store the copied in user command lists
 538          * and shader records.
 539          *
 540          * We don't just copy directly into the BOs because we need to
 541          * read the contents back for validation, and I think the
 542          * bo->vaddr is uncached access.
 543          */
 544         temp = kmalloc(temp_size, GFP_KERNEL);
 545         if (!temp) {
 546                 DRM_ERROR("Failed to allocate storage for copying "
 547                           "in bin/render CLs.\n");
 548                 ret = -ENOMEM;
 549                 goto fail;
 550         }
 551         bin = temp + bin_offset;
 552         exec->shader_rec_u = temp + shader_rec_offset;
 553         exec->uniforms_u = temp + uniforms_offset;
 554         exec->shader_state = temp + exec_size;
 555         exec->shader_state_size = args->shader_rec_count;
 556
 557         if (copy_from_user(bin,
 558                            (void __user *)(uintptr_t)args->bin_cl,
 559                            args->bin_cl_size)) {
 560                 ret = -EFAULT;
 561                 goto fail;
 562         }
 563
 564         if (copy_from_user(exec->shader_rec_u,
 565                            (void __user *)(uintptr_t)args->shader_rec,
 566                            args->shader_rec_size)) {
 567                 ret = -EFAULT;
 568                 goto fail;
 569         }
 570
 571         if (copy_from_user(exec->uniforms_u,
 572                            (void __user *)(uintptr_t)args->uniforms,
 573                            args->uniforms_size)) {
 574                 ret = -EFAULT;
 575                 goto fail;
 576         }
 577
 578         bo = vc4_bo_create(dev, exec_size, true);
 579         if (IS_ERR(bo)) {
 580                 DRM_ERROR("Couldn't allocate BO for binning\n");
 581                 ret = PTR_ERR(bo);
 582                 goto fail;
 583         }
 584         exec->exec_bo = &bo->base;
 585
 586         list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,
 587                       &exec->unref_list);
 588
 589         exec->ct0ca = exec->exec_bo->paddr + bin_offset;
 590
 591         exec->bin_u = bin;
 592
 593         exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;
 594         exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset;
 595         exec->shader_rec_size = args->shader_rec_size;
 596
 597         exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;
 598         exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset;
 599         exec->uniforms_size = args->uniforms_size;
 600
 601         ret = vc4_validate_bin_cl(dev,
 602                                   exec->exec_bo->vaddr + bin_offset,
 603                                   bin,
 604                                   exec);
 605         if (ret)
 606                 goto fail;
 607
 608         ret = vc4_validate_shader_recs(dev, exec);
 609
 610 fail:
 611         kfree(temp);
 612         return ret;
 613 }
 614
 615 static void
 616 vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 617 {
 618         unsigned i;
 619
 620         /* Need the struct lock for drm_gem_object_unreference(). */
 621         mutex_lock(&dev->struct_mutex);
 622         if (exec->bo) {
 623                 for (i = 0; i < exec->bo_count; i++)
 624                         drm_gem_object_unreference(&exec->bo[i]->base);
 625                 kfree(exec->bo);
 626         }
 627
 628         while (!list_empty(&exec->unref_list)) {
 629                 struct vc4_bo *bo = list_first_entry(&exec->unref_list,
 630                                                      struct vc4_bo, unref_head);
 631                 list_del(&bo->unref_head);
 632                 drm_gem_object_unreference(&bo->base.base);
 633         }
 634         mutex_unlock(&dev->struct_mutex);
 635
 636         kfree(exec);
 637 }
 638
 639 void
 640 vc4_job_handle_completed(struct vc4_dev *vc4)
 641 {
 642         unsigned long irqflags;
 643         struct vc4_seqno_cb *cb, *cb_temp;
 644
 645         spin_lock_irqsave(&vc4->job_lock, irqflags);
 646         while (!list_empty(&vc4->job_done_list)) {
 647                 struct vc4_exec_info *exec =
 648                         list_first_entry(&vc4->job_done_list,
 649                                          struct vc4_exec_info, head);
 650                 list_del(&exec->head);
 651
 652                 spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 653                 vc4_complete_exec(vc4->dev, exec);
 654                 spin_lock_irqsave(&vc4->job_lock, irqflags);
 655         }
 656
 657         list_for_each_entry_safe(cb, cb_temp, &vc4->seqno_cb_list, work.entry) {
 658                 if (cb->seqno <= vc4->finished_seqno) {
 659                         list_del_init(&cb->work.entry);
 660                         schedule_work(&cb->work);
 661                 }
 662         }
 663
 664         spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 665 }
 666
 667 static void vc4_seqno_cb_work(struct work_struct *work)
 668 {
 669         struct vc4_seqno_cb *cb = container_of(work, struct vc4_seqno_cb, work);
 670
 671         cb->func(cb);
 672 }
 673
 674 int vc4_queue_seqno_cb(struct drm_device *dev,
 675                        struct vc4_seqno_cb *cb, uint64_t seqno,
 676                        void (*func)(struct vc4_seqno_cb *cb))
 677 {
 678         struct vc4_dev *vc4 = to_vc4_dev(dev);
 679         int ret = 0;
 680         unsigned long irqflags;
 681
 682         cb->func = func;
 683         INIT_WORK(&cb->work, vc4_seqno_cb_work);
 684
 685         spin_lock_irqsave(&vc4->job_lock, irqflags);
 686         if (seqno > vc4->finished_seqno) {
 687                 cb->seqno = seqno;
 688                 list_add_tail(&cb->work.entry, &vc4->seqno_cb_list);
 689         } else {
 690                 schedule_work(&cb->work);
 691         }
 692         spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 693
 694         return ret;
 695 }
 696
 697 /* Scheduled when any job has been completed, this walks the list of
 698  * jobs that had completed and unrefs their BOs and frees their exec
 699  * structs.
 700  */
 701 static void
 702 vc4_job_done_work(struct work_struct *work)
 703 {
 704         struct vc4_dev *vc4 =
 705                 container_of(work, struct vc4_dev, job_done_work);
 706
 707         vc4_job_handle_completed(vc4);
 708 }
 709
 710 static int
 711 vc4_wait_for_seqno_ioctl_helper(struct drm_device *dev,
 712                                 uint64_t seqno,
 713                                 uint64_t *timeout_ns)
 714 {
 715         unsigned long start = jiffies;
 716         int ret = vc4_wait_for_seqno(dev, seqno, *timeout_ns, true);
 717
 718         if ((ret == -EINTR || ret == -ERESTARTSYS) && *timeout_ns != ~0ull) {
 719                 uint64_t delta = jiffies_to_nsecs(jiffies - start);
 720
 721                 if (*timeout_ns >= delta)
 722                         *timeout_ns -= delta;
 723         }
 724
 725         return ret;
 726 }
 727
 728 int
 729 vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,
 730                      struct drm_file *file_priv)
 731 {
 732         struct drm_vc4_wait_seqno *args = data;
 733
 734         return vc4_wait_for_seqno_ioctl_helper(dev, args->seqno,
 735                                                &args->timeout_ns);
 736 }
 737
 738 int
 739 vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
 740                   struct drm_file *file_priv)
 741 {
 742         int ret;
 743         struct drm_vc4_wait_bo *args = data;
 744         struct drm_gem_object *gem_obj;
 745         struct vc4_bo *bo;
 746
 747         if (args->pad != 0)
 748                 return -EINVAL;
 749
 750         gem_obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 751         if (!gem_obj) {
 752                 DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
 753                 return -EINVAL;
 754         }
 755         bo = to_vc4_bo(gem_obj);
 756
 757         ret = vc4_wait_for_seqno_ioctl_helper(dev, bo->seqno,
 758                                               &args->timeout_ns);
 759
 760         drm_gem_object_unreference_unlocked(gem_obj);
 761         return ret;
 762 }
 763
 764 /**
 765  * Submits a command list to the VC4.
 766  *
 767  * This is what is called batchbuffer emitting on other hardware.
 768  */
 769 int
 770 vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 771                     struct drm_file *file_priv)
 772 {
 773         struct vc4_dev *vc4 = to_vc4_dev(dev);
 774         struct drm_vc4_submit_cl *args = data;
 775         struct vc4_exec_info *exec;
 776         int ret;
 777
 778         if ((args->flags & ~VC4_SUBMIT_CL_USE_CLEAR_COLOR) != 0) {
 779                 DRM_ERROR("Unknown flags: 0x%02x\n", args->flags);
 780                 return -EINVAL;
 781         }
 782
 783         exec = kcalloc(1, sizeof(*exec), GFP_KERNEL);
 784         if (!exec) {
 785                 DRM_ERROR("malloc failure on exec struct\n");
 786                 return -ENOMEM;
 787         }
 788
 789         exec->args = args;
 790         INIT_LIST_HEAD(&exec->unref_list);
 791
 792         ret = vc4_cl_lookup_bos(dev, file_priv, exec);
 793         if (ret)
 794                 goto fail;
 795
 796         if (exec->args->bin_cl_size != 0) {
 797                 ret = vc4_get_bcl(dev, exec);
 798                 if (ret)
 799                         goto fail;
 800         } else {
 801                 exec->ct0ca = 0;
 802                 exec->ct0ea = 0;
 803         }
 804
 805         ret = vc4_get_rcl(dev, exec);
 806         if (ret)
 807                 goto fail;
 808
 809         /* Clear this out of the struct we'll be putting in the queue,
 810          * since it's part of our stack.
 811          */
 812         exec->args = NULL;
 813
 814         vc4_queue_submit(dev, exec);
 815
 816         /* Return the seqno for our job. */
 817         args->seqno = vc4->emit_seqno;
 818
 819         return 0;
 820
 821 fail:
 822         vc4_complete_exec(vc4->dev, exec);
 823
 824         return ret;
 825 }
 826
 827 void
 828 vc4_gem_init(struct drm_device *dev)
 829 {
 830         struct vc4_dev *vc4 = to_vc4_dev(dev);
 831
 832         INIT_LIST_HEAD(&vc4->job_list);
 833         INIT_LIST_HEAD(&vc4->job_done_list);
 834         INIT_LIST_HEAD(&vc4->seqno_cb_list);
 835         spin_lock_init(&vc4->job_lock);
 836
 837         INIT_WORK(&vc4->hangcheck.reset_work, vc4_reset_work);
 838         setup_timer(&vc4->hangcheck.timer,
 839                     vc4_hangcheck_elapsed,
 840                     (unsigned long)dev);
 841
 842         INIT_WORK(&vc4->job_done_work, vc4_job_done_work);
 843 }
 844
 845 void
 846 vc4_gem_destroy(struct drm_device *dev)
 847 {
 848         struct vc4_dev *vc4 = to_vc4_dev(dev);
 849
 850         /* Waiting for exec to finish would need to be done before
 851          * unregistering V3D.
 852          */
 853         WARN_ON(vc4->emit_seqno != vc4->finished_seqno);
 854
 855         /* V3D should already have disabled its interrupt and cleared
 856          * the overflow allocation registers.  Now free the object.
 857          */
 858         if (vc4->overflow_mem) {
 859                 drm_gem_object_unreference_unlocked(&vc4->overflow_mem->base.base);
 860                 vc4->overflow_mem = NULL;
 861         }
 862
 863         vc4_bo_cache_destroy(dev);
 864
 865         if (vc4->hang_state)
 866                 vc4_free_hang_state(dev, vc4->hang_state);
 867 }