drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <glisse@freedesktop.org>
  29  *    Dave Airlie
  30  */
  31 #include <linux/seq_file.h>
  32 #include <linux/atomic.h>
  33 #include <linux/wait.h>
  34 #include <linux/kref.h>
  35 #include <linux/slab.h>
  36 #include <linux/firmware.h>
  37 #include <drm/drmP.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40
  41 /*
  42  * Fences
  43  * Fences mark an event in the GPUs pipeline and are used
  44  * for GPU/CPU synchronization.  When the fence is written,
  45  * it is expected that all buffers associated with that fence
  46  * are no longer in use by the associated ring on the GPU and
  47  * that the the relevant GPU caches have been flushed.
  48  */
  49
  50 struct amdgpu_fence {
  51         struct fence base;
  52
  53         /* RB, DMA, etc. */
  54         struct amdgpu_ring              *ring;
  55         uint64_t                        seq;
  56
  57         wait_queue_t                    fence_wake;
  58 };
  59
  60 static struct kmem_cache *amdgpu_fence_slab;
  61 static atomic_t amdgpu_fence_slab_ref = ATOMIC_INIT(0);
  62
  63 /*
  64  * Cast helper
  65  */
  66 static const struct fence_ops amdgpu_fence_ops;
  67 static inline struct amdgpu_fence *to_amdgpu_fence(struct fence *f)
  68 {
  69         struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
  70
  71         if (__f->base.ops == &amdgpu_fence_ops)
  72                 return __f;
  73
  74         return NULL;
  75 }
  76
  77 /**
  78  * amdgpu_fence_write - write a fence value
  79  *
  80  * @ring: ring the fence is associated with
  81  * @seq: sequence number to write
  82  *
  83  * Writes a fence value to memory (all asics).
  84  */
  85 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
  86 {
  87         struct amdgpu_fence_driver *drv = &ring->fence_drv;
  88
  89         if (drv->cpu_addr)
  90                 *drv->cpu_addr = cpu_to_le32(seq);
  91 }
  92
  93 /**
  94  * amdgpu_fence_read - read a fence value
  95  *
  96  * @ring: ring the fence is associated with
  97  *
  98  * Reads a fence value from memory (all asics).
  99  * Returns the value of the fence read from memory.
 100  */
 101 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
 102 {
 103         struct amdgpu_fence_driver *drv = &ring->fence_drv;
 104         u32 seq = 0;
 105
 106         if (drv->cpu_addr)
 107                 seq = le32_to_cpu(*drv->cpu_addr);
 108         else
 109                 seq = lower_32_bits(atomic64_read(&drv->last_seq));
 110
 111         return seq;
 112 }
 113
 114 /**
 115  * amdgpu_fence_emit - emit a fence on the requested ring
 116  *
 117  * @ring: ring the fence is associated with
 118  * @f: resulting fence object
 119  *
 120  * Emits a fence command on the requested ring (all asics).
 121  * Returns 0 on success, -ENOMEM on failure.
 122  */
 123 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f)
 124 {
 125         struct amdgpu_device *adev = ring->adev;
 126         struct amdgpu_fence *fence;
 127
 128         fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
 129         if (fence == NULL)
 130                 return -ENOMEM;
 131
 132         fence->seq = ++ring->fence_drv.sync_seq;
 133         fence->ring = ring;
 134         fence_init(&fence->base, &amdgpu_fence_ops,
 135                    &ring->fence_drv.fence_queue.lock,
 136                    adev->fence_context + ring->idx,
 137                    fence->seq);
 138         amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
 139                                fence->seq, AMDGPU_FENCE_FLAG_INT);
 140         *f = &fence->base;
 141         return 0;
 142 }
 143
 144 /**
 145  * amdgpu_fence_schedule_fallback - schedule fallback check
 146  *
 147  * @ring: pointer to struct amdgpu_ring
 148  *
 149  * Start a timer as fallback to our interrupts.
 150  */
 151 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
 152 {
 153         mod_timer(&ring->fence_drv.fallback_timer,
 154                   jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
 155 }
 156
 157 /**
 158  * amdgpu_fence_activity - check for fence activity
 159  *
 160  * @ring: pointer to struct amdgpu_ring
 161  *
 162  * Checks the current fence value and calculates the last
 163  * signalled fence value. Returns true if activity occured
 164  * on the ring, and the fence_queue should be waken up.
 165  */
 166 static bool amdgpu_fence_activity(struct amdgpu_ring *ring)
 167 {
 168         uint64_t seq, last_seq, last_emitted;
 169         bool wake = false;
 170
 171         last_seq = atomic64_read(&ring->fence_drv.last_seq);
 172         do {
 173                 last_emitted = ring->fence_drv.sync_seq;
 174                 seq = amdgpu_fence_read(ring);
 175                 seq |= last_seq & 0xffffffff00000000LL;
 176                 if (seq < last_seq) {
 177                         seq &= 0xffffffff;
 178                         seq |= last_emitted & 0xffffffff00000000LL;
 179                 }
 180
 181                 if (seq <= last_seq || seq > last_emitted)
 182                         break;
 183
 184                 /* If we loop over we don't want to return without
 185                  * checking if a fence is signaled as it means that the
 186                  * seq we just read is different from the previous on.
 187                  */
 188                 wake = true;
 189                 last_seq = seq;
 190
 191         } while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq);
 192
 193         if (seq < last_emitted)
 194                 amdgpu_fence_schedule_fallback(ring);
 195
 196         return wake;
 197 }
 198
 199 /**
 200  * amdgpu_fence_process - process a fence
 201  *
 202  * @adev: amdgpu_device pointer
 203  * @ring: ring index the fence is associated with
 204  *
 205  * Checks the current fence value and wakes the fence queue
 206  * if the sequence number has increased (all asics).
 207  */
 208 void amdgpu_fence_process(struct amdgpu_ring *ring)
 209 {
 210         if (amdgpu_fence_activity(ring))
 211                 wake_up_all(&ring->fence_drv.fence_queue);
 212 }
 213
 214 /**
 215  * amdgpu_fence_fallback - fallback for hardware interrupts
 216  *
 217  * @work: delayed work item
 218  *
 219  * Checks for fence activity.
 220  */
 221 static void amdgpu_fence_fallback(unsigned long arg)
 222 {
 223         struct amdgpu_ring *ring = (void *)arg;
 224
 225         amdgpu_fence_process(ring);
 226 }
 227
 228 /**
 229  * amdgpu_fence_seq_signaled - check if a fence sequence number has signaled
 230  *
 231  * @ring: ring the fence is associated with
 232  * @seq: sequence number
 233  *
 234  * Check if the last signaled fence sequnce number is >= the requested
 235  * sequence number (all asics).
 236  * Returns true if the fence has signaled (current fence value
 237  * is >= requested value) or false if it has not (current fence
 238  * value is < the requested value.  Helper function for
 239  * amdgpu_fence_signaled().
 240  */
 241 static bool amdgpu_fence_seq_signaled(struct amdgpu_ring *ring, u64 seq)
 242 {
 243         if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
 244                 return true;
 245
 246         /* poll new last sequence at least once */
 247         amdgpu_fence_process(ring);
 248         if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
 249                 return true;
 250
 251         return false;
 252 }
 253
 254 /*
 255  * amdgpu_ring_wait_seq - wait for seq of the specific ring to signal
 256  * @ring: ring to wait on for the seq number
 257  * @seq: seq number wait for
 258  *
 259  * return value:
 260  * 0: seq signaled, and gpu not hang
 261  * -EINVAL: some paramter is not valid
 262  */
 263 static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq)
 264 {
 265         BUG_ON(!ring);
 266         if (seq > ring->fence_drv.sync_seq)
 267                 return -EINVAL;
 268
 269         if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
 270                 return 0;
 271
 272         amdgpu_fence_schedule_fallback(ring);
 273         wait_event(ring->fence_drv.fence_queue,
 274                    amdgpu_fence_seq_signaled(ring, seq));
 275
 276         return 0;
 277 }
 278
 279 /**
 280  * amdgpu_fence_wait_empty - wait for all fences to signal
 281  *
 282  * @adev: amdgpu device pointer
 283  * @ring: ring index the fence is associated with
 284  *
 285  * Wait for all fences on the requested ring to signal (all asics).
 286  * Returns 0 if the fences have passed, error for all other cases.
 287  * Caller must hold ring lock.
 288  */
 289 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
 290 {
 291         uint64_t seq = ring->fence_drv.sync_seq;
 292
 293         if (!seq)
 294                 return 0;
 295
 296         return amdgpu_fence_ring_wait_seq(ring, seq);
 297 }
 298
 299 /**
 300  * amdgpu_fence_count_emitted - get the count of emitted fences
 301  *
 302  * @ring: ring the fence is associated with
 303  *
 304  * Get the number of fences emitted on the requested ring (all asics).
 305  * Returns the number of emitted fences on the ring.  Used by the
 306  * dynpm code to ring track activity.
 307  */
 308 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
 309 {
 310         uint64_t emitted;
 311
 312         /* We are not protected by ring lock when reading the last sequence
 313          * but it's ok to report slightly wrong fence count here.
 314          */
 315         amdgpu_fence_process(ring);
 316         emitted = ring->fence_drv.sync_seq
 317                 - atomic64_read(&ring->fence_drv.last_seq);
 318         /* to avoid 32bits warp around */
 319         if (emitted > 0x10000000)
 320                 emitted = 0x10000000;
 321
 322         return (unsigned)emitted;
 323 }
 324
 325 /**
 326  * amdgpu_fence_driver_start_ring - make the fence driver
 327  * ready for use on the requested ring.
 328  *
 329  * @ring: ring to start the fence driver on
 330  * @irq_src: interrupt source to use for this ring
 331  * @irq_type: interrupt type to use for this ring
 332  *
 333  * Make the fence driver ready for processing (all asics).
 334  * Not all asics have all rings, so each asic will only
 335  * start the fence driver on the rings it has.
 336  * Returns 0 for success, errors for failure.
 337  */
 338 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 339                                    struct amdgpu_irq_src *irq_src,
 340                                    unsigned irq_type)
 341 {
 342         struct amdgpu_device *adev = ring->adev;
 343         uint64_t index;
 344
 345         if (ring != &adev->uvd.ring) {
 346                 ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];
 347                 ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);
 348         } else {
 349                 /* put fence directly behind firmware */
 350                 index = ALIGN(adev->uvd.fw->size, 8);
 351                 ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;
 352                 ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;
 353         }
 354         amdgpu_fence_write(ring, atomic64_read(&ring->fence_drv.last_seq));
 355         amdgpu_irq_get(adev, irq_src, irq_type);
 356
 357         ring->fence_drv.irq_src = irq_src;
 358         ring->fence_drv.irq_type = irq_type;
 359         ring->fence_drv.initialized = true;
 360
 361         dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
 362                  "cpu addr 0x%p\n", ring->idx,
 363                  ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
 364         return 0;
 365 }
 366
 367 /**
 368  * amdgpu_fence_driver_init_ring - init the fence driver
 369  * for the requested ring.
 370  *
 371  * @ring: ring to init the fence driver on
 372  *
 373  * Init the fence driver for the requested ring (all asics).
 374  * Helper function for amdgpu_fence_driver_init().
 375  */
 376 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
 377 {
 378         long timeout;
 379         int r;
 380
 381         ring->fence_drv.cpu_addr = NULL;
 382         ring->fence_drv.gpu_addr = 0;
 383         ring->fence_drv.sync_seq = 0;
 384         atomic64_set(&ring->fence_drv.last_seq, 0);
 385         ring->fence_drv.initialized = false;
 386
 387         setup_timer(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,
 388                     (unsigned long)ring);
 389
 390         init_waitqueue_head(&ring->fence_drv.fence_queue);
 391
 392         timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
 393         if (timeout == 0) {
 394                 /*
 395                  * FIXME:
 396                  * Delayed workqueue cannot use it directly,
 397                  * so the scheduler will not use delayed workqueue if
 398                  * MAX_SCHEDULE_TIMEOUT is set.
 399                  * Currently keep it simple and silly.
 400                  */
 401                 timeout = MAX_SCHEDULE_TIMEOUT;
 402         }
 403         r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
 404                            amdgpu_sched_hw_submission,
 405                            timeout, ring->name);
 406         if (r) {
 407                 DRM_ERROR("Failed to create scheduler on ring %s.\n",
 408                           ring->name);
 409                 return r;
 410         }
 411
 412         return 0;
 413 }
 414
 415 /**
 416  * amdgpu_fence_driver_init - init the fence driver
 417  * for all possible rings.
 418  *
 419  * @adev: amdgpu device pointer
 420  *
 421  * Init the fence driver for all possible rings (all asics).
 422  * Not all asics have all rings, so each asic will only
 423  * start the fence driver on the rings it has using
 424  * amdgpu_fence_driver_start_ring().
 425  * Returns 0 for success.
 426  */
 427 int amdgpu_fence_driver_init(struct amdgpu_device *adev)
 428 {
 429         if (atomic_inc_return(&amdgpu_fence_slab_ref) == 1) {
 430                 amdgpu_fence_slab = kmem_cache_create(
 431                         "amdgpu_fence", sizeof(struct amdgpu_fence), 0,
 432                         SLAB_HWCACHE_ALIGN, NULL);
 433                 if (!amdgpu_fence_slab)
 434                         return -ENOMEM;
 435         }
 436         if (amdgpu_debugfs_fence_init(adev))
 437                 dev_err(adev->dev, "fence debugfs file creation failed\n");
 438
 439         return 0;
 440 }
 441
 442 /**
 443  * amdgpu_fence_driver_fini - tear down the fence driver
 444  * for all possible rings.
 445  *
 446  * @adev: amdgpu device pointer
 447  *
 448  * Tear down the fence driver for all possible rings (all asics).
 449  */
 450 void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
 451 {
 452         int i, r;
 453
 454         if (atomic_dec_and_test(&amdgpu_fence_slab_ref))
 455                 kmem_cache_destroy(amdgpu_fence_slab);
 456         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
 457                 struct amdgpu_ring *ring = adev->rings[i];
 458
 459                 if (!ring || !ring->fence_drv.initialized)
 460                         continue;
 461                 r = amdgpu_fence_wait_empty(ring);
 462                 if (r) {
 463                         /* no need to trigger GPU reset as we are unloading */
 464                         amdgpu_fence_driver_force_completion(adev);
 465                 }
 466                 wake_up_all(&ring->fence_drv.fence_queue);
 467                 amdgpu_irq_put(adev, ring->fence_drv.irq_src,
 468                                ring->fence_drv.irq_type);
 469                 amd_sched_fini(&ring->sched);
 470                 del_timer_sync(&ring->fence_drv.fallback_timer);
 471                 ring->fence_drv.initialized = false;
 472         }
 473 }
 474
 475 /**
 476  * amdgpu_fence_driver_suspend - suspend the fence driver
 477  * for all possible rings.
 478  *
 479  * @adev: amdgpu device pointer
 480  *
 481  * Suspend the fence driver for all possible rings (all asics).
 482  */
 483 void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
 484 {
 485         int i, r;
 486
 487         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
 488                 struct amdgpu_ring *ring = adev->rings[i];
 489                 if (!ring || !ring->fence_drv.initialized)
 490                         continue;
 491
 492                 /* wait for gpu to finish processing current batch */
 493                 r = amdgpu_fence_wait_empty(ring);
 494                 if (r) {
 495                         /* delay GPU reset to resume */
 496                         amdgpu_fence_driver_force_completion(adev);
 497                 }
 498
 499                 /* disable the interrupt */
 500                 amdgpu_irq_put(adev, ring->fence_drv.irq_src,
 501                                ring->fence_drv.irq_type);
 502         }
 503 }
 504
 505 /**
 506  * amdgpu_fence_driver_resume - resume the fence driver
 507  * for all possible rings.
 508  *
 509  * @adev: amdgpu device pointer
 510  *
 511  * Resume the fence driver for all possible rings (all asics).
 512  * Not all asics have all rings, so each asic will only
 513  * start the fence driver on the rings it has using
 514  * amdgpu_fence_driver_start_ring().
 515  * Returns 0 for success.
 516  */
 517 void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
 518 {
 519         int i;
 520
 521         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
 522                 struct amdgpu_ring *ring = adev->rings[i];
 523                 if (!ring || !ring->fence_drv.initialized)
 524                         continue;
 525
 526                 /* enable the interrupt */
 527                 amdgpu_irq_get(adev, ring->fence_drv.irq_src,
 528                                ring->fence_drv.irq_type);
 529         }
 530 }
 531
 532 /**
 533  * amdgpu_fence_driver_force_completion - force all fence waiter to complete
 534  *
 535  * @adev: amdgpu device pointer
 536  *
 537  * In case of GPU reset failure make sure no process keep waiting on fence
 538  * that will never complete.
 539  */
 540 void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev)
 541 {
 542         int i;
 543
 544         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
 545                 struct amdgpu_ring *ring = adev->rings[i];
 546                 if (!ring || !ring->fence_drv.initialized)
 547                         continue;
 548
 549                 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
 550         }
 551 }
 552
 553 /*
 554  * Common fence implementation
 555  */
 556
 557 static const char *amdgpu_fence_get_driver_name(struct fence *fence)
 558 {
 559         return "amdgpu";
 560 }
 561
 562 static const char *amdgpu_fence_get_timeline_name(struct fence *f)
 563 {
 564         struct amdgpu_fence *fence = to_amdgpu_fence(f);
 565         return (const char *)fence->ring->name;
 566 }
 567
 568 /**
 569  * amdgpu_fence_is_signaled - test if fence is signaled
 570  *
 571  * @f: fence to test
 572  *
 573  * Test the fence sequence number if it is already signaled. If it isn't
 574  * signaled start fence processing. Returns True if the fence is signaled.
 575  */
 576 static bool amdgpu_fence_is_signaled(struct fence *f)
 577 {
 578         struct amdgpu_fence *fence = to_amdgpu_fence(f);
 579         struct amdgpu_ring *ring = fence->ring;
 580
 581         if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
 582                 return true;
 583
 584         amdgpu_fence_process(ring);
 585
 586         if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
 587                 return true;
 588
 589         return false;
 590 }
 591
 592 /**
 593  * amdgpu_fence_check_signaled - callback from fence_queue
 594  *
 595  * this function is called with fence_queue lock held, which is also used
 596  * for the fence locking itself, so unlocked variants are used for
 597  * fence_signal, and remove_wait_queue.
 598  */
 599 static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key)
 600 {
 601         struct amdgpu_fence *fence;
 602         struct amdgpu_device *adev;
 603         u64 seq;
 604         int ret;
 605
 606         fence = container_of(wait, struct amdgpu_fence, fence_wake);
 607         adev = fence->ring->adev;
 608
 609         /*
 610          * We cannot use amdgpu_fence_process here because we're already
 611          * in the waitqueue, in a call from wake_up_all.
 612          */
 613         seq = atomic64_read(&fence->ring->fence_drv.last_seq);
 614         if (seq >= fence->seq) {
 615                 ret = fence_signal_locked(&fence->base);
 616                 if (!ret)
 617                         FENCE_TRACE(&fence->base, "signaled from irq context\n");
 618                 else
 619                         FENCE_TRACE(&fence->base, "was already signaled\n");
 620
 621                 __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);
 622                 fence_put(&fence->base);
 623         } else
 624                 FENCE_TRACE(&fence->base, "pending\n");
 625         return 0;
 626 }
 627
 628 /**
 629  * amdgpu_fence_enable_signaling - enable signalling on fence
 630  * @fence: fence
 631  *
 632  * This function is called with fence_queue lock held, and adds a callback
 633  * to fence_queue that checks if this fence is signaled, and if so it
 634  * signals the fence and removes itself.
 635  */
 636 static bool amdgpu_fence_enable_signaling(struct fence *f)
 637 {
 638         struct amdgpu_fence *fence = to_amdgpu_fence(f);
 639         struct amdgpu_ring *ring = fence->ring;
 640
 641         if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
 642                 return false;
 643
 644         fence->fence_wake.flags = 0;
 645         fence->fence_wake.private = NULL;
 646         fence->fence_wake.func = amdgpu_fence_check_signaled;
 647         __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);
 648         fence_get(f);
 649         if (!timer_pending(&ring->fence_drv.fallback_timer))
 650                 amdgpu_fence_schedule_fallback(ring);
 651         FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
 652         return true;
 653 }
 654
 655 static void amdgpu_fence_release(struct fence *f)
 656 {
 657         struct amdgpu_fence *fence = to_amdgpu_fence(f);
 658         kmem_cache_free(amdgpu_fence_slab, fence);
 659 }
 660
 661 static const struct fence_ops amdgpu_fence_ops = {
 662         .get_driver_name = amdgpu_fence_get_driver_name,
 663         .get_timeline_name = amdgpu_fence_get_timeline_name,
 664         .enable_signaling = amdgpu_fence_enable_signaling,
 665         .signaled = amdgpu_fence_is_signaled,
 666         .wait = fence_default_wait,
 667         .release = amdgpu_fence_release,
 668 };
 669
 670 /*
 671  * Fence debugfs
 672  */
 673 #if defined(CONFIG_DEBUG_FS)
 674 static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
 675 {
 676         struct drm_info_node *node = (struct drm_info_node *)m->private;
 677         struct drm_device *dev = node->minor->dev;
 678         struct amdgpu_device *adev = dev->dev_private;
 679         int i;
 680
 681         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 682                 struct amdgpu_ring *ring = adev->rings[i];
 683                 if (!ring || !ring->fence_drv.initialized)
 684                         continue;
 685
 686                 amdgpu_fence_process(ring);
 687
 688                 seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
 689                 seq_printf(m, "Last signaled fence 0x%016llx\n",
 690                            (unsigned long long)atomic64_read(&ring->fence_drv.last_seq));
 691                 seq_printf(m, "Last emitted        0x%016llx\n",
 692                            ring->fence_drv.sync_seq);
 693         }
 694         return 0;
 695 }
 696
 697 /**
 698  * amdgpu_debugfs_gpu_reset - manually trigger a gpu reset
 699  *
 700  * Manually trigger a gpu reset at the next fence wait.
 701  */
 702 static int amdgpu_debugfs_gpu_reset(struct seq_file *m, void *data)
 703 {
 704         struct drm_info_node *node = (struct drm_info_node *) m->private;
 705         struct drm_device *dev = node->minor->dev;
 706         struct amdgpu_device *adev = dev->dev_private;
 707
 708         seq_printf(m, "gpu reset\n");
 709         amdgpu_gpu_reset(adev);
 710
 711         return 0;
 712 }
 713
 714 static struct drm_info_list amdgpu_debugfs_fence_list[] = {
 715         {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
 716         {"amdgpu_gpu_reset", &amdgpu_debugfs_gpu_reset, 0, NULL}
 717 };
 718 #endif
 719
 720 int amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
 721 {
 722 #if defined(CONFIG_DEBUG_FS)
 723         return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, 2);
 724 #else
 725         return 0;
 726 #endif
 727 }
 728