drivers/gpu/host1x/job.c

   1 /*
   2  * Tegra host1x Job
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/kref.h>
  22 #include <linux/module.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/slab.h>
  25 #include <linux/vmalloc.h>
  26 #include <trace/events/host1x.h>
  27
  28 #include "channel.h"
  29 #include "dev.h"
  30 #include "host1x_bo.h"
  31 #include "job.h"
  32 #include "syncpt.h"
  33
  34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  35                                     u32 num_cmdbufs, u32 num_relocs,
  36                                     u32 num_waitchks)
  37 {
  38         struct host1x_job *job = NULL;
  39         unsigned int num_unpins = num_cmdbufs + num_relocs;
  40         u64 total;
  41         void *mem;
  42
  43         /* Check that we're not going to overflow */
  44         total = sizeof(struct host1x_job) +
  45                 num_relocs * sizeof(struct host1x_reloc) +
  46                 num_unpins * sizeof(struct host1x_job_unpin_data) +
  47                 num_waitchks * sizeof(struct host1x_waitchk) +
  48                 num_cmdbufs * sizeof(struct host1x_job_gather) +
  49                 num_unpins * sizeof(dma_addr_t) +
  50                 num_unpins * sizeof(u32 *);
  51         if (total > ULONG_MAX)
  52                 return NULL;
  53
  54         mem = job = kzalloc(total, GFP_KERNEL);
  55         if (!job)
  56                 return NULL;
  57
  58         kref_init(&job->ref);
  59         job->channel = ch;
  60
  61         /* Redistribute memory to the structs  */
  62         mem += sizeof(struct host1x_job);
  63         job->relocarray = num_relocs ? mem : NULL;
  64         mem += num_relocs * sizeof(struct host1x_reloc);
  65         job->unpins = num_unpins ? mem : NULL;
  66         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  67         job->waitchk = num_waitchks ? mem : NULL;
  68         mem += num_waitchks * sizeof(struct host1x_waitchk);
  69         job->gathers = num_cmdbufs ? mem : NULL;
  70         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  71         job->addr_phys = num_unpins ? mem : NULL;
  72
  73         job->reloc_addr_phys = job->addr_phys;
  74         job->gather_addr_phys = &job->addr_phys[num_relocs];
  75
  76         return job;
  77 }
  78
  79 struct host1x_job *host1x_job_get(struct host1x_job *job)
  80 {
  81         kref_get(&job->ref);
  82         return job;
  83 }
  84
  85 static void job_free(struct kref *ref)
  86 {
  87         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  88
  89         kfree(job);
  90 }
  91
  92 void host1x_job_put(struct host1x_job *job)
  93 {
  94         kref_put(&job->ref, job_free);
  95 }
  96
  97 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
  98                            u32 words, u32 offset)
  99 {
 100         struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
 101
 102         cur_gather->words = words;
 103         cur_gather->bo = bo;
 104         cur_gather->offset = offset;
 105         job->num_gathers++;
 106 }
 107
 108 /*
 109  * NULL an already satisfied WAIT_SYNCPT host method, by patching its
 110  * args in the command stream. The method data is changed to reference
 111  * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
 112  * with a matching threshold value of 0, so is guaranteed to be popped
 113  * by the host HW.
 114  */
 115 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
 116                                        struct host1x_bo *h, u32 offset)
 117 {
 118         void *patch_addr = NULL;
 119
 120         /* patch the wait */
 121         patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
 122         if (patch_addr) {
 123                 host1x_syncpt_patch_wait(sp,
 124                                          patch_addr + (offset & ~PAGE_MASK));
 125                 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
 126         } else
 127                 pr_err("Could not map cmdbuf for wait check\n");
 128 }
 129
 130 /*
 131  * Check driver supplied waitchk structs for syncpt thresholds
 132  * that have already been satisfied and NULL the comparison (to
 133  * avoid a wrap condition in the HW).
 134  */
 135 static int do_waitchks(struct host1x_job *job, struct host1x *host,
 136                        struct host1x_bo *patch)
 137 {
 138         int i;
 139
 140         /* compare syncpt vs wait threshold */
 141         for (i = 0; i < job->num_waitchk; i++) {
 142                 struct host1x_waitchk *wait = &job->waitchk[i];
 143                 struct host1x_syncpt *sp =
 144                         host1x_syncpt_get(host, wait->syncpt_id);
 145
 146                 /* validate syncpt id */
 147                 if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
 148                         continue;
 149
 150                 /* skip all other gathers */
 151                 if (patch != wait->bo)
 152                         continue;
 153
 154                 trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
 155                                                wait->syncpt_id, wait->thresh,
 156                                                host1x_syncpt_read_min(sp));
 157
 158                 if (host1x_syncpt_is_expired(sp, wait->thresh)) {
 159                         dev_dbg(host->dev,
 160                                 "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
 161                                 wait->syncpt_id, sp->name, wait->thresh,
 162                                 host1x_syncpt_read_min(sp));
 163
 164                         host1x_syncpt_patch_offset(sp, patch, wait->offset);
 165                 }
 166
 167                 wait->bo = NULL;
 168         }
 169
 170         return 0;
 171 }
 172
 173 static unsigned int pin_job(struct host1x_job *job)
 174 {
 175         unsigned int i;
 176
 177         job->num_unpins = 0;
 178
 179         for (i = 0; i < job->num_relocs; i++) {
 180                 struct host1x_reloc *reloc = &job->relocarray[i];
 181                 struct sg_table *sgt;
 182                 dma_addr_t phys_addr;
 183
 184                 reloc->target = host1x_bo_get(reloc->target);
 185                 if (!reloc->target)
 186                         goto unpin;
 187
 188                 phys_addr = host1x_bo_pin(reloc->target, &sgt);
 189                 if (!phys_addr)
 190                         goto unpin;
 191
 192                 job->addr_phys[job->num_unpins] = phys_addr;
 193                 job->unpins[job->num_unpins].bo = reloc->target;
 194                 job->unpins[job->num_unpins].sgt = sgt;
 195                 job->num_unpins++;
 196         }
 197
 198         for (i = 0; i < job->num_gathers; i++) {
 199                 struct host1x_job_gather *g = &job->gathers[i];
 200                 struct sg_table *sgt;
 201                 dma_addr_t phys_addr;
 202
 203                 g->bo = host1x_bo_get(g->bo);
 204                 if (!g->bo)
 205                         goto unpin;
 206
 207                 phys_addr = host1x_bo_pin(g->bo, &sgt);
 208                 if (!phys_addr)
 209                         goto unpin;
 210
 211                 job->addr_phys[job->num_unpins] = phys_addr;
 212                 job->unpins[job->num_unpins].bo = g->bo;
 213                 job->unpins[job->num_unpins].sgt = sgt;
 214                 job->num_unpins++;
 215         }
 216
 217         return job->num_unpins;
 218
 219 unpin:
 220         host1x_job_unpin(job);
 221         return 0;
 222 }
 223
 224 static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 225 {
 226         int i = 0;
 227         u32 last_page = ~0;
 228         void *cmdbuf_page_addr = NULL;
 229
 230         /* pin & patch the relocs for one gather */
 231         for (i = 0; i < job->num_relocs; i++) {
 232                 struct host1x_reloc *reloc = &job->relocarray[i];
 233                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 234                         reloc->target_offset) >> reloc->shift;
 235                 u32 *target;
 236
 237                 /* skip all other gathers */
 238                 if (cmdbuf != reloc->cmdbuf)
 239                         continue;
 240
 241                 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) {
 242                         if (cmdbuf_page_addr)
 243                                 host1x_bo_kunmap(cmdbuf, last_page,
 244                                                  cmdbuf_page_addr);
 245
 246                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 247                                         reloc->cmdbuf_offset >> PAGE_SHIFT);
 248                         last_page = reloc->cmdbuf_offset >> PAGE_SHIFT;
 249
 250                         if (unlikely(!cmdbuf_page_addr)) {
 251                                 pr_err("Could not map cmdbuf for relocation\n");
 252                                 return -ENOMEM;
 253                         }
 254                 }
 255
 256                 target = cmdbuf_page_addr + (reloc->cmdbuf_offset & ~PAGE_MASK);
 257                 *target = reloc_addr;
 258         }
 259
 260         if (cmdbuf_page_addr)
 261                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 262
 263         return 0;
 264 }
 265
 266 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 267                        unsigned int offset)
 268 {
 269         offset *= sizeof(u32);
 270
 271         if (reloc->cmdbuf != cmdbuf || reloc->cmdbuf_offset != offset)
 272                 return false;
 273
 274         return true;
 275 }
 276
 277 struct host1x_firewall {
 278         struct host1x_job *job;
 279         struct device *dev;
 280
 281         unsigned int num_relocs;
 282         struct host1x_reloc *reloc;
 283
 284         struct host1x_bo *cmdbuf_id;
 285         unsigned int offset;
 286
 287         u32 words;
 288         u32 class;
 289         u32 reg;
 290         u32 mask;
 291         u32 count;
 292 };
 293
 294 static int check_mask(struct host1x_firewall *fw)
 295 {
 296         u32 mask = fw->mask;
 297         u32 reg = fw->reg;
 298
 299         while (mask) {
 300                 if (fw->words == 0)
 301                         return -EINVAL;
 302
 303                 if (mask & 1) {
 304                         if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
 305                                 if (!fw->num_relocs)
 306                                         return -EINVAL;
 307                                 if (!check_reloc(fw->reloc, fw->cmdbuf_id,
 308                                                  fw->offset))
 309                                         return -EINVAL;
 310                                 fw->reloc++;
 311                                 fw->num_relocs--;
 312                         }
 313                         fw->words--;
 314                         fw->offset++;
 315                 }
 316                 mask >>= 1;
 317                 reg++;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static int check_incr(struct host1x_firewall *fw)
 324 {
 325         u32 count = fw->count;
 326         u32 reg = fw->reg;
 327
 328         while (count) {
 329                 if (fw->words == 0)
 330                         return -EINVAL;
 331
 332                 if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
 333                         if (!fw->num_relocs)
 334                                 return -EINVAL;
 335                         if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset))
 336                                 return -EINVAL;
 337                         fw->reloc++;
 338                         fw->num_relocs--;
 339                 }
 340                 reg++;
 341                 fw->words--;
 342                 fw->offset++;
 343                 count--;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static int check_nonincr(struct host1x_firewall *fw)
 350 {
 351         int is_addr_reg = fw->job->is_addr_reg(fw->dev, fw->class, fw->reg);
 352         u32 count = fw->count;
 353
 354         while (count) {
 355                 if (fw->words == 0)
 356                         return -EINVAL;
 357
 358                 if (is_addr_reg) {
 359                         if (!fw->num_relocs)
 360                                 return -EINVAL;
 361                         if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset))
 362                                 return -EINVAL;
 363                         fw->reloc++;
 364                         fw->num_relocs--;
 365                 }
 366                 fw->words--;
 367                 fw->offset++;
 368                 count--;
 369         }
 370
 371         return 0;
 372 }
 373
 374 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 375 {
 376         u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
 377                 (g->offset / sizeof(u32));
 378         int err = 0;
 379
 380         if (!fw->job->is_addr_reg)
 381                 return 0;
 382
 383         fw->words = g->words;
 384         fw->cmdbuf_id = g->bo;
 385         fw->offset = 0;
 386
 387         while (fw->words && !err) {
 388                 u32 word = cmdbuf_base[fw->offset];
 389                 u32 opcode = (word & 0xf0000000) >> 28;
 390
 391                 fw->mask = 0;
 392                 fw->reg = 0;
 393                 fw->count = 0;
 394                 fw->words--;
 395                 fw->offset++;
 396
 397                 switch (opcode) {
 398                 case 0:
 399                         fw->class = word >> 6 & 0x3ff;
 400                         fw->mask = word & 0x3f;
 401                         fw->reg = word >> 16 & 0xfff;
 402                         err = check_mask(fw);
 403                         if (err)
 404                                 goto out;
 405                         break;
 406                 case 1:
 407                         fw->reg = word >> 16 & 0xfff;
 408                         fw->count = word & 0xffff;
 409                         err = check_incr(fw);
 410                         if (err)
 411                                 goto out;
 412                         break;
 413
 414                 case 2:
 415                         fw->reg = word >> 16 & 0xfff;
 416                         fw->count = word & 0xffff;
 417                         err = check_nonincr(fw);
 418                         if (err)
 419                                 goto out;
 420                         break;
 421
 422                 case 3:
 423                         fw->mask = word & 0xffff;
 424                         fw->reg = word >> 16 & 0xfff;
 425                         err = check_mask(fw);
 426                         if (err)
 427                                 goto out;
 428                         break;
 429                 case 4:
 430                 case 5:
 431                 case 14:
 432                         break;
 433                 default:
 434                         err = -EINVAL;
 435                         break;
 436                 }
 437         }
 438
 439         /* No relocs should remain at this point */
 440         if (fw->num_relocs)
 441                 err = -EINVAL;
 442
 443 out:
 444         return err;
 445 }
 446
 447 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 448 {
 449         struct host1x_firewall fw;
 450         size_t size = 0;
 451         size_t offset = 0;
 452         int i;
 453
 454         fw.job = job;
 455         fw.dev = dev;
 456         fw.reloc = job->relocarray;
 457         fw.num_relocs = job->num_relocs;
 458         fw.class = 0;
 459
 460         for (i = 0; i < job->num_gathers; i++) {
 461                 struct host1x_job_gather *g = &job->gathers[i];
 462                 size += g->words * sizeof(u32);
 463         }
 464
 465         job->gather_copy_mapped = dma_alloc_writecombine(dev, size,
 466                                                          &job->gather_copy,
 467                                                          GFP_KERNEL);
 468         if (!job->gather_copy_mapped) {
 469                 int err = PTR_ERR(job->gather_copy_mapped);
 470                 job->gather_copy_mapped = NULL;
 471                 return err;
 472         }
 473
 474         job->gather_copy_size = size;
 475
 476         for (i = 0; i < job->num_gathers; i++) {
 477                 struct host1x_job_gather *g = &job->gathers[i];
 478                 void *gather;
 479
 480                 /* Copy the gather */
 481                 gather = host1x_bo_mmap(g->bo);
 482                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 483                        g->words * sizeof(u32));
 484                 host1x_bo_munmap(g->bo, gather);
 485
 486                 /* Store the location in the buffer */
 487                 g->base = job->gather_copy;
 488                 g->offset = offset;
 489
 490                 /* Validate the job */
 491                 if (validate(&fw, g))
 492                         return -EINVAL;
 493
 494                 offset += g->words * sizeof(u32);
 495         }
 496
 497         return 0;
 498 }
 499
 500 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 501 {
 502         int err;
 503         unsigned int i, j;
 504         struct host1x *host = dev_get_drvdata(dev->parent);
 505         DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
 506
 507         bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
 508         for (i = 0; i < job->num_waitchk; i++) {
 509                 u32 syncpt_id = job->waitchk[i].syncpt_id;
 510                 if (syncpt_id < host1x_syncpt_nb_pts(host))
 511                         set_bit(syncpt_id, waitchk_mask);
 512         }
 513
 514         /* get current syncpt values for waitchk */
 515         for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
 516                 host1x_syncpt_load(host->syncpt + i);
 517
 518         /* pin memory */
 519         err = pin_job(job);
 520         if (!err)
 521                 goto out;
 522
 523         /* patch gathers */
 524         for (i = 0; i < job->num_gathers; i++) {
 525                 struct host1x_job_gather *g = &job->gathers[i];
 526
 527                 /* process each gather mem only once */
 528                 if (g->handled)
 529                         continue;
 530
 531                 g->base = job->gather_addr_phys[i];
 532
 533                 for (j = 0; j < job->num_gathers; j++)
 534                         if (job->gathers[j].bo == g->bo)
 535                                 job->gathers[j].handled = true;
 536
 537                 err = do_relocs(job, g->bo);
 538                 if (err)
 539                         break;
 540
 541                 err = do_waitchks(job, host, g->bo);
 542                 if (err)
 543                         break;
 544         }
 545
 546         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) {
 547                 err = copy_gathers(job, dev);
 548                 if (err) {
 549                         host1x_job_unpin(job);
 550                         return err;
 551                 }
 552         }
 553
 554 out:
 555         wmb();
 556
 557         return err;
 558 }
 559
 560 void host1x_job_unpin(struct host1x_job *job)
 561 {
 562         unsigned int i;
 563
 564         for (i = 0; i < job->num_unpins; i++) {
 565                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 566                 host1x_bo_unpin(unpin->bo, unpin->sgt);
 567                 host1x_bo_put(unpin->bo);
 568         }
 569         job->num_unpins = 0;
 570
 571         if (job->gather_copy_size)
 572                 dma_free_writecombine(job->channel->dev, job->gather_copy_size,
 573                                       job->gather_copy_mapped,
 574                                       job->gather_copy);
 575 }
 576
 577 /*
 578  * Debug routine used to dump job entries
 579  */
 580 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 581 {
 582         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 583         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 584         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 585         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 586         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 587         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 588 }