drivers/gpu/drm/amd/amdkfd/kfd_topology.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/types.h>
  24 #include <linux/kernel.h>
  25 #include <linux/pci.h>
  26 #include <linux/errno.h>
  27 #include <linux/acpi.h>
  28 #include <linux/hash.h>
  29 #include <linux/cpufreq.h>
  30
  31 #include "kfd_priv.h"
  32 #include "kfd_crat.h"
  33 #include "kfd_topology.h"
  34
  35 static struct list_head topology_device_list;
  36 static int topology_crat_parsed;
  37 static struct kfd_system_properties sys_props;
  38
  39 static DECLARE_RWSEM(topology_lock);
  40
  41 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
  42 {
  43         struct kfd_topology_device *top_dev;
  44         struct kfd_dev *device = NULL;
  45
  46         down_read(&topology_lock);
  47
  48         list_for_each_entry(top_dev, &topology_device_list, list)
  49                 if (top_dev->gpu_id == gpu_id) {
  50                         device = top_dev->gpu;
  51                         break;
  52                 }
  53
  54         up_read(&topology_lock);
  55
  56         return device;
  57 }
  58
  59 struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
  60 {
  61         struct kfd_topology_device *top_dev;
  62         struct kfd_dev *device = NULL;
  63
  64         down_read(&topology_lock);
  65
  66         list_for_each_entry(top_dev, &topology_device_list, list)
  67                 if (top_dev->gpu->pdev == pdev) {
  68                         device = top_dev->gpu;
  69                         break;
  70                 }
  71
  72         up_read(&topology_lock);
  73
  74         return device;
  75 }
  76
  77 static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
  78 {
  79         struct acpi_table_header *crat_table;
  80         acpi_status status;
  81
  82         if (!size)
  83                 return -EINVAL;
  84
  85         /*
  86          * Fetch the CRAT table from ACPI
  87          */
  88         status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
  89         if (status == AE_NOT_FOUND) {
  90                 pr_warn("CRAT table not found\n");
  91                 return -ENODATA;
  92         } else if (ACPI_FAILURE(status)) {
  93                 const char *err = acpi_format_exception(status);
  94
  95                 pr_err("CRAT table error: %s\n", err);
  96                 return -EINVAL;
  97         }
  98
  99         if (*size >= crat_table->length && crat_image != NULL)
 100                 memcpy(crat_image, crat_table, crat_table->length);
 101
 102         *size = crat_table->length;
 103
 104         return 0;
 105 }
 106
 107 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
 108                 struct crat_subtype_computeunit *cu)
 109 {
 110         BUG_ON(!dev);
 111         BUG_ON(!cu);
 112
 113         dev->node_props.cpu_cores_count = cu->num_cpu_cores;
 114         dev->node_props.cpu_core_id_base = cu->processor_id_low;
 115         if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
 116                 dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
 117
 118         pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
 119                         cu->processor_id_low);
 120 }
 121
 122 static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
 123                 struct crat_subtype_computeunit *cu)
 124 {
 125         BUG_ON(!dev);
 126         BUG_ON(!cu);
 127
 128         dev->node_props.simd_id_base = cu->processor_id_low;
 129         dev->node_props.simd_count = cu->num_simd_cores;
 130         dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
 131         dev->node_props.max_waves_per_simd = cu->max_waves_simd;
 132         dev->node_props.wave_front_size = cu->wave_front_size;
 133         dev->node_props.mem_banks_count = cu->num_banks;
 134         dev->node_props.array_count = cu->num_arrays;
 135         dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
 136         dev->node_props.simd_per_cu = cu->num_simd_per_cu;
 137         dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
 138         if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
 139                 dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
 140         pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores,
 141                                 cu->processor_id_low);
 142 }
 143
 144 /* kfd_parse_subtype_cu is called when the topology mutex is already acquired */
 145 static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
 146 {
 147         struct kfd_topology_device *dev;
 148         int i = 0;
 149
 150         BUG_ON(!cu);
 151
 152         pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
 153                         cu->proximity_domain, cu->hsa_capability);
 154         list_for_each_entry(dev, &topology_device_list, list) {
 155                 if (cu->proximity_domain == i) {
 156                         if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
 157                                 kfd_populated_cu_info_cpu(dev, cu);
 158
 159                         if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
 160                                 kfd_populated_cu_info_gpu(dev, cu);
 161                         break;
 162                 }
 163                 i++;
 164         }
 165
 166         return 0;
 167 }
 168
 169 /*
 170  * kfd_parse_subtype_mem is called when the topology mutex is
 171  * already acquired
 172  */
 173 static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
 174 {
 175         struct kfd_mem_properties *props;
 176         struct kfd_topology_device *dev;
 177         int i = 0;
 178
 179         BUG_ON(!mem);
 180
 181         pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
 182                         mem->promixity_domain);
 183         list_for_each_entry(dev, &topology_device_list, list) {
 184                 if (mem->promixity_domain == i) {
 185                         props = kfd_alloc_struct(props);
 186                         if (props == NULL)
 187                                 return -ENOMEM;
 188
 189                         if (dev->node_props.cpu_cores_count == 0)
 190                                 props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
 191                         else
 192                                 props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
 193
 194                         if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
 195                                 props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
 196                         if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
 197                                 props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
 198
 199                         props->size_in_bytes =
 200                                 ((uint64_t)mem->length_high << 32) +
 201                                                         mem->length_low;
 202                         props->width = mem->width;
 203
 204                         dev->mem_bank_count++;
 205                         list_add_tail(&props->list, &dev->mem_props);
 206
 207                         break;
 208                 }
 209                 i++;
 210         }
 211
 212         return 0;
 213 }
 214
 215 /*
 216  * kfd_parse_subtype_cache is called when the topology mutex
 217  * is already acquired
 218  */
 219 static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
 220 {
 221         struct kfd_cache_properties *props;
 222         struct kfd_topology_device *dev;
 223         uint32_t id;
 224
 225         BUG_ON(!cache);
 226
 227         id = cache->processor_id_low;
 228
 229         pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
 230         list_for_each_entry(dev, &topology_device_list, list)
 231                 if (id == dev->node_props.cpu_core_id_base ||
 232                     id == dev->node_props.simd_id_base) {
 233                         props = kfd_alloc_struct(props);
 234                         if (props == NULL)
 235                                 return -ENOMEM;
 236
 237                         props->processor_id_low = id;
 238                         props->cache_level = cache->cache_level;
 239                         props->cache_size = cache->cache_size;
 240                         props->cacheline_size = cache->cache_line_size;
 241                         props->cachelines_per_tag = cache->lines_per_tag;
 242                         props->cache_assoc = cache->associativity;
 243                         props->cache_latency = cache->cache_latency;
 244
 245                         if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
 246                                 props->cache_type |= HSA_CACHE_TYPE_DATA;
 247                         if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
 248                                 props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
 249                         if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
 250                                 props->cache_type |= HSA_CACHE_TYPE_CPU;
 251                         if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
 252                                 props->cache_type |= HSA_CACHE_TYPE_HSACU;
 253
 254                         dev->cache_count++;
 255                         dev->node_props.caches_count++;
 256                         list_add_tail(&props->list, &dev->cache_props);
 257
 258                         break;
 259                 }
 260
 261         return 0;
 262 }
 263
 264 /*
 265  * kfd_parse_subtype_iolink is called when the topology mutex
 266  * is already acquired
 267  */
 268 static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
 269 {
 270         struct kfd_iolink_properties *props;
 271         struct kfd_topology_device *dev;
 272         uint32_t i = 0;
 273         uint32_t id_from;
 274         uint32_t id_to;
 275
 276         BUG_ON(!iolink);
 277
 278         id_from = iolink->proximity_domain_from;
 279         id_to = iolink->proximity_domain_to;
 280
 281         pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from);
 282         list_for_each_entry(dev, &topology_device_list, list) {
 283                 if (id_from == i) {
 284                         props = kfd_alloc_struct(props);
 285                         if (props == NULL)
 286                                 return -ENOMEM;
 287
 288                         props->node_from = id_from;
 289                         props->node_to = id_to;
 290                         props->ver_maj = iolink->version_major;
 291                         props->ver_min = iolink->version_minor;
 292
 293                         /*
 294                          * weight factor (derived from CDIR), currently always 1
 295                          */
 296                         props->weight = 1;
 297
 298                         props->min_latency = iolink->minimum_latency;
 299                         props->max_latency = iolink->maximum_latency;
 300                         props->min_bandwidth = iolink->minimum_bandwidth_mbs;
 301                         props->max_bandwidth = iolink->maximum_bandwidth_mbs;
 302                         props->rec_transfer_size =
 303                                         iolink->recommended_transfer_size;
 304
 305                         dev->io_link_count++;
 306                         dev->node_props.io_links_count++;
 307                         list_add_tail(&props->list, &dev->io_link_props);
 308
 309                         break;
 310                 }
 311                 i++;
 312         }
 313
 314         return 0;
 315 }
 316
 317 static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
 318 {
 319         struct crat_subtype_computeunit *cu;
 320         struct crat_subtype_memory *mem;
 321         struct crat_subtype_cache *cache;
 322         struct crat_subtype_iolink *iolink;
 323         int ret = 0;
 324
 325         BUG_ON(!sub_type_hdr);
 326
 327         switch (sub_type_hdr->type) {
 328         case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
 329                 cu = (struct crat_subtype_computeunit *)sub_type_hdr;
 330                 ret = kfd_parse_subtype_cu(cu);
 331                 break;
 332         case CRAT_SUBTYPE_MEMORY_AFFINITY:
 333                 mem = (struct crat_subtype_memory *)sub_type_hdr;
 334                 ret = kfd_parse_subtype_mem(mem);
 335                 break;
 336         case CRAT_SUBTYPE_CACHE_AFFINITY:
 337                 cache = (struct crat_subtype_cache *)sub_type_hdr;
 338                 ret = kfd_parse_subtype_cache(cache);
 339                 break;
 340         case CRAT_SUBTYPE_TLB_AFFINITY:
 341                 /*
 342                  * For now, nothing to do here
 343                  */
 344                 pr_info("Found TLB entry in CRAT table (not processing)\n");
 345                 break;
 346         case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
 347                 /*
 348                  * For now, nothing to do here
 349                  */
 350                 pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n");
 351                 break;
 352         case CRAT_SUBTYPE_IOLINK_AFFINITY:
 353                 iolink = (struct crat_subtype_iolink *)sub_type_hdr;
 354                 ret = kfd_parse_subtype_iolink(iolink);
 355                 break;
 356         default:
 357                 pr_warn("Unknown subtype (%d) in CRAT\n",
 358                                 sub_type_hdr->type);
 359         }
 360
 361         return ret;
 362 }
 363
 364 static void kfd_release_topology_device(struct kfd_topology_device *dev)
 365 {
 366         struct kfd_mem_properties *mem;
 367         struct kfd_cache_properties *cache;
 368         struct kfd_iolink_properties *iolink;
 369
 370         BUG_ON(!dev);
 371
 372         list_del(&dev->list);
 373
 374         while (dev->mem_props.next != &dev->mem_props) {
 375                 mem = container_of(dev->mem_props.next,
 376                                 struct kfd_mem_properties, list);
 377                 list_del(&mem->list);
 378                 kfree(mem);
 379         }
 380
 381         while (dev->cache_props.next != &dev->cache_props) {
 382                 cache = container_of(dev->cache_props.next,
 383                                 struct kfd_cache_properties, list);
 384                 list_del(&cache->list);
 385                 kfree(cache);
 386         }
 387
 388         while (dev->io_link_props.next != &dev->io_link_props) {
 389                 iolink = container_of(dev->io_link_props.next,
 390                                 struct kfd_iolink_properties, list);
 391                 list_del(&iolink->list);
 392                 kfree(iolink);
 393         }
 394
 395         kfree(dev);
 396
 397         sys_props.num_devices--;
 398 }
 399
 400 static void kfd_release_live_view(void)
 401 {
 402         struct kfd_topology_device *dev;
 403
 404         while (topology_device_list.next != &topology_device_list) {
 405                 dev = container_of(topology_device_list.next,
 406                                  struct kfd_topology_device, list);
 407                 kfd_release_topology_device(dev);
 408 }
 409
 410         memset(&sys_props, 0, sizeof(sys_props));
 411 }
 412
 413 static struct kfd_topology_device *kfd_create_topology_device(void)
 414 {
 415         struct kfd_topology_device *dev;
 416
 417         dev = kfd_alloc_struct(dev);
 418         if (dev == NULL) {
 419                 pr_err("No memory to allocate a topology device");
 420                 return NULL;
 421         }
 422
 423         INIT_LIST_HEAD(&dev->mem_props);
 424         INIT_LIST_HEAD(&dev->cache_props);
 425         INIT_LIST_HEAD(&dev->io_link_props);
 426
 427         list_add_tail(&dev->list, &topology_device_list);
 428         sys_props.num_devices++;
 429
 430         return dev;
 431 }
 432
 433 static int kfd_parse_crat_table(void *crat_image)
 434 {
 435         struct kfd_topology_device *top_dev;
 436         struct crat_subtype_generic *sub_type_hdr;
 437         uint16_t node_id;
 438         int ret;
 439         struct crat_header *crat_table = (struct crat_header *)crat_image;
 440         uint16_t num_nodes;
 441         uint32_t image_len;
 442
 443         if (!crat_image)
 444                 return -EINVAL;
 445
 446         num_nodes = crat_table->num_domains;
 447         image_len = crat_table->length;
 448
 449         pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
 450
 451         for (node_id = 0; node_id < num_nodes; node_id++) {
 452                 top_dev = kfd_create_topology_device();
 453                 if (!top_dev) {
 454                         kfd_release_live_view();
 455                         return -ENOMEM;
 456                 }
 457         }
 458
 459         sys_props.platform_id =
 460                 (*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK;
 461         sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id);
 462         sys_props.platform_rev = crat_table->revision;
 463
 464         sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
 465         while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
 466                         ((char *)crat_image) + image_len) {
 467                 if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
 468                         ret = kfd_parse_subtype(sub_type_hdr);
 469                         if (ret != 0) {
 470                                 kfd_release_live_view();
 471                                 return ret;
 472                         }
 473                 }
 474
 475                 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
 476                                 sub_type_hdr->length);
 477         }
 478
 479         sys_props.generation_count++;
 480         topology_crat_parsed = 1;
 481
 482         return 0;
 483 }
 484
 485
 486 #define sysfs_show_gen_prop(buffer, fmt, ...) \
 487                 snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
 488 #define sysfs_show_32bit_prop(buffer, name, value) \
 489                 sysfs_show_gen_prop(buffer, "%s %u\n", name, value)
 490 #define sysfs_show_64bit_prop(buffer, name, value) \
 491                 sysfs_show_gen_prop(buffer, "%s %llu\n", name, value)
 492 #define sysfs_show_32bit_val(buffer, value) \
 493                 sysfs_show_gen_prop(buffer, "%u\n", value)
 494 #define sysfs_show_str_val(buffer, value) \
 495                 sysfs_show_gen_prop(buffer, "%s\n", value)
 496
 497 static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
 498                 char *buffer)
 499 {
 500         ssize_t ret;
 501
 502         /* Making sure that the buffer is an empty string */
 503         buffer[0] = 0;
 504
 505         if (attr == &sys_props.attr_genid) {
 506                 ret = sysfs_show_32bit_val(buffer, sys_props.generation_count);
 507         } else if (attr == &sys_props.attr_props) {
 508                 sysfs_show_64bit_prop(buffer, "platform_oem",
 509                                 sys_props.platform_oem);
 510                 sysfs_show_64bit_prop(buffer, "platform_id",
 511                                 sys_props.platform_id);
 512                 ret = sysfs_show_64bit_prop(buffer, "platform_rev",
 513                                 sys_props.platform_rev);
 514         } else {
 515                 ret = -EINVAL;
 516         }
 517
 518         return ret;
 519 }
 520
 521 static const struct sysfs_ops sysprops_ops = {
 522         .show = sysprops_show,
 523 };
 524
 525 static struct kobj_type sysprops_type = {
 526         .sysfs_ops = &sysprops_ops,
 527 };
 528
 529 static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
 530                 char *buffer)
 531 {
 532         ssize_t ret;
 533         struct kfd_iolink_properties *iolink;
 534
 535         /* Making sure that the buffer is an empty string */
 536         buffer[0] = 0;
 537
 538         iolink = container_of(attr, struct kfd_iolink_properties, attr);
 539         sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type);
 540         sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj);
 541         sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min);
 542         sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from);
 543         sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to);
 544         sysfs_show_32bit_prop(buffer, "weight", iolink->weight);
 545         sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency);
 546         sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency);
 547         sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth);
 548         sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth);
 549         sysfs_show_32bit_prop(buffer, "recommended_transfer_size",
 550                         iolink->rec_transfer_size);
 551         ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags);
 552
 553         return ret;
 554 }
 555
 556 static const struct sysfs_ops iolink_ops = {
 557         .show = iolink_show,
 558 };
 559
 560 static struct kobj_type iolink_type = {
 561         .sysfs_ops = &iolink_ops,
 562 };
 563
 564 static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
 565                 char *buffer)
 566 {
 567         ssize_t ret;
 568         struct kfd_mem_properties *mem;
 569
 570         /* Making sure that the buffer is an empty string */
 571         buffer[0] = 0;
 572
 573         mem = container_of(attr, struct kfd_mem_properties, attr);
 574         sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type);
 575         sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes);
 576         sysfs_show_32bit_prop(buffer, "flags", mem->flags);
 577         sysfs_show_32bit_prop(buffer, "width", mem->width);
 578         ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max);
 579
 580         return ret;
 581 }
 582
 583 static const struct sysfs_ops mem_ops = {
 584         .show = mem_show,
 585 };
 586
 587 static struct kobj_type mem_type = {
 588         .sysfs_ops = &mem_ops,
 589 };
 590
 591 static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 592                 char *buffer)
 593 {
 594         ssize_t ret;
 595         uint32_t i;
 596         struct kfd_cache_properties *cache;
 597
 598         /* Making sure that the buffer is an empty string */
 599         buffer[0] = 0;
 600
 601         cache = container_of(attr, struct kfd_cache_properties, attr);
 602         sysfs_show_32bit_prop(buffer, "processor_id_low",
 603                         cache->processor_id_low);
 604         sysfs_show_32bit_prop(buffer, "level", cache->cache_level);
 605         sysfs_show_32bit_prop(buffer, "size", cache->cache_size);
 606         sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size);
 607         sysfs_show_32bit_prop(buffer, "cache_lines_per_tag",
 608                         cache->cachelines_per_tag);
 609         sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc);
 610         sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
 611         sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
 612         snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
 613         for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++)
 614                 ret = snprintf(buffer, PAGE_SIZE, "%s%d%s",
 615                                 buffer, cache->sibling_map[i],
 616                                 (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ?
 617                                                 "\n" : ",");
 618
 619         return ret;
 620 }
 621
 622 static const struct sysfs_ops cache_ops = {
 623         .show = kfd_cache_show,
 624 };
 625
 626 static struct kobj_type cache_type = {
 627         .sysfs_ops = &cache_ops,
 628 };
 629
 630 static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 631                 char *buffer)
 632 {
 633         ssize_t ret;
 634         struct kfd_topology_device *dev;
 635         char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
 636         uint32_t i;
 637
 638         /* Making sure that the buffer is an empty string */
 639         buffer[0] = 0;
 640
 641         if (strcmp(attr->name, "gpu_id") == 0) {
 642                 dev = container_of(attr, struct kfd_topology_device,
 643                                 attr_gpuid);
 644                 ret = sysfs_show_32bit_val(buffer, dev->gpu_id);
 645         } else if (strcmp(attr->name, "name") == 0) {
 646                 dev = container_of(attr, struct kfd_topology_device,
 647                                 attr_name);
 648                 for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) {
 649                         public_name[i] =
 650                                         (char)dev->node_props.marketing_name[i];
 651                         if (dev->node_props.marketing_name[i] == 0)
 652                                 break;
 653                 }
 654                 public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0;
 655                 ret = sysfs_show_str_val(buffer, public_name);
 656         } else {
 657                 dev = container_of(attr, struct kfd_topology_device,
 658                                 attr_props);
 659                 sysfs_show_32bit_prop(buffer, "cpu_cores_count",
 660                                 dev->node_props.cpu_cores_count);
 661                 sysfs_show_32bit_prop(buffer, "simd_count",
 662                                 dev->node_props.simd_count);
 663
 664                 if (dev->mem_bank_count < dev->node_props.mem_banks_count) {
 665                         pr_warn("kfd: mem_banks_count truncated from %d to %d\n",
 666                                         dev->node_props.mem_banks_count,
 667                                         dev->mem_bank_count);
 668                         sysfs_show_32bit_prop(buffer, "mem_banks_count",
 669                                         dev->mem_bank_count);
 670                 } else {
 671                         sysfs_show_32bit_prop(buffer, "mem_banks_count",
 672                                         dev->node_props.mem_banks_count);
 673                 }
 674
 675                 sysfs_show_32bit_prop(buffer, "caches_count",
 676                                 dev->node_props.caches_count);
 677                 sysfs_show_32bit_prop(buffer, "io_links_count",
 678                                 dev->node_props.io_links_count);
 679                 sysfs_show_32bit_prop(buffer, "cpu_core_id_base",
 680                                 dev->node_props.cpu_core_id_base);
 681                 sysfs_show_32bit_prop(buffer, "simd_id_base",
 682                                 dev->node_props.simd_id_base);
 683                 sysfs_show_32bit_prop(buffer, "capability",
 684                                 dev->node_props.capability);
 685                 sysfs_show_32bit_prop(buffer, "max_waves_per_simd",
 686                                 dev->node_props.max_waves_per_simd);
 687                 sysfs_show_32bit_prop(buffer, "lds_size_in_kb",
 688                                 dev->node_props.lds_size_in_kb);
 689                 sysfs_show_32bit_prop(buffer, "gds_size_in_kb",
 690                                 dev->node_props.gds_size_in_kb);
 691                 sysfs_show_32bit_prop(buffer, "wave_front_size",
 692                                 dev->node_props.wave_front_size);
 693                 sysfs_show_32bit_prop(buffer, "array_count",
 694                                 dev->node_props.array_count);
 695                 sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine",
 696                                 dev->node_props.simd_arrays_per_engine);
 697                 sysfs_show_32bit_prop(buffer, "cu_per_simd_array",
 698                                 dev->node_props.cu_per_simd_array);
 699                 sysfs_show_32bit_prop(buffer, "simd_per_cu",
 700                                 dev->node_props.simd_per_cu);
 701                 sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu",
 702                                 dev->node_props.max_slots_scratch_cu);
 703                 sysfs_show_32bit_prop(buffer, "engine_id",
 704                                 dev->node_props.engine_id);
 705                 sysfs_show_32bit_prop(buffer, "vendor_id",
 706                                 dev->node_props.vendor_id);
 707                 sysfs_show_32bit_prop(buffer, "device_id",
 708                                 dev->node_props.device_id);
 709                 sysfs_show_32bit_prop(buffer, "location_id",
 710                                 dev->node_props.location_id);
 711
 712                 if (dev->gpu) {
 713                         sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
 714                                         kfd2kgd->get_max_engine_clock_in_mhz(
 715                                                 dev->gpu->kgd));
 716                         sysfs_show_64bit_prop(buffer, "local_mem_size",
 717                                         kfd2kgd->get_vmem_size(dev->gpu->kgd));
 718                 }
 719
 720                 ret = sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute",
 721                                 cpufreq_quick_get_max(0)/1000);
 722         }
 723
 724         return ret;
 725 }
 726
 727 static const struct sysfs_ops node_ops = {
 728         .show = node_show,
 729 };
 730
 731 static struct kobj_type node_type = {
 732         .sysfs_ops = &node_ops,
 733 };
 734
 735 static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
 736 {
 737         sysfs_remove_file(kobj, attr);
 738         kobject_del(kobj);
 739         kobject_put(kobj);
 740 }
 741
 742 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 743 {
 744         struct kfd_iolink_properties *iolink;
 745         struct kfd_cache_properties *cache;
 746         struct kfd_mem_properties *mem;
 747
 748         BUG_ON(!dev);
 749
 750         if (dev->kobj_iolink) {
 751                 list_for_each_entry(iolink, &dev->io_link_props, list)
 752                         if (iolink->kobj) {
 753                                 kfd_remove_sysfs_file(iolink->kobj,
 754                                                         &iolink->attr);
 755                                 iolink->kobj = NULL;
 756                         }
 757                 kobject_del(dev->kobj_iolink);
 758                 kobject_put(dev->kobj_iolink);
 759                 dev->kobj_iolink = NULL;
 760         }
 761
 762         if (dev->kobj_cache) {
 763                 list_for_each_entry(cache, &dev->cache_props, list)
 764                         if (cache->kobj) {
 765                                 kfd_remove_sysfs_file(cache->kobj,
 766                                                         &cache->attr);
 767                                 cache->kobj = NULL;
 768                         }
 769                 kobject_del(dev->kobj_cache);
 770                 kobject_put(dev->kobj_cache);
 771                 dev->kobj_cache = NULL;
 772         }
 773
 774         if (dev->kobj_mem) {
 775                 list_for_each_entry(mem, &dev->mem_props, list)
 776                         if (mem->kobj) {
 777                                 kfd_remove_sysfs_file(mem->kobj, &mem->attr);
 778                                 mem->kobj = NULL;
 779                         }
 780                 kobject_del(dev->kobj_mem);
 781                 kobject_put(dev->kobj_mem);
 782                 dev->kobj_mem = NULL;
 783         }
 784
 785         if (dev->kobj_node) {
 786                 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
 787                 sysfs_remove_file(dev->kobj_node, &dev->attr_name);
 788                 sysfs_remove_file(dev->kobj_node, &dev->attr_props);
 789                 kobject_del(dev->kobj_node);
 790                 kobject_put(dev->kobj_node);
 791                 dev->kobj_node = NULL;
 792         }
 793 }
 794
 795 static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 796                 uint32_t id)
 797 {
 798         struct kfd_iolink_properties *iolink;
 799         struct kfd_cache_properties *cache;
 800         struct kfd_mem_properties *mem;
 801         int ret;
 802         uint32_t i;
 803
 804         BUG_ON(!dev);
 805
 806         /*
 807          * Creating the sysfs folders
 808          */
 809         BUG_ON(dev->kobj_node);
 810         dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
 811         if (!dev->kobj_node)
 812                 return -ENOMEM;
 813
 814         ret = kobject_init_and_add(dev->kobj_node, &node_type,
 815                         sys_props.kobj_nodes, "%d", id);
 816         if (ret < 0)
 817                 return ret;
 818
 819         dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
 820         if (!dev->kobj_mem)
 821                 return -ENOMEM;
 822
 823         dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
 824         if (!dev->kobj_cache)
 825                 return -ENOMEM;
 826
 827         dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
 828         if (!dev->kobj_iolink)
 829                 return -ENOMEM;
 830
 831         /*
 832          * Creating sysfs files for node properties
 833          */
 834         dev->attr_gpuid.name = "gpu_id";
 835         dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
 836         sysfs_attr_init(&dev->attr_gpuid);
 837         dev->attr_name.name = "name";
 838         dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
 839         sysfs_attr_init(&dev->attr_name);
 840         dev->attr_props.name = "properties";
 841         dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
 842         sysfs_attr_init(&dev->attr_props);
 843         ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
 844         if (ret < 0)
 845                 return ret;
 846         ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
 847         if (ret < 0)
 848                 return ret;
 849         ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
 850         if (ret < 0)
 851                 return ret;
 852
 853         i = 0;
 854         list_for_each_entry(mem, &dev->mem_props, list) {
 855                 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 856                 if (!mem->kobj)
 857                         return -ENOMEM;
 858                 ret = kobject_init_and_add(mem->kobj, &mem_type,
 859                                 dev->kobj_mem, "%d", i);
 860                 if (ret < 0)
 861                         return ret;
 862
 863                 mem->attr.name = "properties";
 864                 mem->attr.mode = KFD_SYSFS_FILE_MODE;
 865                 sysfs_attr_init(&mem->attr);
 866                 ret = sysfs_create_file(mem->kobj, &mem->attr);
 867                 if (ret < 0)
 868                         return ret;
 869                 i++;
 870         }
 871
 872         i = 0;
 873         list_for_each_entry(cache, &dev->cache_props, list) {
 874                 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 875                 if (!cache->kobj)
 876                         return -ENOMEM;
 877                 ret = kobject_init_and_add(cache->kobj, &cache_type,
 878                                 dev->kobj_cache, "%d", i);
 879                 if (ret < 0)
 880                         return ret;
 881
 882                 cache->attr.name = "properties";
 883                 cache->attr.mode = KFD_SYSFS_FILE_MODE;
 884                 sysfs_attr_init(&cache->attr);
 885                 ret = sysfs_create_file(cache->kobj, &cache->attr);
 886                 if (ret < 0)
 887                         return ret;
 888                 i++;
 889         }
 890
 891         i = 0;
 892         list_for_each_entry(iolink, &dev->io_link_props, list) {
 893                 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 894                 if (!iolink->kobj)
 895                         return -ENOMEM;
 896                 ret = kobject_init_and_add(iolink->kobj, &iolink_type,
 897                                 dev->kobj_iolink, "%d", i);
 898                 if (ret < 0)
 899                         return ret;
 900
 901                 iolink->attr.name = "properties";
 902                 iolink->attr.mode = KFD_SYSFS_FILE_MODE;
 903                 sysfs_attr_init(&iolink->attr);
 904                 ret = sysfs_create_file(iolink->kobj, &iolink->attr);
 905                 if (ret < 0)
 906                         return ret;
 907                 i++;
 908 }
 909
 910         return 0;
 911 }
 912
 913 static int kfd_build_sysfs_node_tree(void)
 914 {
 915         struct kfd_topology_device *dev;
 916         int ret;
 917         uint32_t i = 0;
 918
 919         list_for_each_entry(dev, &topology_device_list, list) {
 920                 ret = kfd_build_sysfs_node_entry(dev, 0);
 921                 if (ret < 0)
 922                         return ret;
 923                 i++;
 924         }
 925
 926         return 0;
 927 }
 928
 929 static void kfd_remove_sysfs_node_tree(void)
 930 {
 931         struct kfd_topology_device *dev;
 932
 933         list_for_each_entry(dev, &topology_device_list, list)
 934                 kfd_remove_sysfs_node_entry(dev);
 935 }
 936
 937 static int kfd_topology_update_sysfs(void)
 938 {
 939         int ret;
 940
 941         pr_info("Creating topology SYSFS entries\n");
 942         if (sys_props.kobj_topology == NULL) {
 943                 sys_props.kobj_topology =
 944                                 kfd_alloc_struct(sys_props.kobj_topology);
 945                 if (!sys_props.kobj_topology)
 946                         return -ENOMEM;
 947
 948                 ret = kobject_init_and_add(sys_props.kobj_topology,
 949                                 &sysprops_type,  &kfd_device->kobj,
 950                                 "topology");
 951                 if (ret < 0)
 952                         return ret;
 953
 954                 sys_props.kobj_nodes = kobject_create_and_add("nodes",
 955                                 sys_props.kobj_topology);
 956                 if (!sys_props.kobj_nodes)
 957                         return -ENOMEM;
 958
 959                 sys_props.attr_genid.name = "generation_id";
 960                 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
 961                 sysfs_attr_init(&sys_props.attr_genid);
 962                 ret = sysfs_create_file(sys_props.kobj_topology,
 963                                 &sys_props.attr_genid);
 964                 if (ret < 0)
 965                         return ret;
 966
 967                 sys_props.attr_props.name = "system_properties";
 968                 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
 969                 sysfs_attr_init(&sys_props.attr_props);
 970                 ret = sysfs_create_file(sys_props.kobj_topology,
 971                                 &sys_props.attr_props);
 972                 if (ret < 0)
 973                         return ret;
 974         }
 975
 976         kfd_remove_sysfs_node_tree();
 977
 978         return kfd_build_sysfs_node_tree();
 979 }
 980
 981 static void kfd_topology_release_sysfs(void)
 982 {
 983         kfd_remove_sysfs_node_tree();
 984         if (sys_props.kobj_topology) {
 985                 sysfs_remove_file(sys_props.kobj_topology,
 986                                 &sys_props.attr_genid);
 987                 sysfs_remove_file(sys_props.kobj_topology,
 988                                 &sys_props.attr_props);
 989                 if (sys_props.kobj_nodes) {
 990                         kobject_del(sys_props.kobj_nodes);
 991                         kobject_put(sys_props.kobj_nodes);
 992                         sys_props.kobj_nodes = NULL;
 993                 }
 994                 kobject_del(sys_props.kobj_topology);
 995                 kobject_put(sys_props.kobj_topology);
 996                 sys_props.kobj_topology = NULL;
 997         }
 998 }
 999
1000 int kfd_topology_init(void)
1001 {
1002         void *crat_image = NULL;
1003         size_t image_size = 0;
1004         int ret;
1005
1006         /*
1007          * Initialize the head for the topology device list
1008          */
1009         INIT_LIST_HEAD(&topology_device_list);
1010         init_rwsem(&topology_lock);
1011         topology_crat_parsed = 0;
1012
1013         memset(&sys_props, 0, sizeof(sys_props));
1014
1015         /*
1016          * Get the CRAT image from the ACPI
1017          */
1018         ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
1019         if (ret == 0 && image_size > 0) {
1020                 pr_info("Found CRAT image with size=%zd\n", image_size);
1021                 crat_image = kmalloc(image_size, GFP_KERNEL);
1022                 if (!crat_image) {
1023                         ret = -ENOMEM;
1024                         pr_err("No memory for allocating CRAT image\n");
1025                         goto err;
1026                 }
1027                 ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
1028
1029                 if (ret == 0) {
1030                         down_write(&topology_lock);
1031                         ret = kfd_parse_crat_table(crat_image);
1032                         if (ret == 0)
1033                                 ret = kfd_topology_update_sysfs();
1034                         up_write(&topology_lock);
1035                 } else {
1036                         pr_err("Couldn't get CRAT table size from ACPI\n");
1037                 }
1038                 kfree(crat_image);
1039         } else if (ret == -ENODATA) {
1040                 ret = 0;
1041         } else {
1042                 pr_err("Couldn't get CRAT table size from ACPI\n");
1043         }
1044
1045 err:
1046         pr_info("Finished initializing topology ret=%d\n", ret);
1047         return ret;
1048 }
1049
1050 void kfd_topology_shutdown(void)
1051 {
1052         kfd_topology_release_sysfs();
1053         kfd_release_live_view();
1054 }
1055
1056 static void kfd_debug_print_topology(void)
1057 {
1058         struct kfd_topology_device *dev;
1059         uint32_t i = 0;
1060
1061         pr_info("DEBUG PRINT OF TOPOLOGY:");
1062         list_for_each_entry(dev, &topology_device_list, list) {
1063                 pr_info("Node: %d\n", i);
1064                 pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no"));
1065                 pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count);
1066                 pr_info("\tSIMD count: %d", dev->node_props.simd_count);
1067                 i++;
1068         }
1069 }
1070
1071 static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
1072 {
1073         uint32_t hashout;
1074         uint32_t buf[7];
1075         int i;
1076
1077         if (!gpu)
1078                 return 0;
1079
1080         buf[0] = gpu->pdev->devfn;
1081         buf[1] = gpu->pdev->subsystem_vendor;
1082         buf[2] = gpu->pdev->subsystem_device;
1083         buf[3] = gpu->pdev->device;
1084         buf[4] = gpu->pdev->bus->number;
1085         buf[5] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) & 0xffffffff);
1086         buf[6] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) >> 32);
1087
1088         for (i = 0, hashout = 0; i < 7; i++)
1089                 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
1090
1091         return hashout;
1092 }
1093
1094 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
1095 {
1096         struct kfd_topology_device *dev;
1097         struct kfd_topology_device *out_dev = NULL;
1098
1099         BUG_ON(!gpu);
1100
1101         list_for_each_entry(dev, &topology_device_list, list)
1102                 if (dev->gpu == NULL && dev->node_props.simd_count > 0) {
1103                         dev->gpu = gpu;
1104                         out_dev = dev;
1105                         break;
1106                 }
1107
1108         return out_dev;
1109 }
1110
1111 static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1112 {
1113         /*
1114          * TODO: Generate an event for thunk about the arrival/removal
1115          * of the GPU
1116          */
1117 }
1118
1119 int kfd_topology_add_device(struct kfd_dev *gpu)
1120 {
1121         uint32_t gpu_id;
1122         struct kfd_topology_device *dev;
1123         int res;
1124
1125         BUG_ON(!gpu);
1126
1127         gpu_id = kfd_generate_gpu_id(gpu);
1128
1129         pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
1130
1131         down_write(&topology_lock);
1132         /*
1133          * Try to assign the GPU to existing topology device (generated from
1134          * CRAT table
1135          */
1136         dev = kfd_assign_gpu(gpu);
1137         if (!dev) {
1138                 pr_info("GPU was not found in the current topology. Extending.\n");
1139                 kfd_debug_print_topology();
1140                 dev = kfd_create_topology_device();
1141                 if (!dev) {
1142                         res = -ENOMEM;
1143                         goto err;
1144                 }
1145                 dev->gpu = gpu;
1146
1147                 /*
1148                  * TODO: Make a call to retrieve topology information from the
1149                  * GPU vBIOS
1150                  */
1151
1152                 /*
1153                  * Update the SYSFS tree, since we added another topology device
1154                  */
1155                 if (kfd_topology_update_sysfs() < 0)
1156                         kfd_topology_release_sysfs();
1157
1158         }
1159
1160         dev->gpu_id = gpu_id;
1161         gpu->id = gpu_id;
1162         dev->node_props.vendor_id = gpu->pdev->vendor;
1163         dev->node_props.device_id = gpu->pdev->device;
1164         dev->node_props.location_id = (gpu->pdev->bus->number << 24) +
1165                         (gpu->pdev->devfn & 0xffffff);
1166         /*
1167          * TODO: Retrieve max engine clock values from KGD
1168          */
1169
1170         res = 0;
1171
1172 err:
1173         up_write(&topology_lock);
1174
1175         if (res == 0)
1176                 kfd_notify_gpu_change(gpu_id, 1);
1177
1178         return res;
1179 }
1180
1181 int kfd_topology_remove_device(struct kfd_dev *gpu)
1182 {
1183         struct kfd_topology_device *dev;
1184         uint32_t gpu_id;
1185         int res = -ENODEV;
1186
1187         BUG_ON(!gpu);
1188
1189         down_write(&topology_lock);
1190
1191         list_for_each_entry(dev, &topology_device_list, list)
1192                 if (dev->gpu == gpu) {
1193                         gpu_id = dev->gpu_id;
1194                         kfd_remove_sysfs_node_entry(dev);
1195                         kfd_release_topology_device(dev);
1196                         res = 0;
1197                         if (kfd_topology_update_sysfs() < 0)
1198                                 kfd_topology_release_sysfs();
1199                         break;
1200                 }
1201
1202         up_write(&topology_lock);
1203
1204         if (res == 0)
1205                 kfd_notify_gpu_change(gpu_id, 0);
1206
1207         return res;
1208 }
1209
1210 /*
1211  * When idx is out of bounds, the function will return NULL
1212  */
1213 struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
1214 {
1215
1216         struct kfd_topology_device *top_dev;
1217         struct kfd_dev *device = NULL;
1218         uint8_t device_idx = 0;
1219
1220         down_read(&topology_lock);
1221
1222         list_for_each_entry(top_dev, &topology_device_list, list) {
1223                 if (device_idx == idx) {
1224                         device = top_dev->gpu;
1225                         break;
1226                 }
1227
1228                 device_idx++;
1229         }
1230
1231         up_read(&topology_lock);
1232
1233         return device;
1234
1235 }