drivers/nvdimm/bus.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14 #include <linux/vmalloc.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/module.h>
  17 #include <linux/blkdev.h>
  18 #include <linux/fcntl.h>
  19 #include <linux/async.h>
  20 #include <linux/genhd.h>
  21 #include <linux/ndctl.h>
  22 #include <linux/sched.h>
  23 #include <linux/slab.h>
  24 #include <linux/fs.h>
  25 #include <linux/io.h>
  26 #include <linux/mm.h>
  27 #include <linux/nd.h>
  28 #include "nd-core.h"
  29 #include "nd.h"
  30
  31 int nvdimm_major;
  32 static int nvdimm_bus_major;
  33 static struct class *nd_class;
  34 static DEFINE_IDA(nd_ida);
  35
  36 static int to_nd_device_type(struct device *dev)
  37 {
  38         if (is_nvdimm(dev))
  39                 return ND_DEVICE_DIMM;
  40         else if (is_nd_pmem(dev))
  41                 return ND_DEVICE_REGION_PMEM;
  42         else if (is_nd_blk(dev))
  43                 return ND_DEVICE_REGION_BLK;
  44         else if (is_nd_dax(dev))
  45                 return ND_DEVICE_DAX_PMEM;
  46         else if (is_nd_pmem(dev->parent) || is_nd_blk(dev->parent))
  47                 return nd_region_to_nstype(to_nd_region(dev->parent));
  48
  49         return 0;
  50 }
  51
  52 static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  53 {
  54         /*
  55          * Ensure that region devices always have their numa node set as
  56          * early as possible.
  57          */
  58         if (is_nd_pmem(dev) || is_nd_blk(dev))
  59                 set_dev_node(dev, to_nd_region(dev)->numa_node);
  60         return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
  61                         to_nd_device_type(dev));
  62 }
  63
  64 static struct module *to_bus_provider(struct device *dev)
  65 {
  66         /* pin bus providers while regions are enabled */
  67         if (is_nd_pmem(dev) || is_nd_blk(dev)) {
  68                 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  69
  70                 return nvdimm_bus->nd_desc->module;
  71         }
  72         return NULL;
  73 }
  74
  75 static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
  76 {
  77         nvdimm_bus_lock(&nvdimm_bus->dev);
  78         nvdimm_bus->probe_active++;
  79         nvdimm_bus_unlock(&nvdimm_bus->dev);
  80 }
  81
  82 static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
  83 {
  84         nvdimm_bus_lock(&nvdimm_bus->dev);
  85         if (--nvdimm_bus->probe_active == 0)
  86                 wake_up(&nvdimm_bus->probe_wait);
  87         nvdimm_bus_unlock(&nvdimm_bus->dev);
  88 }
  89
  90 static int nvdimm_bus_probe(struct device *dev)
  91 {
  92         struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
  93         struct module *provider = to_bus_provider(dev);
  94         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  95         int rc;
  96
  97         if (!try_module_get(provider))
  98                 return -ENXIO;
  99
 100         nvdimm_bus_probe_start(nvdimm_bus);
 101         rc = nd_drv->probe(dev);
 102         if (rc == 0)
 103                 nd_region_probe_success(nvdimm_bus, dev);
 104         else
 105                 nd_region_disable(nvdimm_bus, dev);
 106         nvdimm_bus_probe_end(nvdimm_bus);
 107
 108         dev_dbg(&nvdimm_bus->dev, "%s.probe(%s) = %d\n", dev->driver->name,
 109                         dev_name(dev), rc);
 110
 111         if (rc != 0)
 112                 module_put(provider);
 113         return rc;
 114 }
 115
 116 static int nvdimm_bus_remove(struct device *dev)
 117 {
 118         struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
 119         struct module *provider = to_bus_provider(dev);
 120         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 121         int rc = 0;
 122
 123         if (nd_drv->remove)
 124                 rc = nd_drv->remove(dev);
 125         nd_region_disable(nvdimm_bus, dev);
 126
 127         dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name,
 128                         dev_name(dev), rc);
 129         module_put(provider);
 130         return rc;
 131 }
 132
 133 static void nvdimm_bus_shutdown(struct device *dev)
 134 {
 135         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 136         struct nd_device_driver *nd_drv = NULL;
 137
 138         if (dev->driver)
 139                 nd_drv = to_nd_device_driver(dev->driver);
 140
 141         if (nd_drv && nd_drv->shutdown) {
 142                 nd_drv->shutdown(dev);
 143                 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
 144                                 dev->driver->name, dev_name(dev));
 145         }
 146 }
 147
 148 void nd_device_notify(struct device *dev, enum nvdimm_event event)
 149 {
 150         device_lock(dev);
 151         if (dev->driver) {
 152                 struct nd_device_driver *nd_drv;
 153
 154                 nd_drv = to_nd_device_driver(dev->driver);
 155                 if (nd_drv->notify)
 156                         nd_drv->notify(dev, event);
 157         }
 158         device_unlock(dev);
 159 }
 160 EXPORT_SYMBOL(nd_device_notify);
 161
 162 void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
 163 {
 164         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
 165
 166         if (!nvdimm_bus)
 167                 return;
 168
 169         /* caller is responsible for holding a reference on the device */
 170         nd_device_notify(&nd_region->dev, event);
 171 }
 172 EXPORT_SYMBOL_GPL(nvdimm_region_notify);
 173
 174 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 175                 unsigned int len)
 176 {
 177         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 178         struct nvdimm_bus_descriptor *nd_desc;
 179         struct nd_cmd_clear_error clear_err;
 180         struct nd_cmd_ars_cap ars_cap;
 181         u32 clear_err_unit, mask;
 182         int cmd_rc, rc;
 183
 184         if (!nvdimm_bus)
 185                 return -ENXIO;
 186
 187         nd_desc = nvdimm_bus->nd_desc;
 188         /*
 189          * if ndctl does not exist, it's PMEM_LEGACY and
 190          * we want to just pretend everything is handled.
 191          */
 192         if (!nd_desc->ndctl)
 193                 return len;
 194
 195         memset(&ars_cap, 0, sizeof(ars_cap));
 196         ars_cap.address = phys;
 197         ars_cap.length = len;
 198         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
 199                         sizeof(ars_cap), &cmd_rc);
 200         if (rc < 0)
 201                 return rc;
 202         if (cmd_rc < 0)
 203                 return cmd_rc;
 204         clear_err_unit = ars_cap.clear_err_unit;
 205         if (!clear_err_unit || !is_power_of_2(clear_err_unit))
 206                 return -ENXIO;
 207
 208         mask = clear_err_unit - 1;
 209         if ((phys | len) & mask)
 210                 return -ENXIO;
 211         memset(&clear_err, 0, sizeof(clear_err));
 212         clear_err.address = phys;
 213         clear_err.length = len;
 214         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
 215                         sizeof(clear_err), &cmd_rc);
 216         if (rc < 0)
 217                 return rc;
 218         if (cmd_rc < 0)
 219                 return cmd_rc;
 220         return clear_err.cleared;
 221 }
 222 EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
 223
 224 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
 225
 226 static struct bus_type nvdimm_bus_type = {
 227         .name = "nd",
 228         .uevent = nvdimm_bus_uevent,
 229         .match = nvdimm_bus_match,
 230         .probe = nvdimm_bus_probe,
 231         .remove = nvdimm_bus_remove,
 232         .shutdown = nvdimm_bus_shutdown,
 233 };
 234
 235 static void nvdimm_bus_release(struct device *dev)
 236 {
 237         struct nvdimm_bus *nvdimm_bus;
 238
 239         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
 240         ida_simple_remove(&nd_ida, nvdimm_bus->id);
 241         kfree(nvdimm_bus);
 242 }
 243
 244 static bool is_nvdimm_bus(struct device *dev)
 245 {
 246         return dev->release == nvdimm_bus_release;
 247 }
 248
 249 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
 250 {
 251         struct device *dev;
 252
 253         for (dev = nd_dev; dev; dev = dev->parent)
 254                 if (is_nvdimm_bus(dev))
 255                         break;
 256         dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
 257         if (dev)
 258                 return to_nvdimm_bus(dev);
 259         return NULL;
 260 }
 261
 262 struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
 263 {
 264         struct nvdimm_bus *nvdimm_bus;
 265
 266         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
 267         WARN_ON(!is_nvdimm_bus(dev));
 268         return nvdimm_bus;
 269 }
 270 EXPORT_SYMBOL_GPL(to_nvdimm_bus);
 271
 272 struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
 273                 struct nvdimm_bus_descriptor *nd_desc)
 274 {
 275         struct nvdimm_bus *nvdimm_bus;
 276         int rc;
 277
 278         nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
 279         if (!nvdimm_bus)
 280                 return NULL;
 281         INIT_LIST_HEAD(&nvdimm_bus->list);
 282         INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
 283         INIT_LIST_HEAD(&nvdimm_bus->poison_list);
 284         init_waitqueue_head(&nvdimm_bus->probe_wait);
 285         nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
 286         mutex_init(&nvdimm_bus->reconfig_mutex);
 287         if (nvdimm_bus->id < 0) {
 288                 kfree(nvdimm_bus);
 289                 return NULL;
 290         }
 291         nvdimm_bus->nd_desc = nd_desc;
 292         nvdimm_bus->dev.parent = parent;
 293         nvdimm_bus->dev.release = nvdimm_bus_release;
 294         nvdimm_bus->dev.groups = nd_desc->attr_groups;
 295         nvdimm_bus->dev.bus = &nvdimm_bus_type;
 296         dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
 297         rc = device_register(&nvdimm_bus->dev);
 298         if (rc) {
 299                 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
 300                 goto err;
 301         }
 302
 303         return nvdimm_bus;
 304  err:
 305         put_device(&nvdimm_bus->dev);
 306         return NULL;
 307 }
 308 EXPORT_SYMBOL_GPL(nvdimm_bus_register);
 309
 310 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
 311 {
 312         if (!nvdimm_bus)
 313                 return;
 314         device_unregister(&nvdimm_bus->dev);
 315 }
 316 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
 317
 318 static int child_unregister(struct device *dev, void *data)
 319 {
 320         /*
 321          * the singular ndctl class device per bus needs to be
 322          * "device_destroy"ed, so skip it here
 323          *
 324          * i.e. remove classless children
 325          */
 326         if (dev->class)
 327                 /* pass */;
 328         else
 329                 nd_device_unregister(dev, ND_SYNC);
 330         return 0;
 331 }
 332
 333 static void free_poison_list(struct list_head *poison_list)
 334 {
 335         struct nd_poison *pl, *next;
 336
 337         list_for_each_entry_safe(pl, next, poison_list, list) {
 338                 list_del(&pl->list);
 339                 kfree(pl);
 340         }
 341         list_del_init(poison_list);
 342 }
 343
 344 static int nd_bus_remove(struct device *dev)
 345 {
 346         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 347
 348         mutex_lock(&nvdimm_bus_list_mutex);
 349         list_del_init(&nvdimm_bus->list);
 350         mutex_unlock(&nvdimm_bus_list_mutex);
 351
 352         nd_synchronize();
 353         device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
 354
 355         nvdimm_bus_lock(&nvdimm_bus->dev);
 356         free_poison_list(&nvdimm_bus->poison_list);
 357         nvdimm_bus_unlock(&nvdimm_bus->dev);
 358
 359         nvdimm_bus_destroy_ndctl(nvdimm_bus);
 360
 361         return 0;
 362 }
 363
 364 static int nd_bus_probe(struct device *dev)
 365 {
 366         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 367         int rc;
 368
 369         rc = nvdimm_bus_create_ndctl(nvdimm_bus);
 370         if (rc)
 371                 return rc;
 372
 373         mutex_lock(&nvdimm_bus_list_mutex);
 374         list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
 375         mutex_unlock(&nvdimm_bus_list_mutex);
 376
 377         /* enable bus provider attributes to look up their local context */
 378         dev_set_drvdata(dev, nvdimm_bus->nd_desc);
 379
 380         return 0;
 381 }
 382
 383 static struct nd_device_driver nd_bus_driver = {
 384         .probe = nd_bus_probe,
 385         .remove = nd_bus_remove,
 386         .drv = {
 387                 .name = "nd_bus",
 388                 .suppress_bind_attrs = true,
 389                 .bus = &nvdimm_bus_type,
 390                 .owner = THIS_MODULE,
 391                 .mod_name = KBUILD_MODNAME,
 392         },
 393 };
 394
 395 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
 396 {
 397         struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
 398
 399         if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
 400                 return true;
 401
 402         return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
 403 }
 404
 405 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
 406
 407 void nd_synchronize(void)
 408 {
 409         async_synchronize_full_domain(&nd_async_domain);
 410 }
 411 EXPORT_SYMBOL_GPL(nd_synchronize);
 412
 413 static void nd_async_device_register(void *d, async_cookie_t cookie)
 414 {
 415         struct device *dev = d;
 416
 417         if (device_add(dev) != 0) {
 418                 dev_err(dev, "%s: failed\n", __func__);
 419                 put_device(dev);
 420         }
 421         put_device(dev);
 422 }
 423
 424 static void nd_async_device_unregister(void *d, async_cookie_t cookie)
 425 {
 426         struct device *dev = d;
 427
 428         /* flush bus operations before delete */
 429         nvdimm_bus_lock(dev);
 430         nvdimm_bus_unlock(dev);
 431
 432         device_unregister(dev);
 433         put_device(dev);
 434 }
 435
 436 void __nd_device_register(struct device *dev)
 437 {
 438         if (!dev)
 439                 return;
 440         dev->bus = &nvdimm_bus_type;
 441         get_device(dev);
 442         async_schedule_domain(nd_async_device_register, dev,
 443                         &nd_async_domain);
 444 }
 445
 446 void nd_device_register(struct device *dev)
 447 {
 448         device_initialize(dev);
 449         __nd_device_register(dev);
 450 }
 451 EXPORT_SYMBOL(nd_device_register);
 452
 453 void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
 454 {
 455         switch (mode) {
 456         case ND_ASYNC:
 457                 get_device(dev);
 458                 async_schedule_domain(nd_async_device_unregister, dev,
 459                                 &nd_async_domain);
 460                 break;
 461         case ND_SYNC:
 462                 nd_synchronize();
 463                 device_unregister(dev);
 464                 break;
 465         }
 466 }
 467 EXPORT_SYMBOL(nd_device_unregister);
 468
 469 /**
 470  * __nd_driver_register() - register a region or a namespace driver
 471  * @nd_drv: driver to register
 472  * @owner: automatically set by nd_driver_register() macro
 473  * @mod_name: automatically set by nd_driver_register() macro
 474  */
 475 int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
 476                 const char *mod_name)
 477 {
 478         struct device_driver *drv = &nd_drv->drv;
 479
 480         if (!nd_drv->type) {
 481                 pr_debug("driver type bitmask not set (%pf)\n",
 482                                 __builtin_return_address(0));
 483                 return -EINVAL;
 484         }
 485
 486         if (!nd_drv->probe) {
 487                 pr_debug("%s ->probe() must be specified\n", mod_name);
 488                 return -EINVAL;
 489         }
 490
 491         drv->bus = &nvdimm_bus_type;
 492         drv->owner = owner;
 493         drv->mod_name = mod_name;
 494
 495         return driver_register(drv);
 496 }
 497 EXPORT_SYMBOL(__nd_driver_register);
 498
 499 int nvdimm_revalidate_disk(struct gendisk *disk)
 500 {
 501         struct device *dev = disk_to_dev(disk)->parent;
 502         struct nd_region *nd_region = to_nd_region(dev->parent);
 503         const char *pol = nd_region->ro ? "only" : "write";
 504
 505         if (nd_region->ro == get_disk_ro(disk))
 506                 return 0;
 507
 508         dev_info(dev, "%s read-%s, marking %s read-%s\n",
 509                         dev_name(&nd_region->dev), pol, disk->disk_name, pol);
 510         set_disk_ro(disk, nd_region->ro);
 511
 512         return 0;
 513
 514 }
 515 EXPORT_SYMBOL(nvdimm_revalidate_disk);
 516
 517 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
 518                 char *buf)
 519 {
 520         return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
 521                         to_nd_device_type(dev));
 522 }
 523 static DEVICE_ATTR_RO(modalias);
 524
 525 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
 526                 char *buf)
 527 {
 528         return sprintf(buf, "%s\n", dev->type->name);
 529 }
 530 static DEVICE_ATTR_RO(devtype);
 531
 532 static struct attribute *nd_device_attributes[] = {
 533         &dev_attr_modalias.attr,
 534         &dev_attr_devtype.attr,
 535         NULL,
 536 };
 537
 538 /**
 539  * nd_device_attribute_group - generic attributes for all devices on an nd bus
 540  */
 541 struct attribute_group nd_device_attribute_group = {
 542         .attrs = nd_device_attributes,
 543 };
 544 EXPORT_SYMBOL_GPL(nd_device_attribute_group);
 545
 546 static ssize_t numa_node_show(struct device *dev,
 547                 struct device_attribute *attr, char *buf)
 548 {
 549         return sprintf(buf, "%d\n", dev_to_node(dev));
 550 }
 551 static DEVICE_ATTR_RO(numa_node);
 552
 553 static struct attribute *nd_numa_attributes[] = {
 554         &dev_attr_numa_node.attr,
 555         NULL,
 556 };
 557
 558 static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
 559                 int n)
 560 {
 561         if (!IS_ENABLED(CONFIG_NUMA))
 562                 return 0;
 563
 564         return a->mode;
 565 }
 566
 567 /**
 568  * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
 569  */
 570 struct attribute_group nd_numa_attribute_group = {
 571         .attrs = nd_numa_attributes,
 572         .is_visible = nd_numa_attr_visible,
 573 };
 574 EXPORT_SYMBOL_GPL(nd_numa_attribute_group);
 575
 576 int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
 577 {
 578         dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
 579         struct device *dev;
 580
 581         dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus,
 582                         "ndctl%d", nvdimm_bus->id);
 583
 584         if (IS_ERR(dev))
 585                 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n",
 586                                 nvdimm_bus->id, PTR_ERR(dev));
 587         return PTR_ERR_OR_ZERO(dev);
 588 }
 589
 590 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
 591 {
 592         device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
 593 }
 594
 595 static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
 596         [ND_CMD_IMPLEMENTED] = { },
 597         [ND_CMD_SMART] = {
 598                 .out_num = 2,
 599                 .out_sizes = { 4, 128, },
 600         },
 601         [ND_CMD_SMART_THRESHOLD] = {
 602                 .out_num = 2,
 603                 .out_sizes = { 4, 8, },
 604         },
 605         [ND_CMD_DIMM_FLAGS] = {
 606                 .out_num = 2,
 607                 .out_sizes = { 4, 4 },
 608         },
 609         [ND_CMD_GET_CONFIG_SIZE] = {
 610                 .out_num = 3,
 611                 .out_sizes = { 4, 4, 4, },
 612         },
 613         [ND_CMD_GET_CONFIG_DATA] = {
 614                 .in_num = 2,
 615                 .in_sizes = { 4, 4, },
 616                 .out_num = 2,
 617                 .out_sizes = { 4, UINT_MAX, },
 618         },
 619         [ND_CMD_SET_CONFIG_DATA] = {
 620                 .in_num = 3,
 621                 .in_sizes = { 4, 4, UINT_MAX, },
 622                 .out_num = 1,
 623                 .out_sizes = { 4, },
 624         },
 625         [ND_CMD_VENDOR] = {
 626                 .in_num = 3,
 627                 .in_sizes = { 4, 4, UINT_MAX, },
 628                 .out_num = 3,
 629                 .out_sizes = { 4, 4, UINT_MAX, },
 630         },
 631         [ND_CMD_CALL] = {
 632                 .in_num = 2,
 633                 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
 634                 .out_num = 1,
 635                 .out_sizes = { UINT_MAX, },
 636         },
 637 };
 638
 639 const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
 640 {
 641         if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
 642                 return &__nd_cmd_dimm_descs[cmd];
 643         return NULL;
 644 }
 645 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
 646
 647 static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
 648         [ND_CMD_IMPLEMENTED] = { },
 649         [ND_CMD_ARS_CAP] = {
 650                 .in_num = 2,
 651                 .in_sizes = { 8, 8, },
 652                 .out_num = 4,
 653                 .out_sizes = { 4, 4, 4, 4, },
 654         },
 655         [ND_CMD_ARS_START] = {
 656                 .in_num = 5,
 657                 .in_sizes = { 8, 8, 2, 1, 5, },
 658                 .out_num = 2,
 659                 .out_sizes = { 4, 4, },
 660         },
 661         [ND_CMD_ARS_STATUS] = {
 662                 .out_num = 3,
 663                 .out_sizes = { 4, 4, UINT_MAX, },
 664         },
 665         [ND_CMD_CLEAR_ERROR] = {
 666                 .in_num = 2,
 667                 .in_sizes = { 8, 8, },
 668                 .out_num = 3,
 669                 .out_sizes = { 4, 4, 8, },
 670         },
 671         [ND_CMD_CALL] = {
 672                 .in_num = 2,
 673                 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
 674                 .out_num = 1,
 675                 .out_sizes = { UINT_MAX, },
 676         },
 677 };
 678
 679 const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
 680 {
 681         if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
 682                 return &__nd_cmd_bus_descs[cmd];
 683         return NULL;
 684 }
 685 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
 686
 687 u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
 688                 const struct nd_cmd_desc *desc, int idx, void *buf)
 689 {
 690         if (idx >= desc->in_num)
 691                 return UINT_MAX;
 692
 693         if (desc->in_sizes[idx] < UINT_MAX)
 694                 return desc->in_sizes[idx];
 695
 696         if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
 697                 struct nd_cmd_set_config_hdr *hdr = buf;
 698
 699                 return hdr->in_length;
 700         } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
 701                 struct nd_cmd_vendor_hdr *hdr = buf;
 702
 703                 return hdr->in_length;
 704         } else if (cmd == ND_CMD_CALL) {
 705                 struct nd_cmd_pkg *pkg = buf;
 706
 707                 return pkg->nd_size_in;
 708         }
 709
 710         return UINT_MAX;
 711 }
 712 EXPORT_SYMBOL_GPL(nd_cmd_in_size);
 713
 714 u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
 715                 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
 716                 const u32 *out_field)
 717 {
 718         if (idx >= desc->out_num)
 719                 return UINT_MAX;
 720
 721         if (desc->out_sizes[idx] < UINT_MAX)
 722                 return desc->out_sizes[idx];
 723
 724         if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
 725                 return in_field[1];
 726         else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
 727                 return out_field[1];
 728         else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2)
 729                 return out_field[1] - 8;
 730         else if (cmd == ND_CMD_CALL) {
 731                 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
 732
 733                 return pkg->nd_size_out;
 734         }
 735
 736
 737         return UINT_MAX;
 738 }
 739 EXPORT_SYMBOL_GPL(nd_cmd_out_size);
 740
 741 void wait_nvdimm_bus_probe_idle(struct device *dev)
 742 {
 743         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 744
 745         do {
 746                 if (nvdimm_bus->probe_active == 0)
 747                         break;
 748                 nvdimm_bus_unlock(&nvdimm_bus->dev);
 749                 wait_event(nvdimm_bus->probe_wait,
 750                                 nvdimm_bus->probe_active == 0);
 751                 nvdimm_bus_lock(&nvdimm_bus->dev);
 752         } while (true);
 753 }
 754
 755 static int pmem_active(struct device *dev, void *data)
 756 {
 757         if (is_nd_pmem(dev) && dev->driver)
 758                 return -EBUSY;
 759         return 0;
 760 }
 761
 762 /* set_config requires an idle interleave set */
 763 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
 764                 struct nvdimm *nvdimm, unsigned int cmd)
 765 {
 766         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 767
 768         /* ask the bus provider if it would like to block this request */
 769         if (nd_desc->clear_to_send) {
 770                 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd);
 771
 772                 if (rc)
 773                         return rc;
 774         }
 775
 776         /* require clear error to go through the pmem driver */
 777         if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
 778                 return device_for_each_child(&nvdimm_bus->dev, NULL,
 779                                 pmem_active);
 780
 781         if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
 782                 return 0;
 783
 784         /* prevent label manipulation while the kernel owns label updates */
 785         wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
 786         if (atomic_read(&nvdimm->busy))
 787                 return -EBUSY;
 788         return 0;
 789 }
 790
 791 static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
 792                 int read_only, unsigned int ioctl_cmd, unsigned long arg)
 793 {
 794         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 795         size_t buf_len = 0, in_len = 0, out_len = 0;
 796         static char out_env[ND_CMD_MAX_ENVELOPE];
 797         static char in_env[ND_CMD_MAX_ENVELOPE];
 798         const struct nd_cmd_desc *desc = NULL;
 799         unsigned int cmd = _IOC_NR(ioctl_cmd);
 800         void __user *p = (void __user *) arg;
 801         struct device *dev = &nvdimm_bus->dev;
 802         struct nd_cmd_pkg pkg;
 803         const char *cmd_name, *dimm_name;
 804         unsigned long cmd_mask;
 805         void *buf;
 806         int rc, i;
 807
 808         if (nvdimm) {
 809                 desc = nd_cmd_dimm_desc(cmd);
 810                 cmd_name = nvdimm_cmd_name(cmd);
 811                 cmd_mask = nvdimm->cmd_mask;
 812                 dimm_name = dev_name(&nvdimm->dev);
 813         } else {
 814                 desc = nd_cmd_bus_desc(cmd);
 815                 cmd_name = nvdimm_bus_cmd_name(cmd);
 816                 cmd_mask = nd_desc->cmd_mask;
 817                 dimm_name = "bus";
 818         }
 819
 820         if (cmd == ND_CMD_CALL) {
 821                 if (copy_from_user(&pkg, p, sizeof(pkg)))
 822                         return -EFAULT;
 823         }
 824
 825         if (!desc || (desc->out_num + desc->in_num == 0) ||
 826                         !test_bit(cmd, &cmd_mask))
 827                 return -ENOTTY;
 828
 829         /* fail write commands (when read-only) */
 830         if (read_only)
 831                 switch (cmd) {
 832                 case ND_CMD_VENDOR:
 833                 case ND_CMD_SET_CONFIG_DATA:
 834                 case ND_CMD_ARS_START:
 835                 case ND_CMD_CLEAR_ERROR:
 836                 case ND_CMD_CALL:
 837                         dev_dbg(&nvdimm_bus->dev, "'%s' command while read-only.\n",
 838                                         nvdimm ? nvdimm_cmd_name(cmd)
 839                                         : nvdimm_bus_cmd_name(cmd));
 840                         return -EPERM;
 841                 default:
 842                         break;
 843                 }
 844
 845         /* process an input envelope */
 846         for (i = 0; i < desc->in_num; i++) {
 847                 u32 in_size, copy;
 848
 849                 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
 850                 if (in_size == UINT_MAX) {
 851                         dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
 852                                         __func__, dimm_name, cmd_name, i);
 853                         return -ENXIO;
 854                 }
 855                 if (in_len < sizeof(in_env))
 856                         copy = min_t(u32, sizeof(in_env) - in_len, in_size);
 857                 else
 858                         copy = 0;
 859                 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy))
 860                         return -EFAULT;
 861                 in_len += in_size;
 862         }
 863
 864         if (cmd == ND_CMD_CALL) {
 865                 dev_dbg(dev, "%s:%s, idx: %llu, in: %zu, out: %zu, len %zu\n",
 866                                 __func__, dimm_name, pkg.nd_command,
 867                                 in_len, out_len, buf_len);
 868
 869                 for (i = 0; i < ARRAY_SIZE(pkg.nd_reserved2); i++)
 870                         if (pkg.nd_reserved2[i])
 871                                 return -EINVAL;
 872         }
 873
 874         /* process an output envelope */
 875         for (i = 0; i < desc->out_num; i++) {
 876                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
 877                                 (u32 *) in_env, (u32 *) out_env);
 878                 u32 copy;
 879
 880                 if (out_size == UINT_MAX) {
 881                         dev_dbg(dev, "%s:%s unknown output size cmd: %s field: %d\n",
 882                                         __func__, dimm_name, cmd_name, i);
 883                         return -EFAULT;
 884                 }
 885                 if (out_len < sizeof(out_env))
 886                         copy = min_t(u32, sizeof(out_env) - out_len, out_size);
 887                 else
 888                         copy = 0;
 889                 if (copy && copy_from_user(&out_env[out_len],
 890                                         p + in_len + out_len, copy))
 891                         return -EFAULT;
 892                 out_len += out_size;
 893         }
 894
 895         buf_len = out_len + in_len;
 896         if (buf_len > ND_IOCTL_MAX_BUFLEN) {
 897                 dev_dbg(dev, "%s:%s cmd: %s buf_len: %zu > %d\n", __func__,
 898                                 dimm_name, cmd_name, buf_len,
 899                                 ND_IOCTL_MAX_BUFLEN);
 900                 return -EINVAL;
 901         }
 902
 903         buf = vmalloc(buf_len);
 904         if (!buf)
 905                 return -ENOMEM;
 906
 907         if (copy_from_user(buf, p, buf_len)) {
 908                 rc = -EFAULT;
 909                 goto out;
 910         }
 911
 912         nvdimm_bus_lock(&nvdimm_bus->dev);
 913         rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, cmd);
 914         if (rc)
 915                 goto out_unlock;
 916
 917         rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, NULL);
 918         if (rc < 0)
 919                 goto out_unlock;
 920         if (copy_to_user(p, buf, buf_len))
 921                 rc = -EFAULT;
 922  out_unlock:
 923         nvdimm_bus_unlock(&nvdimm_bus->dev);
 924  out:
 925         vfree(buf);
 926         return rc;
 927 }
 928
 929 static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 930 {
 931         long id = (long) file->private_data;
 932         int rc = -ENXIO, ro;
 933         struct nvdimm_bus *nvdimm_bus;
 934
 935         ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
 936         mutex_lock(&nvdimm_bus_list_mutex);
 937         list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
 938                 if (nvdimm_bus->id == id) {
 939                         rc = __nd_ioctl(nvdimm_bus, NULL, ro, cmd, arg);
 940                         break;
 941                 }
 942         }
 943         mutex_unlock(&nvdimm_bus_list_mutex);
 944
 945         return rc;
 946 }
 947
 948 static int match_dimm(struct device *dev, void *data)
 949 {
 950         long id = (long) data;
 951
 952         if (is_nvdimm(dev)) {
 953                 struct nvdimm *nvdimm = to_nvdimm(dev);
 954
 955                 return nvdimm->id == id;
 956         }
 957
 958         return 0;
 959 }
 960
 961 static long nvdimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 962 {
 963         int rc = -ENXIO, ro;
 964         struct nvdimm_bus *nvdimm_bus;
 965
 966         ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
 967         mutex_lock(&nvdimm_bus_list_mutex);
 968         list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
 969                 struct device *dev = device_find_child(&nvdimm_bus->dev,
 970                                 file->private_data, match_dimm);
 971                 struct nvdimm *nvdimm;
 972
 973                 if (!dev)
 974                         continue;
 975
 976                 nvdimm = to_nvdimm(dev);
 977                 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
 978                 put_device(dev);
 979                 break;
 980         }
 981         mutex_unlock(&nvdimm_bus_list_mutex);
 982
 983         return rc;
 984 }
 985
 986 static int nd_open(struct inode *inode, struct file *file)
 987 {
 988         long minor = iminor(inode);
 989
 990         file->private_data = (void *) minor;
 991         return 0;
 992 }
 993
 994 static const struct file_operations nvdimm_bus_fops = {
 995         .owner = THIS_MODULE,
 996         .open = nd_open,
 997         .unlocked_ioctl = nd_ioctl,
 998         .compat_ioctl = nd_ioctl,
 999         .llseek = noop_llseek,
1000 };
1001
1002 static const struct file_operations nvdimm_fops = {
1003         .owner = THIS_MODULE,
1004         .open = nd_open,
1005         .unlocked_ioctl = nvdimm_ioctl,
1006         .compat_ioctl = nvdimm_ioctl,
1007         .llseek = noop_llseek,
1008 };
1009
1010 int __init nvdimm_bus_init(void)
1011 {
1012         int rc;
1013
1014         BUILD_BUG_ON(sizeof(struct nd_smart_payload) != 128);
1015         BUILD_BUG_ON(sizeof(struct nd_smart_threshold_payload) != 8);
1016
1017         rc = bus_register(&nvdimm_bus_type);
1018         if (rc)
1019                 return rc;
1020
1021         rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1022         if (rc < 0)
1023                 goto err_bus_chrdev;
1024         nvdimm_bus_major = rc;
1025
1026         rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1027         if (rc < 0)
1028                 goto err_dimm_chrdev;
1029         nvdimm_major = rc;
1030
1031         nd_class = class_create(THIS_MODULE, "nd");
1032         if (IS_ERR(nd_class)) {
1033                 rc = PTR_ERR(nd_class);
1034                 goto err_class;
1035         }
1036
1037         rc = driver_register(&nd_bus_driver.drv);
1038         if (rc)
1039                 goto err_nd_bus;
1040
1041         return 0;
1042
1043  err_nd_bus:
1044         class_destroy(nd_class);
1045  err_class:
1046         unregister_chrdev(nvdimm_major, "dimmctl");
1047  err_dimm_chrdev:
1048         unregister_chrdev(nvdimm_bus_major, "ndctl");
1049  err_bus_chrdev:
1050         bus_unregister(&nvdimm_bus_type);
1051
1052         return rc;
1053 }
1054
1055 void nvdimm_bus_exit(void)
1056 {
1057         driver_unregister(&nd_bus_driver.drv);
1058         class_destroy(nd_class);
1059         unregister_chrdev(nvdimm_bus_major, "ndctl");
1060         unregister_chrdev(nvdimm_major, "dimmctl");
1061         bus_unregister(&nvdimm_bus_type);
1062         ida_destroy(&nd_ida);
1063 }