drivers/misc/vmw_vmci/vmci_doorbell.c

   1 /*
   2  * VMware VMCI Driver
   3  *
   4  * Copyright (C) 2012 VMware, Inc. All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation version 2 and no later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * for more details.
  14  */
  15
  16 #include <linux/vmw_vmci_defs.h>
  17 #include <linux/vmw_vmci_api.h>
  18 #include <linux/completion.h>
  19 #include <linux/hash.h>
  20 #include <linux/kernel.h>
  21 #include <linux/list.h>
  22 #include <linux/module.h>
  23 #include <linux/sched.h>
  24 #include <linux/slab.h>
  25
  26 #include "vmci_datagram.h"
  27 #include "vmci_doorbell.h"
  28 #include "vmci_resource.h"
  29 #include "vmci_driver.h"
  30 #include "vmci_route.h"
  31
  32
  33 #define VMCI_DOORBELL_INDEX_BITS        6
  34 #define VMCI_DOORBELL_INDEX_TABLE_SIZE  (1 << VMCI_DOORBELL_INDEX_BITS)
  35 #define VMCI_DOORBELL_HASH(_idx)        hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
  36
  37 /*
  38  * DoorbellEntry describes the a doorbell notification handle allocated by the
  39  * host.
  40  */
  41 struct dbell_entry {
  42         struct vmci_resource resource;
  43         struct hlist_node node;
  44         struct work_struct work;
  45         vmci_callback notify_cb;
  46         void *client_data;
  47         u32 idx;
  48         u32 priv_flags;
  49         bool run_delayed;
  50         atomic_t active;        /* Only used by guest personality */
  51 };
  52
  53 /* The VMCI index table keeps track of currently registered doorbells. */
  54 struct dbell_index_table {
  55         spinlock_t lock;        /* Index table lock */
  56         struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
  57 };
  58
  59 static struct dbell_index_table vmci_doorbell_it = {
  60         .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
  61 };
  62
  63 /*
  64  * The max_notify_idx is one larger than the currently known bitmap index in
  65  * use, and is used to determine how much of the bitmap needs to be scanned.
  66  */
  67 static u32 max_notify_idx;
  68
  69 /*
  70  * The notify_idx_count is used for determining whether there are free entries
  71  * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
  72  */
  73 static u32 notify_idx_count;
  74
  75 /*
  76  * The last_notify_idx_reserved is used to track the last index handed out - in
  77  * the case where multiple handles share a notification index, we hand out
  78  * indexes round robin based on last_notify_idx_reserved.
  79  */
  80 static u32 last_notify_idx_reserved;
  81
  82 /* This is a one entry cache used to by the index allocation. */
  83 static u32 last_notify_idx_released = PAGE_SIZE;
  84
  85
  86 /*
  87  * Utility function that retrieves the privilege flags associated
  88  * with a given doorbell handle. For guest endpoints, the
  89  * privileges are determined by the context ID, but for host
  90  * endpoints privileges are associated with the complete
  91  * handle. Hypervisor endpoints are not yet supported.
  92  */
  93 int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
  94 {
  95         if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
  96                 return VMCI_ERROR_INVALID_ARGS;
  97
  98         if (handle.context == VMCI_HOST_CONTEXT_ID) {
  99                 struct dbell_entry *entry;
 100                 struct vmci_resource *resource;
 101
 102                 resource = vmci_resource_by_handle(handle,
 103                                                    VMCI_RESOURCE_TYPE_DOORBELL);
 104                 if (!resource)
 105                         return VMCI_ERROR_NOT_FOUND;
 106
 107                 entry = container_of(resource, struct dbell_entry, resource);
 108                 *priv_flags = entry->priv_flags;
 109                 vmci_resource_put(resource);
 110         } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
 111                 /*
 112                  * Hypervisor endpoints for notifications are not
 113                  * supported (yet).
 114                  */
 115                 return VMCI_ERROR_INVALID_ARGS;
 116         } else {
 117                 *priv_flags = vmci_context_get_priv_flags(handle.context);
 118         }
 119
 120         return VMCI_SUCCESS;
 121 }
 122
 123 /*
 124  * Find doorbell entry by bitmap index.
 125  */
 126 static struct dbell_entry *dbell_index_table_find(u32 idx)
 127 {
 128         u32 bucket = VMCI_DOORBELL_HASH(idx);
 129         struct dbell_entry *dbell;
 130
 131         hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
 132                              node) {
 133                 if (idx == dbell->idx)
 134                         return dbell;
 135         }
 136
 137         return NULL;
 138 }
 139
 140 /*
 141  * Add the given entry to the index table.  This willi take a reference to the
 142  * entry's resource so that the entry is not deleted before it is removed from
 143  * the * table.
 144  */
 145 static void dbell_index_table_add(struct dbell_entry *entry)
 146 {
 147         u32 bucket;
 148         u32 new_notify_idx;
 149
 150         vmci_resource_get(&entry->resource);
 151
 152         spin_lock_bh(&vmci_doorbell_it.lock);
 153
 154         /*
 155          * Below we try to allocate an index in the notification
 156          * bitmap with "not too much" sharing between resources. If we
 157          * use less that the full bitmap, we either add to the end if
 158          * there are no unused flags within the currently used area,
 159          * or we search for unused ones. If we use the full bitmap, we
 160          * allocate the index round robin.
 161          */
 162         if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
 163                 if (last_notify_idx_released < max_notify_idx &&
 164                     !dbell_index_table_find(last_notify_idx_released)) {
 165                         new_notify_idx = last_notify_idx_released;
 166                         last_notify_idx_released = PAGE_SIZE;
 167                 } else {
 168                         bool reused = false;
 169                         new_notify_idx = last_notify_idx_reserved;
 170                         if (notify_idx_count + 1 < max_notify_idx) {
 171                                 do {
 172                                         if (!dbell_index_table_find
 173                                             (new_notify_idx)) {
 174                                                 reused = true;
 175                                                 break;
 176                                         }
 177                                         new_notify_idx = (new_notify_idx + 1) %
 178                                             max_notify_idx;
 179                                 } while (new_notify_idx !=
 180                                          last_notify_idx_released);
 181                         }
 182                         if (!reused) {
 183                                 new_notify_idx = max_notify_idx;
 184                                 max_notify_idx++;
 185                         }
 186                 }
 187         } else {
 188                 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
 189         }
 190
 191         last_notify_idx_reserved = new_notify_idx;
 192         notify_idx_count++;
 193
 194         entry->idx = new_notify_idx;
 195         bucket = VMCI_DOORBELL_HASH(entry->idx);
 196         hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
 197
 198         spin_unlock_bh(&vmci_doorbell_it.lock);
 199 }
 200
 201 /*
 202  * Remove the given entry from the index table.  This will release() the
 203  * entry's resource.
 204  */
 205 static void dbell_index_table_remove(struct dbell_entry *entry)
 206 {
 207         spin_lock_bh(&vmci_doorbell_it.lock);
 208
 209         hlist_del_init(&entry->node);
 210
 211         notify_idx_count--;
 212         if (entry->idx == max_notify_idx - 1) {
 213                 /*
 214                  * If we delete an entry with the maximum known
 215                  * notification index, we take the opportunity to
 216                  * prune the current max. As there might be other
 217                  * unused indices immediately below, we lower the
 218                  * maximum until we hit an index in use.
 219                  */
 220                 while (max_notify_idx > 0 &&
 221                        !dbell_index_table_find(max_notify_idx - 1))
 222                         max_notify_idx--;
 223         }
 224
 225         last_notify_idx_released = entry->idx;
 226
 227         spin_unlock_bh(&vmci_doorbell_it.lock);
 228
 229         vmci_resource_put(&entry->resource);
 230 }
 231
 232 /*
 233  * Creates a link between the given doorbell handle and the given
 234  * index in the bitmap in the device backend. A notification state
 235  * is created in hypervisor.
 236  */
 237 static int dbell_link(struct vmci_handle handle, u32 notify_idx)
 238 {
 239         struct vmci_doorbell_link_msg link_msg;
 240
 241         link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 242                                             VMCI_DOORBELL_LINK);
 243         link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 244         link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
 245         link_msg.handle = handle;
 246         link_msg.notify_idx = notify_idx;
 247
 248         return vmci_send_datagram(&link_msg.hdr);
 249 }
 250
 251 /*
 252  * Unlinks the given doorbell handle from an index in the bitmap in
 253  * the device backend. The notification state is destroyed in hypervisor.
 254  */
 255 static int dbell_unlink(struct vmci_handle handle)
 256 {
 257         struct vmci_doorbell_unlink_msg unlink_msg;
 258
 259         unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 260                                               VMCI_DOORBELL_UNLINK);
 261         unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 262         unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
 263         unlink_msg.handle = handle;
 264
 265         return vmci_send_datagram(&unlink_msg.hdr);
 266 }
 267
 268 /*
 269  * Notify another guest or the host.  We send a datagram down to the
 270  * host via the hypervisor with the notification info.
 271  */
 272 static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
 273 {
 274         struct vmci_doorbell_notify_msg notify_msg;
 275
 276         notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 277                                               VMCI_DOORBELL_NOTIFY);
 278         notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 279         notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
 280         notify_msg.handle = handle;
 281
 282         return vmci_send_datagram(&notify_msg.hdr);
 283 }
 284
 285 /*
 286  * Calls the specified callback in a delayed context.
 287  */
 288 static void dbell_delayed_dispatch(struct work_struct *work)
 289 {
 290         struct dbell_entry *entry = container_of(work,
 291                                                  struct dbell_entry, work);
 292
 293         entry->notify_cb(entry->client_data);
 294         vmci_resource_put(&entry->resource);
 295 }
 296
 297 /*
 298  * Dispatches a doorbell notification to the host context.
 299  */
 300 int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
 301 {
 302         struct dbell_entry *entry;
 303         struct vmci_resource *resource;
 304
 305         if (vmci_handle_is_invalid(handle)) {
 306                 pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
 307                          handle.context, handle.resource);
 308                 return VMCI_ERROR_INVALID_ARGS;
 309         }
 310
 311         resource = vmci_resource_by_handle(handle,
 312                                            VMCI_RESOURCE_TYPE_DOORBELL);
 313         if (!resource) {
 314                 pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
 315                          handle.context, handle.resource);
 316                 return VMCI_ERROR_NOT_FOUND;
 317         }
 318
 319         entry = container_of(resource, struct dbell_entry, resource);
 320         if (entry->run_delayed) {
 321                 schedule_work(&entry->work);
 322         } else {
 323                 entry->notify_cb(entry->client_data);
 324                 vmci_resource_put(resource);
 325         }
 326
 327         return VMCI_SUCCESS;
 328 }
 329
 330 /*
 331  * Register the notification bitmap with the host.
 332  */
 333 bool vmci_dbell_register_notification_bitmap(u32 bitmap_ppn)
 334 {
 335         int result;
 336         struct vmci_notify_bm_set_msg bitmap_set_msg;
 337
 338         bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 339                                                   VMCI_SET_NOTIFY_BITMAP);
 340         bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 341         bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
 342             VMCI_DG_HEADERSIZE;
 343         bitmap_set_msg.bitmap_ppn = bitmap_ppn;
 344
 345         result = vmci_send_datagram(&bitmap_set_msg.hdr);
 346         if (result != VMCI_SUCCESS) {
 347                 pr_devel("Failed to register (PPN=%u) as notification bitmap (error=%d)\n",
 348                          bitmap_ppn, result);
 349                 return false;
 350         }
 351         return true;
 352 }
 353
 354 /*
 355  * Executes or schedules the handlers for a given notify index.
 356  */
 357 static void dbell_fire_entries(u32 notify_idx)
 358 {
 359         u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
 360         struct dbell_entry *dbell;
 361
 362         spin_lock_bh(&vmci_doorbell_it.lock);
 363
 364         hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
 365                 if (dbell->idx == notify_idx &&
 366                     atomic_read(&dbell->active) == 1) {
 367                         if (dbell->run_delayed) {
 368                                 vmci_resource_get(&dbell->resource);
 369                                 schedule_work(&dbell->work);
 370                         } else {
 371                                 dbell->notify_cb(dbell->client_data);
 372                         }
 373                 }
 374         }
 375
 376         spin_unlock_bh(&vmci_doorbell_it.lock);
 377 }
 378
 379 /*
 380  * Scans the notification bitmap, collects pending notifications,
 381  * resets the bitmap and invokes appropriate callbacks.
 382  */
 383 void vmci_dbell_scan_notification_entries(u8 *bitmap)
 384 {
 385         u32 idx;
 386
 387         for (idx = 0; idx < max_notify_idx; idx++) {
 388                 if (bitmap[idx] & 0x1) {
 389                         bitmap[idx] &= ~1;
 390                         dbell_fire_entries(idx);
 391                 }
 392         }
 393 }
 394
 395 /*
 396  * vmci_doorbell_create() - Creates a doorbell
 397  * @handle:     A handle used to track the resource.  Can be invalid.
 398  * @flags:      Flag that determines context of callback.
 399  * @priv_flags: Privileges flags.
 400  * @notify_cb:  The callback to be ivoked when the doorbell fires.
 401  * @client_data:        A parameter to be passed to the callback.
 402  *
 403  * Creates a doorbell with the given callback. If the handle is
 404  * VMCI_INVALID_HANDLE, a free handle will be assigned, if
 405  * possible. The callback can be run immediately (potentially with
 406  * locks held - the default) or delayed (in a kernel thread) by
 407  * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
 408  * is selected, a given callback may not be run if the kernel is
 409  * unable to allocate memory for the delayed execution (highly
 410  * unlikely).
 411  */
 412 int vmci_doorbell_create(struct vmci_handle *handle,
 413                          u32 flags,
 414                          u32 priv_flags,
 415                          vmci_callback notify_cb, void *client_data)
 416 {
 417         struct dbell_entry *entry;
 418         struct vmci_handle new_handle;
 419         int result;
 420
 421         if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
 422             priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
 423                 return VMCI_ERROR_INVALID_ARGS;
 424
 425         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 426         if (entry == NULL) {
 427                 pr_warn("Failed allocating memory for datagram entry\n");
 428                 return VMCI_ERROR_NO_MEM;
 429         }
 430
 431         if (vmci_handle_is_invalid(*handle)) {
 432                 u32 context_id = vmci_get_context_id();
 433
 434                 if (context_id == VMCI_INVALID_ID) {
 435                         pr_warn("Failed to get context ID\n");
 436                         result = VMCI_ERROR_NO_RESOURCES;
 437                         goto free_mem;
 438                 }
 439
 440                 /* Let resource code allocate a free ID for us */
 441                 new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
 442         } else {
 443                 bool valid_context = false;
 444
 445                 /*
 446                  * Validate the handle.  We must do both of the checks below
 447                  * because we can be acting as both a host and a guest at the
 448                  * same time. We always allow the host context ID, since the
 449                  * host functionality is in practice always there with the
 450                  * unified driver.
 451                  */
 452                 if (handle->context == VMCI_HOST_CONTEXT_ID ||
 453                     (vmci_guest_code_active() &&
 454                      vmci_get_context_id() == handle->context)) {
 455                         valid_context = true;
 456                 }
 457
 458                 if (!valid_context || handle->resource == VMCI_INVALID_ID) {
 459                         pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
 460                                  handle->context, handle->resource);
 461                         result = VMCI_ERROR_INVALID_ARGS;
 462                         goto free_mem;
 463                 }
 464
 465                 new_handle = *handle;
 466         }
 467
 468         entry->idx = 0;
 469         INIT_HLIST_NODE(&entry->node);
 470         entry->priv_flags = priv_flags;
 471         INIT_WORK(&entry->work, dbell_delayed_dispatch);
 472         entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
 473         entry->notify_cb = notify_cb;
 474         entry->client_data = client_data;
 475         atomic_set(&entry->active, 0);
 476
 477         result = vmci_resource_add(&entry->resource,
 478                                    VMCI_RESOURCE_TYPE_DOORBELL,
 479                                    new_handle);
 480         if (result != VMCI_SUCCESS) {
 481                 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
 482                         new_handle.context, new_handle.resource, result);
 483                 goto free_mem;
 484         }
 485
 486         new_handle = vmci_resource_handle(&entry->resource);
 487         if (vmci_guest_code_active()) {
 488                 dbell_index_table_add(entry);
 489                 result = dbell_link(new_handle, entry->idx);
 490                 if (VMCI_SUCCESS != result)
 491                         goto destroy_resource;
 492
 493                 atomic_set(&entry->active, 1);
 494         }
 495
 496         *handle = new_handle;
 497
 498         return result;
 499
 500  destroy_resource:
 501         dbell_index_table_remove(entry);
 502         vmci_resource_remove(&entry->resource);
 503  free_mem:
 504         kfree(entry);
 505         return result;
 506 }
 507 EXPORT_SYMBOL_GPL(vmci_doorbell_create);
 508
 509 /*
 510  * vmci_doorbell_destroy() - Destroy a doorbell.
 511  * @handle:     The handle tracking the resource.
 512  *
 513  * Destroys a doorbell previously created with vmcii_doorbell_create. This
 514  * operation may block waiting for a callback to finish.
 515  */
 516 int vmci_doorbell_destroy(struct vmci_handle handle)
 517 {
 518         struct dbell_entry *entry;
 519         struct vmci_resource *resource;
 520
 521         if (vmci_handle_is_invalid(handle))
 522                 return VMCI_ERROR_INVALID_ARGS;
 523
 524         resource = vmci_resource_by_handle(handle,
 525                                            VMCI_RESOURCE_TYPE_DOORBELL);
 526         if (!resource) {
 527                 pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
 528                          handle.context, handle.resource);
 529                 return VMCI_ERROR_NOT_FOUND;
 530         }
 531
 532         entry = container_of(resource, struct dbell_entry, resource);
 533
 534         if (!hlist_unhashed(&entry->node)) {
 535                 int result;
 536
 537                 dbell_index_table_remove(entry);
 538
 539                 result = dbell_unlink(handle);
 540                 if (VMCI_SUCCESS != result) {
 541
 542                         /*
 543                          * The only reason this should fail would be
 544                          * an inconsistency between guest and
 545                          * hypervisor state, where the guest believes
 546                          * it has an active registration whereas the
 547                          * hypervisor doesn't. One case where this may
 548                          * happen is if a doorbell is unregistered
 549                          * following a hibernation at a time where the
 550                          * doorbell state hasn't been restored on the
 551                          * hypervisor side yet. Since the handle has
 552                          * now been removed in the guest, we just
 553                          * print a warning and return success.
 554                          */
 555                         pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
 556                                  handle.context, handle.resource, result);
 557                 }
 558         }
 559
 560         /*
 561          * Now remove the resource from the table.  It might still be in use
 562          * after this, in a callback or still on the delayed work queue.
 563          */
 564         vmci_resource_put(&entry->resource);
 565         vmci_resource_remove(&entry->resource);
 566
 567         kfree(entry);
 568
 569         return VMCI_SUCCESS;
 570 }
 571 EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
 572
 573 /*
 574  * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
 575  * @dst:        The handlle identifying the doorbell resource
 576  * @priv_flags: Priviledge flags.
 577  *
 578  * Generates a notification on the doorbell identified by the
 579  * handle. For host side generation of notifications, the caller
 580  * can specify what the privilege of the calling side is.
 581  */
 582 int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
 583 {
 584         int retval;
 585         enum vmci_route route;
 586         struct vmci_handle src;
 587
 588         if (vmci_handle_is_invalid(dst) ||
 589             (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
 590                 return VMCI_ERROR_INVALID_ARGS;
 591
 592         src = VMCI_INVALID_HANDLE;
 593         retval = vmci_route(&src, &dst, false, &route);
 594         if (retval < VMCI_SUCCESS)
 595                 return retval;
 596
 597         if (VMCI_ROUTE_AS_HOST == route)
 598                 return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
 599                                              dst, priv_flags);
 600
 601         if (VMCI_ROUTE_AS_GUEST == route)
 602                 return dbell_notify_as_guest(dst, priv_flags);
 603
 604         pr_warn("Unknown route (%d) for doorbell\n", route);
 605         return VMCI_ERROR_DST_UNREACHABLE;
 606 }
 607 EXPORT_SYMBOL_GPL(vmci_doorbell_notify);