drivers/staging/greybus/operation.c

   1 /*
   2  * Greybus operations
   3  *
   4  * Copyright 2014-2015 Google Inc.
   5  * Copyright 2014-2015 Linaro Ltd.
   6  *
   7  * Released under the GPLv2 only.
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/slab.h>
  12 #include <linux/module.h>
  13 #include <linux/sched.h>
  14 #include <linux/wait.h>
  15 #include <linux/workqueue.h>
  16
  17 #include "greybus.h"
  18 #include "greybus_trace.h"
  19
  20 static struct kmem_cache *gb_operation_cache;
  21 static struct kmem_cache *gb_message_cache;
  22
  23 /* Workqueue to handle Greybus operation completions. */
  24 static struct workqueue_struct *gb_operation_completion_wq;
  25
  26 /* Wait queue for synchronous cancellations. */
  27 static DECLARE_WAIT_QUEUE_HEAD(gb_operation_cancellation_queue);
  28
  29 /*
  30  * Protects updates to operation->errno.
  31  */
  32 static DEFINE_SPINLOCK(gb_operations_lock);
  33
  34 static int gb_operation_response_send(struct gb_operation *operation,
  35                                         int errno);
  36
  37 /*
  38  * Increment operation active count and add to connection list unless the
  39  * connection is going away.
  40  *
  41  * Caller holds operation reference.
  42  */
  43 static int gb_operation_get_active(struct gb_operation *operation)
  44 {
  45         struct gb_connection *connection = operation->connection;
  46         unsigned long flags;
  47
  48         spin_lock_irqsave(&connection->lock, flags);
  49
  50         switch (connection->state) {
  51         case GB_CONNECTION_STATE_ENABLED:
  52                 break;
  53         case GB_CONNECTION_STATE_ENABLED_TX:
  54                 if (gb_operation_is_incoming(operation))
  55                         goto err_unlock;
  56                 break;
  57         default:
  58                 goto err_unlock;
  59         }
  60
  61         if (operation->active++ == 0)
  62                 list_add_tail(&operation->links, &connection->operations);
  63
  64         trace_gb_operation_get_active(operation);
  65
  66         spin_unlock_irqrestore(&connection->lock, flags);
  67
  68         return 0;
  69
  70 err_unlock:
  71         spin_unlock_irqrestore(&connection->lock, flags);
  72
  73         return -ENOTCONN;
  74 }
  75
  76 /* Caller holds operation reference. */
  77 static void gb_operation_put_active(struct gb_operation *operation)
  78 {
  79         struct gb_connection *connection = operation->connection;
  80         unsigned long flags;
  81
  82         spin_lock_irqsave(&connection->lock, flags);
  83
  84         trace_gb_operation_put_active(operation);
  85
  86         if (--operation->active == 0) {
  87                 list_del(&operation->links);
  88                 if (atomic_read(&operation->waiters))
  89                         wake_up(&gb_operation_cancellation_queue);
  90         }
  91         spin_unlock_irqrestore(&connection->lock, flags);
  92 }
  93
  94 static bool gb_operation_is_active(struct gb_operation *operation)
  95 {
  96         struct gb_connection *connection = operation->connection;
  97         unsigned long flags;
  98         bool ret;
  99
 100         spin_lock_irqsave(&connection->lock, flags);
 101         ret = operation->active;
 102         spin_unlock_irqrestore(&connection->lock, flags);
 103
 104         return ret;
 105 }
 106
 107 /*
 108  * Set an operation's result.
 109  *
 110  * Initially an outgoing operation's errno value is -EBADR.
 111  * If no error occurs before sending the request message the only
 112  * valid value operation->errno can be set to is -EINPROGRESS,
 113  * indicating the request has been (or rather is about to be) sent.
 114  * At that point nobody should be looking at the result until the
 115  * response arrives.
 116  *
 117  * The first time the result gets set after the request has been
 118  * sent, that result "sticks."  That is, if two concurrent threads
 119  * race to set the result, the first one wins.  The return value
 120  * tells the caller whether its result was recorded; if not the
 121  * caller has nothing more to do.
 122  *
 123  * The result value -EILSEQ is reserved to signal an implementation
 124  * error; if it's ever observed, the code performing the request has
 125  * done something fundamentally wrong.  It is an error to try to set
 126  * the result to -EBADR, and attempts to do so result in a warning,
 127  * and -EILSEQ is used instead.  Similarly, the only valid result
 128  * value to set for an operation in initial state is -EINPROGRESS.
 129  * Attempts to do otherwise will also record a (successful) -EILSEQ
 130  * operation result.
 131  */
 132 static bool gb_operation_result_set(struct gb_operation *operation, int result)
 133 {
 134         unsigned long flags;
 135         int prev;
 136
 137         if (result == -EINPROGRESS) {
 138                 /*
 139                  * -EINPROGRESS is used to indicate the request is
 140                  * in flight.  It should be the first result value
 141                  * set after the initial -EBADR.  Issue a warning
 142                  * and record an implementation error if it's
 143                  * set at any other time.
 144                  */
 145                 spin_lock_irqsave(&gb_operations_lock, flags);
 146                 prev = operation->errno;
 147                 if (prev == -EBADR)
 148                         operation->errno = result;
 149                 else
 150                         operation->errno = -EILSEQ;
 151                 spin_unlock_irqrestore(&gb_operations_lock, flags);
 152                 WARN_ON(prev != -EBADR);
 153
 154                 return true;
 155         }
 156
 157         /*
 158          * The first result value set after a request has been sent
 159          * will be the final result of the operation.  Subsequent
 160          * attempts to set the result are ignored.
 161          *
 162          * Note that -EBADR is a reserved "initial state" result
 163          * value.  Attempts to set this value result in a warning,
 164          * and the result code is set to -EILSEQ instead.
 165          */
 166         if (WARN_ON(result == -EBADR))
 167                 result = -EILSEQ; /* Nobody should be setting -EBADR */
 168
 169         spin_lock_irqsave(&gb_operations_lock, flags);
 170         prev = operation->errno;
 171         if (prev == -EINPROGRESS)
 172                 operation->errno = result;      /* First and final result */
 173         spin_unlock_irqrestore(&gb_operations_lock, flags);
 174
 175         return prev == -EINPROGRESS;
 176 }
 177
 178 int gb_operation_result(struct gb_operation *operation)
 179 {
 180         int result = operation->errno;
 181
 182         WARN_ON(result == -EBADR);
 183         WARN_ON(result == -EINPROGRESS);
 184
 185         return result;
 186 }
 187 EXPORT_SYMBOL_GPL(gb_operation_result);
 188
 189 /*
 190  * Looks up an outgoing operation on a connection and returns a refcounted
 191  * pointer if found, or NULL otherwise.
 192  */
 193 static struct gb_operation *
 194 gb_operation_find_outgoing(struct gb_connection *connection, u16 operation_id)
 195 {
 196         struct gb_operation *operation;
 197         unsigned long flags;
 198         bool found = false;
 199
 200         spin_lock_irqsave(&connection->lock, flags);
 201         list_for_each_entry(operation, &connection->operations, links)
 202                 if (operation->id == operation_id &&
 203                                 !gb_operation_is_incoming(operation)) {
 204                         gb_operation_get(operation);
 205                         found = true;
 206                         break;
 207                 }
 208         spin_unlock_irqrestore(&connection->lock, flags);
 209
 210         return found ? operation : NULL;
 211 }
 212
 213 static int gb_message_send(struct gb_message *message, gfp_t gfp)
 214 {
 215         struct gb_connection *connection = message->operation->connection;
 216
 217         trace_gb_message_send(message);
 218         return connection->hd->driver->message_send(connection->hd,
 219                                         connection->hd_cport_id,
 220                                         message,
 221                                         gfp);
 222 }
 223
 224 /*
 225  * Cancel a message we have passed to the host device layer to be sent.
 226  */
 227 static void gb_message_cancel(struct gb_message *message)
 228 {
 229         struct gb_host_device *hd = message->operation->connection->hd;
 230
 231         hd->driver->message_cancel(message);
 232 }
 233
 234 static void gb_operation_request_handle(struct gb_operation *operation)
 235 {
 236         struct gb_connection *connection = operation->connection;
 237         int status;
 238         int ret;
 239
 240         if (connection->handler) {
 241                 status = connection->handler(operation);
 242         } else {
 243                 dev_err(&connection->hd->dev,
 244                         "%s: unexpected incoming request of type 0x%02x\n",
 245                         connection->name, operation->type);
 246
 247                 status = -EPROTONOSUPPORT;
 248         }
 249
 250         ret = gb_operation_response_send(operation, status);
 251         if (ret) {
 252                 dev_err(&connection->hd->dev,
 253                         "%s: failed to send response %d for type 0x%02x: %d\n",
 254                         connection->name, status, operation->type, ret);
 255                 return;
 256         }
 257 }
 258
 259 /*
 260  * Process operation work.
 261  *
 262  * For incoming requests, call the protocol request handler. The operation
 263  * result should be -EINPROGRESS at this point.
 264  *
 265  * For outgoing requests, the operation result value should have
 266  * been set before queueing this.  The operation callback function
 267  * allows the original requester to know the request has completed
 268  * and its result is available.
 269  */
 270 static void gb_operation_work(struct work_struct *work)
 271 {
 272         struct gb_operation *operation;
 273
 274         operation = container_of(work, struct gb_operation, work);
 275
 276         if (gb_operation_is_incoming(operation))
 277                 gb_operation_request_handle(operation);
 278         else
 279                 operation->callback(operation);
 280
 281         gb_operation_put_active(operation);
 282         gb_operation_put(operation);
 283 }
 284
 285 static void gb_operation_message_init(struct gb_host_device *hd,
 286                                 struct gb_message *message, u16 operation_id,
 287                                 size_t payload_size, u8 type)
 288 {
 289         struct gb_operation_msg_hdr *header;
 290
 291         header = message->buffer;
 292
 293         message->header = header;
 294         message->payload = payload_size ? header + 1 : NULL;
 295         message->payload_size = payload_size;
 296
 297         /*
 298          * The type supplied for incoming message buffers will be
 299          * GB_REQUEST_TYPE_INVALID. Such buffers will be overwritten by
 300          * arriving data so there's no need to initialize the message header.
 301          */
 302         if (type != GB_REQUEST_TYPE_INVALID) {
 303                 u16 message_size = (u16)(sizeof(*header) + payload_size);
 304
 305                 /*
 306                  * For a request, the operation id gets filled in
 307                  * when the message is sent.  For a response, it
 308                  * will be copied from the request by the caller.
 309                  *
 310                  * The result field in a request message must be
 311                  * zero.  It will be set just prior to sending for
 312                  * a response.
 313                  */
 314                 header->size = cpu_to_le16(message_size);
 315                 header->operation_id = 0;
 316                 header->type = type;
 317                 header->result = 0;
 318         }
 319 }
 320
 321 /*
 322  * Allocate a message to be used for an operation request or response.
 323  * Both types of message contain a common header.  The request message
 324  * for an outgoing operation is outbound, as is the response message
 325  * for an incoming operation.  The message header for an outbound
 326  * message is partially initialized here.
 327  *
 328  * The headers for inbound messages don't need to be initialized;
 329  * they'll be filled in by arriving data.
 330  *
 331  * Our message buffers have the following layout:
 332  *      message header  \_ these combined are
 333  *      message payload /  the message size
 334  */
 335 static struct gb_message *
 336 gb_operation_message_alloc(struct gb_host_device *hd, u8 type,
 337                                 size_t payload_size, gfp_t gfp_flags)
 338 {
 339         struct gb_message *message;
 340         struct gb_operation_msg_hdr *header;
 341         size_t message_size = payload_size + sizeof(*header);
 342
 343         if (message_size > hd->buffer_size_max) {
 344                 dev_warn(&hd->dev, "requested message size too big (%zu > %zu)\n",
 345                                 message_size, hd->buffer_size_max);
 346                 return NULL;
 347         }
 348
 349         /* Allocate the message structure and buffer. */
 350         message = kmem_cache_zalloc(gb_message_cache, gfp_flags);
 351         if (!message)
 352                 return NULL;
 353
 354         message->buffer = kzalloc(message_size, gfp_flags);
 355         if (!message->buffer)
 356                 goto err_free_message;
 357
 358         /* Initialize the message.  Operation id is filled in later. */
 359         gb_operation_message_init(hd, message, 0, payload_size, type);
 360
 361         return message;
 362
 363 err_free_message:
 364         kmem_cache_free(gb_message_cache, message);
 365
 366         return NULL;
 367 }
 368
 369 static void gb_operation_message_free(struct gb_message *message)
 370 {
 371         kfree(message->buffer);
 372         kmem_cache_free(gb_message_cache, message);
 373 }
 374
 375 /*
 376  * Map an enum gb_operation_status value (which is represented in a
 377  * message as a single byte) to an appropriate Linux negative errno.
 378  */
 379 static int gb_operation_status_map(u8 status)
 380 {
 381         switch (status) {
 382         case GB_OP_SUCCESS:
 383                 return 0;
 384         case GB_OP_INTERRUPTED:
 385                 return -EINTR;
 386         case GB_OP_TIMEOUT:
 387                 return -ETIMEDOUT;
 388         case GB_OP_NO_MEMORY:
 389                 return -ENOMEM;
 390         case GB_OP_PROTOCOL_BAD:
 391                 return -EPROTONOSUPPORT;
 392         case GB_OP_OVERFLOW:
 393                 return -EMSGSIZE;
 394         case GB_OP_INVALID:
 395                 return -EINVAL;
 396         case GB_OP_RETRY:
 397                 return -EAGAIN;
 398         case GB_OP_NONEXISTENT:
 399                 return -ENODEV;
 400         case GB_OP_MALFUNCTION:
 401                 return -EILSEQ;
 402         case GB_OP_UNKNOWN_ERROR:
 403         default:
 404                 return -EIO;
 405         }
 406 }
 407
 408 /*
 409  * Map a Linux errno value (from operation->errno) into the value
 410  * that should represent it in a response message status sent
 411  * over the wire.  Returns an enum gb_operation_status value (which
 412  * is represented in a message as a single byte).
 413  */
 414 static u8 gb_operation_errno_map(int errno)
 415 {
 416         switch (errno) {
 417         case 0:
 418                 return GB_OP_SUCCESS;
 419         case -EINTR:
 420                 return GB_OP_INTERRUPTED;
 421         case -ETIMEDOUT:
 422                 return GB_OP_TIMEOUT;
 423         case -ENOMEM:
 424                 return GB_OP_NO_MEMORY;
 425         case -EPROTONOSUPPORT:
 426                 return GB_OP_PROTOCOL_BAD;
 427         case -EMSGSIZE:
 428                 return GB_OP_OVERFLOW;  /* Could be underflow too */
 429         case -EINVAL:
 430                 return GB_OP_INVALID;
 431         case -EAGAIN:
 432                 return GB_OP_RETRY;
 433         case -EILSEQ:
 434                 return GB_OP_MALFUNCTION;
 435         case -ENODEV:
 436                 return GB_OP_NONEXISTENT;
 437         case -EIO:
 438         default:
 439                 return GB_OP_UNKNOWN_ERROR;
 440         }
 441 }
 442
 443 bool gb_operation_response_alloc(struct gb_operation *operation,
 444                                         size_t response_size, gfp_t gfp)
 445 {
 446         struct gb_host_device *hd = operation->connection->hd;
 447         struct gb_operation_msg_hdr *request_header;
 448         struct gb_message *response;
 449         u8 type;
 450
 451         type = operation->type | GB_MESSAGE_TYPE_RESPONSE;
 452         response = gb_operation_message_alloc(hd, type, response_size, gfp);
 453         if (!response)
 454                 return false;
 455         response->operation = operation;
 456
 457         /*
 458          * Size and type get initialized when the message is
 459          * allocated.  The errno will be set before sending.  All
 460          * that's left is the operation id, which we copy from the
 461          * request message header (as-is, in little-endian order).
 462          */
 463         request_header = operation->request->header;
 464         response->header->operation_id = request_header->operation_id;
 465         operation->response = response;
 466
 467         return true;
 468 }
 469 EXPORT_SYMBOL_GPL(gb_operation_response_alloc);
 470
 471 /*
 472  * Create a Greybus operation to be sent over the given connection.
 473  * The request buffer will be big enough for a payload of the given
 474  * size.
 475  *
 476  * For outgoing requests, the request message's header will be
 477  * initialized with the type of the request and the message size.
 478  * Outgoing operations must also specify the response buffer size,
 479  * which must be sufficient to hold all expected response data.  The
 480  * response message header will eventually be overwritten, so there's
 481  * no need to initialize it here.
 482  *
 483  * Request messages for incoming operations can arrive in interrupt
 484  * context, so they must be allocated with GFP_ATOMIC.  In this case
 485  * the request buffer will be immediately overwritten, so there is
 486  * no need to initialize the message header.  Responsibility for
 487  * allocating a response buffer lies with the incoming request
 488  * handler for a protocol.  So we don't allocate that here.
 489  *
 490  * Returns a pointer to the new operation or a null pointer if an
 491  * error occurs.
 492  */
 493 static struct gb_operation *
 494 gb_operation_create_common(struct gb_connection *connection, u8 type,
 495                                 size_t request_size, size_t response_size,
 496                                 unsigned long op_flags, gfp_t gfp_flags)
 497 {
 498         struct gb_host_device *hd = connection->hd;
 499         struct gb_operation *operation;
 500
 501         operation = kmem_cache_zalloc(gb_operation_cache, gfp_flags);
 502         if (!operation)
 503                 return NULL;
 504         operation->connection = connection;
 505
 506         operation->request = gb_operation_message_alloc(hd, type, request_size,
 507                                                         gfp_flags);
 508         if (!operation->request)
 509                 goto err_cache;
 510         operation->request->operation = operation;
 511
 512         /* Allocate the response buffer for outgoing operations */
 513         if (!(op_flags & GB_OPERATION_FLAG_INCOMING)) {
 514                 if (!gb_operation_response_alloc(operation, response_size,
 515                                                  gfp_flags)) {
 516                         goto err_request;
 517                 }
 518         }
 519
 520         operation->flags = op_flags;
 521         operation->type = type;
 522         operation->errno = -EBADR;  /* Initial value--means "never set" */
 523
 524         INIT_WORK(&operation->work, gb_operation_work);
 525         init_completion(&operation->completion);
 526         kref_init(&operation->kref);
 527         atomic_set(&operation->waiters, 0);
 528
 529         return operation;
 530
 531 err_request:
 532         gb_operation_message_free(operation->request);
 533 err_cache:
 534         kmem_cache_free(gb_operation_cache, operation);
 535
 536         return NULL;
 537 }
 538
 539 /*
 540  * Create a new operation associated with the given connection.  The
 541  * request and response sizes provided are the number of bytes
 542  * required to hold the request/response payload only.  Both of
 543  * these are allowed to be 0.  Note that 0x00 is reserved as an
 544  * invalid operation type for all protocols, and this is enforced
 545  * here.
 546  */
 547 struct gb_operation *
 548 gb_operation_create_flags(struct gb_connection *connection,
 549                                 u8 type, size_t request_size,
 550                                 size_t response_size, unsigned long flags,
 551                                 gfp_t gfp)
 552 {
 553         struct gb_operation *operation;
 554
 555         if (WARN_ON_ONCE(type == GB_REQUEST_TYPE_INVALID))
 556                 return NULL;
 557         if (WARN_ON_ONCE(type & GB_MESSAGE_TYPE_RESPONSE))
 558                 type &= ~GB_MESSAGE_TYPE_RESPONSE;
 559
 560         if (WARN_ON_ONCE(flags & ~GB_OPERATION_FLAG_USER_MASK))
 561                 flags &= GB_OPERATION_FLAG_USER_MASK;
 562
 563         operation = gb_operation_create_common(connection, type,
 564                                                 request_size, response_size,
 565                                                 flags, gfp);
 566         if (operation)
 567                 trace_gb_operation_create(operation);
 568
 569         return operation;
 570
 571 }
 572 EXPORT_SYMBOL_GPL(gb_operation_create_flags);
 573
 574 size_t gb_operation_get_payload_size_max(struct gb_connection *connection)
 575 {
 576         struct gb_host_device *hd = connection->hd;
 577
 578         return hd->buffer_size_max - sizeof(struct gb_operation_msg_hdr);
 579 }
 580 EXPORT_SYMBOL_GPL(gb_operation_get_payload_size_max);
 581
 582 static struct gb_operation *
 583 gb_operation_create_incoming(struct gb_connection *connection, u16 id,
 584                                 u8 type, void *data, size_t size)
 585 {
 586         struct gb_operation *operation;
 587         size_t request_size;
 588         unsigned long flags = GB_OPERATION_FLAG_INCOMING;
 589
 590         /* Caller has made sure we at least have a message header. */
 591         request_size = size - sizeof(struct gb_operation_msg_hdr);
 592
 593         if (!id)
 594                 flags |= GB_OPERATION_FLAG_UNIDIRECTIONAL;
 595
 596         operation = gb_operation_create_common(connection, type,
 597                                                 request_size,
 598                                                 GB_REQUEST_TYPE_INVALID,
 599                                                 flags, GFP_ATOMIC);
 600         if (!operation)
 601                 return NULL;
 602
 603         operation->id = id;
 604         memcpy(operation->request->header, data, size);
 605         trace_gb_operation_create_incoming(operation);
 606
 607         return operation;
 608 }
 609
 610 /*
 611  * Get an additional reference on an operation.
 612  */
 613 void gb_operation_get(struct gb_operation *operation)
 614 {
 615         kref_get(&operation->kref);
 616 }
 617 EXPORT_SYMBOL_GPL(gb_operation_get);
 618
 619 /*
 620  * Destroy a previously created operation.
 621  */
 622 static void _gb_operation_destroy(struct kref *kref)
 623 {
 624         struct gb_operation *operation;
 625
 626         operation = container_of(kref, struct gb_operation, kref);
 627
 628         trace_gb_operation_destroy(operation);
 629
 630         if (operation->response)
 631                 gb_operation_message_free(operation->response);
 632         gb_operation_message_free(operation->request);
 633
 634         kmem_cache_free(gb_operation_cache, operation);
 635 }
 636
 637 /*
 638  * Drop a reference on an operation, and destroy it when the last
 639  * one is gone.
 640  */
 641 void gb_operation_put(struct gb_operation *operation)
 642 {
 643         if (WARN_ON(!operation))
 644                 return;
 645
 646         kref_put(&operation->kref, _gb_operation_destroy);
 647 }
 648 EXPORT_SYMBOL_GPL(gb_operation_put);
 649
 650 /* Tell the requester we're done */
 651 static void gb_operation_sync_callback(struct gb_operation *operation)
 652 {
 653         complete(&operation->completion);
 654 }
 655
 656 /**
 657  * gb_operation_request_send() - send an operation request message
 658  * @operation:  the operation to initiate
 659  * @callback:   the operation completion callback
 660  * @gfp:        the memory flags to use for any allocations
 661  *
 662  * The caller has filled in any payload so the request message is ready to go.
 663  * The callback function supplied will be called when the response message has
 664  * arrived, a unidirectional request has been sent, or the operation is
 665  * cancelled, indicating that the operation is complete. The callback function
 666  * can fetch the result of the operation using gb_operation_result() if
 667  * desired.
 668  *
 669  * Return: 0 if the request was successfully queued in the host-driver queues,
 670  * or a negative errno.
 671  */
 672 int gb_operation_request_send(struct gb_operation *operation,
 673                                 gb_operation_callback callback,
 674                                 gfp_t gfp)
 675 {
 676         struct gb_connection *connection = operation->connection;
 677         struct gb_operation_msg_hdr *header;
 678         unsigned int cycle;
 679         int ret;
 680
 681         if (gb_connection_is_offloaded(connection))
 682                 return -EBUSY;
 683
 684         if (!callback)
 685                 return -EINVAL;
 686
 687         /*
 688          * Record the callback function, which is executed in
 689          * non-atomic (workqueue) context when the final result
 690          * of an operation has been set.
 691          */
 692         operation->callback = callback;
 693
 694         /*
 695          * Assign the operation's id, and store it in the request header.
 696          * Zero is a reserved operation id for unidirectional operations.
 697          */
 698         if (gb_operation_is_unidirectional(operation)) {
 699                 operation->id = 0;
 700         } else {
 701                 cycle = (unsigned int)atomic_inc_return(&connection->op_cycle);
 702                 operation->id = (u16)(cycle % U16_MAX + 1);
 703         }
 704
 705         header = operation->request->header;
 706         header->operation_id = cpu_to_le16(operation->id);
 707
 708         gb_operation_result_set(operation, -EINPROGRESS);
 709
 710         /*
 711          * Get an extra reference on the operation. It'll be dropped when the
 712          * operation completes.
 713          */
 714         gb_operation_get(operation);
 715         ret = gb_operation_get_active(operation);
 716         if (ret)
 717                 goto err_put;
 718
 719         ret = gb_message_send(operation->request, gfp);
 720         if (ret)
 721                 goto err_put_active;
 722
 723         return 0;
 724
 725 err_put_active:
 726         gb_operation_put_active(operation);
 727 err_put:
 728         gb_operation_put(operation);
 729
 730         return ret;
 731 }
 732 EXPORT_SYMBOL_GPL(gb_operation_request_send);
 733
 734 /*
 735  * Send a synchronous operation.  This function is expected to
 736  * block, returning only when the response has arrived, (or when an
 737  * error is detected.  The return value is the result of the
 738  * operation.
 739  */
 740 int gb_operation_request_send_sync_timeout(struct gb_operation *operation,
 741                                                 unsigned int timeout)
 742 {
 743         int ret;
 744         unsigned long timeout_jiffies;
 745
 746         ret = gb_operation_request_send(operation, gb_operation_sync_callback,
 747                                         GFP_KERNEL);
 748         if (ret)
 749                 return ret;
 750
 751         if (timeout)
 752                 timeout_jiffies = msecs_to_jiffies(timeout);
 753         else
 754                 timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
 755
 756         ret = wait_for_completion_interruptible_timeout(&operation->completion,
 757                                                         timeout_jiffies);
 758         if (ret < 0) {
 759                 /* Cancel the operation if interrupted */
 760                 gb_operation_cancel(operation, -ECANCELED);
 761         } else if (ret == 0) {
 762                 /* Cancel the operation if op timed out */
 763                 gb_operation_cancel(operation, -ETIMEDOUT);
 764         }
 765
 766         return gb_operation_result(operation);
 767 }
 768 EXPORT_SYMBOL_GPL(gb_operation_request_send_sync_timeout);
 769
 770 /*
 771  * Send a response for an incoming operation request.  A non-zero
 772  * errno indicates a failed operation.
 773  *
 774  * If there is any response payload, the incoming request handler is
 775  * responsible for allocating the response message.  Otherwise the
 776  * it can simply supply the result errno; this function will
 777  * allocate the response message if necessary.
 778  */
 779 static int gb_operation_response_send(struct gb_operation *operation,
 780                                         int errno)
 781 {
 782         struct gb_connection *connection = operation->connection;
 783         int ret;
 784
 785         if (!operation->response &&
 786                         !gb_operation_is_unidirectional(operation)) {
 787                 if (!gb_operation_response_alloc(operation, 0, GFP_KERNEL))
 788                         return -ENOMEM;
 789         }
 790
 791         /* Record the result */
 792         if (!gb_operation_result_set(operation, errno)) {
 793                 dev_err(&connection->hd->dev, "request result already set\n");
 794                 return -EIO;    /* Shouldn't happen */
 795         }
 796
 797         /* Sender of request does not care about response. */
 798         if (gb_operation_is_unidirectional(operation))
 799                 return 0;
 800
 801         /* Reference will be dropped when message has been sent. */
 802         gb_operation_get(operation);
 803         ret = gb_operation_get_active(operation);
 804         if (ret)
 805                 goto err_put;
 806
 807         /* Fill in the response header and send it */
 808         operation->response->header->result = gb_operation_errno_map(errno);
 809
 810         ret = gb_message_send(operation->response, GFP_KERNEL);
 811         if (ret)
 812                 goto err_put_active;
 813
 814         return 0;
 815
 816 err_put_active:
 817         gb_operation_put_active(operation);
 818 err_put:
 819         gb_operation_put(operation);
 820
 821         return ret;
 822 }
 823
 824 /*
 825  * This function is called when a message send request has completed.
 826  */
 827 void greybus_message_sent(struct gb_host_device *hd,
 828                                         struct gb_message *message, int status)
 829 {
 830         struct gb_operation *operation = message->operation;
 831         struct gb_connection *connection = operation->connection;
 832
 833         /*
 834          * If the message was a response, we just need to drop our
 835          * reference to the operation.  If an error occurred, report
 836          * it.
 837          *
 838          * For requests, if there's no error and the operation in not
 839          * unidirectional, there's nothing more to do until the response
 840          * arrives. If an error occurred attempting to send it, or if the
 841          * operation is unidrectional, record the result of the operation and
 842          * schedule its completion.
 843          */
 844         if (message == operation->response) {
 845                 if (status) {
 846                         dev_err(&connection->hd->dev,
 847                                 "%s: error sending response 0x%02x: %d\n",
 848                                 connection->name, operation->type, status);
 849                 }
 850
 851                 gb_operation_put_active(operation);
 852                 gb_operation_put(operation);
 853         } else if (status || gb_operation_is_unidirectional(operation)) {
 854                 if (gb_operation_result_set(operation, status)) {
 855                         queue_work(gb_operation_completion_wq,
 856                                         &operation->work);
 857                 }
 858         }
 859 }
 860 EXPORT_SYMBOL_GPL(greybus_message_sent);
 861
 862 /*
 863  * We've received data on a connection, and it doesn't look like a
 864  * response, so we assume it's a request.
 865  *
 866  * This is called in interrupt context, so just copy the incoming
 867  * data into the request buffer and handle the rest via workqueue.
 868  */
 869 static void gb_connection_recv_request(struct gb_connection *connection,
 870                                        u16 operation_id, u8 type,
 871                                        void *data, size_t size)
 872 {
 873         struct gb_operation *operation;
 874         int ret;
 875
 876         operation = gb_operation_create_incoming(connection, operation_id,
 877                                                 type, data, size);
 878         if (!operation) {
 879                 dev_err(&connection->hd->dev,
 880                         "%s: can't create incoming operation\n",
 881                         connection->name);
 882                 return;
 883         }
 884
 885         ret = gb_operation_get_active(operation);
 886         if (ret) {
 887                 gb_operation_put(operation);
 888                 return;
 889         }
 890         trace_gb_message_recv_request(operation->request);
 891
 892         /*
 893          * The initial reference to the operation will be dropped when the
 894          * request handler returns.
 895          */
 896         if (gb_operation_result_set(operation, -EINPROGRESS))
 897                 queue_work(connection->wq, &operation->work);
 898 }
 899
 900 /*
 901  * We've received data that appears to be an operation response
 902  * message.  Look up the operation, and record that we've received
 903  * its response.
 904  *
 905  * This is called in interrupt context, so just copy the incoming
 906  * data into the response buffer and handle the rest via workqueue.
 907  */
 908 static void gb_connection_recv_response(struct gb_connection *connection,
 909                         u16 operation_id, u8 result, void *data, size_t size)
 910 {
 911         struct gb_operation_msg_hdr *header;
 912         struct gb_operation *operation;
 913         struct gb_message *message;
 914         int errno = gb_operation_status_map(result);
 915         size_t message_size;
 916
 917         if (!operation_id) {
 918                 dev_err_ratelimited(&connection->hd->dev,
 919                                 "%s: invalid response id 0 received\n",
 920                                 connection->name);
 921                 return;
 922         }
 923
 924         operation = gb_operation_find_outgoing(connection, operation_id);
 925         if (!operation) {
 926                 dev_err_ratelimited(&connection->hd->dev,
 927                                 "%s: unexpected response id 0x%04x received\n",
 928                                 connection->name, operation_id);
 929                 return;
 930         }
 931
 932         message = operation->response;
 933         header = message->header;
 934         message_size = sizeof(*header) + message->payload_size;
 935         if (!errno && size > message_size) {
 936                 dev_err_ratelimited(&connection->hd->dev,
 937                                 "%s: malformed response 0x%02x received (%zu > %zu)\n",
 938                                 connection->name, header->type,
 939                                 size, message_size);
 940                 errno = -EMSGSIZE;
 941         } else if (!errno && size < message_size) {
 942                 if (gb_operation_short_response_allowed(operation)) {
 943                         message->payload_size = size - sizeof(*header);
 944                 } else {
 945                         dev_err_ratelimited(&connection->hd->dev,
 946                                         "%s: short response 0x%02x received (%zu < %zu)\n",
 947                                         connection->name, header->type,
 948                                         size, message_size);
 949                         errno = -EMSGSIZE;
 950                 }
 951         }
 952         trace_gb_message_recv_response(operation->response);
 953
 954         /* We must ignore the payload if a bad status is returned */
 955         if (errno)
 956                 size = sizeof(*header);
 957
 958         /* The rest will be handled in work queue context */
 959         if (gb_operation_result_set(operation, errno)) {
 960                 memcpy(header, data, size);
 961                 queue_work(gb_operation_completion_wq, &operation->work);
 962         }
 963
 964         gb_operation_put(operation);
 965 }
 966
 967 /*
 968  * Handle data arriving on a connection.  As soon as we return the
 969  * supplied data buffer will be reused (so unless we do something
 970  * with, it's effectively dropped).
 971  */
 972 void gb_connection_recv(struct gb_connection *connection,
 973                                 void *data, size_t size)
 974 {
 975         struct gb_operation_msg_hdr header;
 976         struct device *dev = &connection->hd->dev;
 977         size_t msg_size;
 978         u16 operation_id;
 979
 980         if (connection->state == GB_CONNECTION_STATE_DISABLED ||
 981                         gb_connection_is_offloaded(connection)) {
 982                 dev_warn_ratelimited(dev, "%s: dropping %zu received bytes\n",
 983                                 connection->name, size);
 984                 return;
 985         }
 986
 987         if (size < sizeof(header)) {
 988                 dev_err_ratelimited(dev, "%s: short message received\n",
 989                                 connection->name);
 990                 return;
 991         }
 992
 993         /* Use memcpy as data may be unaligned */
 994         memcpy(&header, data, sizeof(header));
 995         msg_size = le16_to_cpu(header.size);
 996         if (size < msg_size) {
 997                 dev_err_ratelimited(dev,
 998                                 "%s: incomplete message 0x%04x of type 0x%02x received (%zu < %zu)\n",
 999                                 connection->name,
1000                                 le16_to_cpu(header.operation_id),
1001                                 header.type, size, msg_size);
1002                 return;         /* XXX Should still complete operation */
1003         }
1004
1005         operation_id = le16_to_cpu(header.operation_id);
1006         if (header.type & GB_MESSAGE_TYPE_RESPONSE)
1007                 gb_connection_recv_response(connection, operation_id,
1008                                                 header.result, data, msg_size);
1009         else
1010                 gb_connection_recv_request(connection, operation_id,
1011                                                 header.type, data, msg_size);
1012 }
1013
1014 /*
1015  * Cancel an outgoing operation synchronously, and record the given error to
1016  * indicate why.
1017  */
1018 void gb_operation_cancel(struct gb_operation *operation, int errno)
1019 {
1020         if (WARN_ON(gb_operation_is_incoming(operation)))
1021                 return;
1022
1023         if (gb_operation_result_set(operation, errno)) {
1024                 gb_message_cancel(operation->request);
1025                 queue_work(gb_operation_completion_wq, &operation->work);
1026         }
1027         trace_gb_message_cancel_outgoing(operation->request);
1028
1029         atomic_inc(&operation->waiters);
1030         wait_event(gb_operation_cancellation_queue,
1031                         !gb_operation_is_active(operation));
1032         atomic_dec(&operation->waiters);
1033 }
1034 EXPORT_SYMBOL_GPL(gb_operation_cancel);
1035
1036 /*
1037  * Cancel an incoming operation synchronously. Called during connection tear
1038  * down.
1039  */
1040 void gb_operation_cancel_incoming(struct gb_operation *operation, int errno)
1041 {
1042         if (WARN_ON(!gb_operation_is_incoming(operation)))
1043                 return;
1044
1045         if (!gb_operation_is_unidirectional(operation)) {
1046                 /*
1047                  * Make sure the request handler has submitted the response
1048                  * before cancelling it.
1049                  */
1050                 flush_work(&operation->work);
1051                 if (!gb_operation_result_set(operation, errno))
1052                         gb_message_cancel(operation->response);
1053         }
1054         trace_gb_message_cancel_incoming(operation->response);
1055
1056         atomic_inc(&operation->waiters);
1057         wait_event(gb_operation_cancellation_queue,
1058                         !gb_operation_is_active(operation));
1059         atomic_dec(&operation->waiters);
1060 }
1061
1062 /**
1063  * gb_operation_sync_timeout() - implement a "simple" synchronous operation
1064  * @connection: the Greybus connection to send this to
1065  * @type: the type of operation to send
1066  * @request: pointer to a memory buffer to copy the request from
1067  * @request_size: size of @request
1068  * @response: pointer to a memory buffer to copy the response to
1069  * @response_size: the size of @response.
1070  * @timeout: operation timeout in milliseconds
1071  *
1072  * This function implements a simple synchronous Greybus operation.  It sends
1073  * the provided operation request and waits (sleeps) until the corresponding
1074  * operation response message has been successfully received, or an error
1075  * occurs.  @request and @response are buffers to hold the request and response
1076  * data respectively, and if they are not NULL, their size must be specified in
1077  * @request_size and @response_size.
1078  *
1079  * If a response payload is to come back, and @response is not NULL,
1080  * @response_size number of bytes will be copied into @response if the operation
1081  * is successful.
1082  *
1083  * If there is an error, the response buffer is left alone.
1084  */
1085 int gb_operation_sync_timeout(struct gb_connection *connection, int type,
1086                                 void *request, int request_size,
1087                                 void *response, int response_size,
1088                                 unsigned int timeout)
1089 {
1090         struct gb_operation *operation;
1091         int ret;
1092
1093         if ((response_size && !response) ||
1094             (request_size && !request))
1095                 return -EINVAL;
1096
1097         operation = gb_operation_create(connection, type,
1098                                         request_size, response_size,
1099                                         GFP_KERNEL);
1100         if (!operation)
1101                 return -ENOMEM;
1102
1103         if (request_size)
1104                 memcpy(operation->request->payload, request, request_size);
1105
1106         ret = gb_operation_request_send_sync_timeout(operation, timeout);
1107         if (ret) {
1108                 dev_err(&connection->hd->dev,
1109                         "%s: synchronous operation of type 0x%02x failed: %d\n",
1110                         connection->name, type, ret);
1111         } else {
1112                 if (response_size) {
1113                         memcpy(response, operation->response->payload,
1114                                response_size);
1115                 }
1116         }
1117
1118         gb_operation_put(operation);
1119
1120         return ret;
1121 }
1122 EXPORT_SYMBOL_GPL(gb_operation_sync_timeout);
1123
1124 /**
1125  * gb_operation_unidirectional_timeout() - initiate a unidirectional operation
1126  * @connection:         connection to use
1127  * @type:               type of operation to send
1128  * @request:            memory buffer to copy the request from
1129  * @request_size:       size of @request
1130  * @timeout:            send timeout in milliseconds
1131  *
1132  * Initiate a unidirectional operation by sending a request message and
1133  * waiting for it to be acknowledged as sent by the host device.
1134  *
1135  * Note that successful send of a unidirectional operation does not imply that
1136  * the request as actually reached the remote end of the connection.
1137  */
1138 int gb_operation_unidirectional_timeout(struct gb_connection *connection,
1139                                 int type, void *request, int request_size,
1140                                 unsigned int timeout)
1141 {
1142         struct gb_operation *operation;
1143         int ret;
1144
1145         if (request_size && !request)
1146                 return -EINVAL;
1147
1148         operation = gb_operation_create_flags(connection, type,
1149                                         request_size, 0,
1150                                         GB_OPERATION_FLAG_UNIDIRECTIONAL,
1151                                         GFP_KERNEL);
1152         if (!operation)
1153                 return -ENOMEM;
1154
1155         if (request_size)
1156                 memcpy(operation->request->payload, request, request_size);
1157
1158         ret = gb_operation_request_send_sync_timeout(operation, timeout);
1159         if (ret) {
1160                 dev_err(&connection->hd->dev,
1161                         "%s: unidirectional operation of type 0x%02x failed: %d\n",
1162                         connection->name, type, ret);
1163         }
1164
1165         gb_operation_put(operation);
1166
1167         return ret;
1168 }
1169 EXPORT_SYMBOL_GPL(gb_operation_unidirectional_timeout);
1170
1171 int __init gb_operation_init(void)
1172 {
1173         gb_message_cache = kmem_cache_create("gb_message_cache",
1174                                 sizeof(struct gb_message), 0, 0, NULL);
1175         if (!gb_message_cache)
1176                 return -ENOMEM;
1177
1178         gb_operation_cache = kmem_cache_create("gb_operation_cache",
1179                                 sizeof(struct gb_operation), 0, 0, NULL);
1180         if (!gb_operation_cache)
1181                 goto err_destroy_message_cache;
1182
1183         gb_operation_completion_wq = alloc_workqueue("greybus_completion",
1184                                 0, 0);
1185         if (!gb_operation_completion_wq)
1186                 goto err_destroy_operation_cache;
1187
1188         return 0;
1189
1190 err_destroy_operation_cache:
1191         kmem_cache_destroy(gb_operation_cache);
1192         gb_operation_cache = NULL;
1193 err_destroy_message_cache:
1194         kmem_cache_destroy(gb_message_cache);
1195         gb_message_cache = NULL;
1196
1197         return -ENOMEM;
1198 }
1199
1200 void gb_operation_exit(void)
1201 {
1202         destroy_workqueue(gb_operation_completion_wq);
1203         gb_operation_completion_wq = NULL;
1204         kmem_cache_destroy(gb_operation_cache);
1205         gb_operation_cache = NULL;
1206         kmem_cache_destroy(gb_message_cache);
1207         gb_message_cache = NULL;
1208 }