net/rxrpc/ar-output.c

   1 /* RxRPC packet transmission
   2  *
   3  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #include <linux/net.h>
  13 #include <linux/gfp.h>
  14 #include <linux/skbuff.h>
  15 #include <linux/circ_buf.h>
  16 #include <linux/export.h>
  17 #include <net/sock.h>
  18 #include <net/af_rxrpc.h>
  19 #include "ar-internal.h"
  20
  21 /*
  22  * Time till packet resend (in jiffies).
  23  */
  24 unsigned rxrpc_resend_timeout = 4 * HZ;
  25
  26 static int rxrpc_send_data(struct kiocb *iocb,
  27                            struct rxrpc_sock *rx,
  28                            struct rxrpc_call *call,
  29                            struct msghdr *msg, size_t len);
  30
  31 /*
  32  * extract control messages from the sendmsg() control buffer
  33  */
  34 static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
  35                               unsigned long *user_call_ID,
  36                               enum rxrpc_command *command,
  37                               u32 *abort_code,
  38                               bool server)
  39 {
  40         struct cmsghdr *cmsg;
  41         int len;
  42
  43         *command = RXRPC_CMD_SEND_DATA;
  44
  45         if (msg->msg_controllen == 0)
  46                 return -EINVAL;
  47
  48         for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
  49                 if (!CMSG_OK(msg, cmsg))
  50                         return -EINVAL;
  51
  52                 len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
  53                 _debug("CMSG %d, %d, %d",
  54                        cmsg->cmsg_level, cmsg->cmsg_type, len);
  55
  56                 if (cmsg->cmsg_level != SOL_RXRPC)
  57                         continue;
  58
  59                 switch (cmsg->cmsg_type) {
  60                 case RXRPC_USER_CALL_ID:
  61                         if (msg->msg_flags & MSG_CMSG_COMPAT) {
  62                                 if (len != sizeof(u32))
  63                                         return -EINVAL;
  64                                 *user_call_ID = *(u32 *) CMSG_DATA(cmsg);
  65                         } else {
  66                                 if (len != sizeof(unsigned long))
  67                                         return -EINVAL;
  68                                 *user_call_ID = *(unsigned long *)
  69                                         CMSG_DATA(cmsg);
  70                         }
  71                         _debug("User Call ID %lx", *user_call_ID);
  72                         break;
  73
  74                 case RXRPC_ABORT:
  75                         if (*command != RXRPC_CMD_SEND_DATA)
  76                                 return -EINVAL;
  77                         *command = RXRPC_CMD_SEND_ABORT;
  78                         if (len != sizeof(*abort_code))
  79                                 return -EINVAL;
  80                         *abort_code = *(unsigned int *) CMSG_DATA(cmsg);
  81                         _debug("Abort %x", *abort_code);
  82                         if (*abort_code == 0)
  83                                 return -EINVAL;
  84                         break;
  85
  86                 case RXRPC_ACCEPT:
  87                         if (*command != RXRPC_CMD_SEND_DATA)
  88                                 return -EINVAL;
  89                         *command = RXRPC_CMD_ACCEPT;
  90                         if (len != 0)
  91                                 return -EINVAL;
  92                         if (!server)
  93                                 return -EISCONN;
  94                         break;
  95
  96                 default:
  97                         return -EINVAL;
  98                 }
  99         }
 100
 101         _leave(" = 0");
 102         return 0;
 103 }
 104
 105 /*
 106  * abort a call, sending an ABORT packet to the peer
 107  */
 108 static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
 109 {
 110         write_lock_bh(&call->state_lock);
 111
 112         if (call->state <= RXRPC_CALL_COMPLETE) {
 113                 call->state = RXRPC_CALL_LOCALLY_ABORTED;
 114                 call->abort_code = abort_code;
 115                 set_bit(RXRPC_CALL_ABORT, &call->events);
 116                 del_timer_sync(&call->resend_timer);
 117                 del_timer_sync(&call->ack_timer);
 118                 clear_bit(RXRPC_CALL_RESEND_TIMER, &call->events);
 119                 clear_bit(RXRPC_CALL_ACK, &call->events);
 120                 clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
 121                 rxrpc_queue_call(call);
 122         }
 123
 124         write_unlock_bh(&call->state_lock);
 125 }
 126
 127 /*
 128  * send a message forming part of a client call through an RxRPC socket
 129  * - caller holds the socket locked
 130  * - the socket may be either a client socket or a server socket
 131  */
 132 int rxrpc_client_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx,
 133                          struct rxrpc_transport *trans, struct msghdr *msg,
 134                          size_t len)
 135 {
 136         struct rxrpc_conn_bundle *bundle;
 137         enum rxrpc_command cmd;
 138         struct rxrpc_call *call;
 139         unsigned long user_call_ID = 0;
 140         struct key *key;
 141         __be16 service_id;
 142         u32 abort_code = 0;
 143         int ret;
 144
 145         _enter("");
 146
 147         ASSERT(trans != NULL);
 148
 149         ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
 150                                  false);
 151         if (ret < 0)
 152                 return ret;
 153
 154         bundle = NULL;
 155         if (trans) {
 156                 service_id = rx->service_id;
 157                 if (msg->msg_name) {
 158                         DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
 159                                          msg->msg_name);
 160                         service_id = htons(srx->srx_service);
 161                 }
 162                 key = rx->key;
 163                 if (key && !rx->key->payload.data)
 164                         key = NULL;
 165                 bundle = rxrpc_get_bundle(rx, trans, key, service_id,
 166                                           GFP_KERNEL);
 167                 if (IS_ERR(bundle))
 168                         return PTR_ERR(bundle);
 169         }
 170
 171         call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
 172                                      abort_code == 0, GFP_KERNEL);
 173         if (trans)
 174                 rxrpc_put_bundle(trans, bundle);
 175         if (IS_ERR(call)) {
 176                 _leave(" = %ld", PTR_ERR(call));
 177                 return PTR_ERR(call);
 178         }
 179
 180         _debug("CALL %d USR %lx ST %d on CONN %p",
 181                call->debug_id, call->user_call_ID, call->state, call->conn);
 182
 183         if (call->state >= RXRPC_CALL_COMPLETE) {
 184                 /* it's too late for this call */
 185                 ret = -ESHUTDOWN;
 186         } else if (cmd == RXRPC_CMD_SEND_ABORT) {
 187                 rxrpc_send_abort(call, abort_code);
 188         } else if (cmd != RXRPC_CMD_SEND_DATA) {
 189                 ret = -EINVAL;
 190         } else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
 191                 /* request phase complete for this client call */
 192                 ret = -EPROTO;
 193         } else {
 194                 ret = rxrpc_send_data(iocb, rx, call, msg, len);
 195         }
 196
 197         rxrpc_put_call(call);
 198         _leave(" = %d", ret);
 199         return ret;
 200 }
 201
 202 /**
 203  * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 204  * @call: The call to send data through
 205  * @msg: The data to send
 206  * @len: The amount of data to send
 207  *
 208  * Allow a kernel service to send data on a call.  The call must be in an state
 209  * appropriate to sending data.  No control data should be supplied in @msg,
 210  * nor should an address be supplied.  MSG_MORE should be flagged if there's
 211  * more data to come, otherwise this data will end the transmission phase.
 212  */
 213 int rxrpc_kernel_send_data(struct rxrpc_call *call, struct msghdr *msg,
 214                            size_t len)
 215 {
 216         int ret;
 217
 218         _enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);
 219
 220         ASSERTCMP(msg->msg_name, ==, NULL);
 221         ASSERTCMP(msg->msg_control, ==, NULL);
 222
 223         lock_sock(&call->socket->sk);
 224
 225         _debug("CALL %d USR %lx ST %d on CONN %p",
 226                call->debug_id, call->user_call_ID, call->state, call->conn);
 227
 228         if (call->state >= RXRPC_CALL_COMPLETE) {
 229                 ret = -ESHUTDOWN; /* it's too late for this call */
 230         } else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
 231                    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
 232                    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
 233                 ret = -EPROTO; /* request phase complete for this client call */
 234         } else {
 235                 mm_segment_t oldfs = get_fs();
 236                 set_fs(KERNEL_DS);
 237                 ret = rxrpc_send_data(NULL, call->socket, call, msg, len);
 238                 set_fs(oldfs);
 239         }
 240
 241         release_sock(&call->socket->sk);
 242         _leave(" = %d", ret);
 243         return ret;
 244 }
 245
 246 EXPORT_SYMBOL(rxrpc_kernel_send_data);
 247
 248 /**
 249  * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 250  * @call: The call to be aborted
 251  * @abort_code: The abort code to stick into the ABORT packet
 252  *
 253  * Allow a kernel service to abort a call, if it's still in an abortable state.
 254  */
 255 void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
 256 {
 257         _enter("{%d},%d", call->debug_id, abort_code);
 258
 259         lock_sock(&call->socket->sk);
 260
 261         _debug("CALL %d USR %lx ST %d on CONN %p",
 262                call->debug_id, call->user_call_ID, call->state, call->conn);
 263
 264         if (call->state < RXRPC_CALL_COMPLETE)
 265                 rxrpc_send_abort(call, abort_code);
 266
 267         release_sock(&call->socket->sk);
 268         _leave("");
 269 }
 270
 271 EXPORT_SYMBOL(rxrpc_kernel_abort_call);
 272
 273 /*
 274  * send a message through a server socket
 275  * - caller holds the socket locked
 276  */
 277 int rxrpc_server_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx,
 278                          struct msghdr *msg, size_t len)
 279 {
 280         enum rxrpc_command cmd;
 281         struct rxrpc_call *call;
 282         unsigned long user_call_ID = 0;
 283         u32 abort_code = 0;
 284         int ret;
 285
 286         _enter("");
 287
 288         ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
 289                                  true);
 290         if (ret < 0)
 291                 return ret;
 292
 293         if (cmd == RXRPC_CMD_ACCEPT) {
 294                 call = rxrpc_accept_call(rx, user_call_ID);
 295                 if (IS_ERR(call))
 296                         return PTR_ERR(call);
 297                 rxrpc_put_call(call);
 298                 return 0;
 299         }
 300
 301         call = rxrpc_find_server_call(rx, user_call_ID);
 302         if (!call)
 303                 return -EBADSLT;
 304         if (call->state >= RXRPC_CALL_COMPLETE) {
 305                 ret = -ESHUTDOWN;
 306                 goto out;
 307         }
 308
 309         switch (cmd) {
 310         case RXRPC_CMD_SEND_DATA:
 311                 if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
 312                     call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
 313                     call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
 314                         /* Tx phase not yet begun for this call */
 315                         ret = -EPROTO;
 316                         break;
 317                 }
 318
 319                 ret = rxrpc_send_data(iocb, rx, call, msg, len);
 320                 break;
 321
 322         case RXRPC_CMD_SEND_ABORT:
 323                 rxrpc_send_abort(call, abort_code);
 324                 break;
 325         default:
 326                 BUG();
 327         }
 328
 329         out:
 330         rxrpc_put_call(call);
 331         _leave(" = %d", ret);
 332         return ret;
 333 }
 334
 335 /*
 336  * send a packet through the transport endpoint
 337  */
 338 int rxrpc_send_packet(struct rxrpc_transport *trans, struct sk_buff *skb)
 339 {
 340         struct kvec iov[1];
 341         struct msghdr msg;
 342         int ret, opt;
 343
 344         _enter(",{%d}", skb->len);
 345
 346         iov[0].iov_base = skb->head;
 347         iov[0].iov_len = skb->len;
 348
 349         msg.msg_name = &trans->peer->srx.transport.sin;
 350         msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
 351         msg.msg_control = NULL;
 352         msg.msg_controllen = 0;
 353         msg.msg_flags = 0;
 354
 355         /* send the packet with the don't fragment bit set if we currently
 356          * think it's small enough */
 357         if (skb->len - sizeof(struct rxrpc_header) < trans->peer->maxdata) {
 358                 down_read(&trans->local->defrag_sem);
 359                 /* send the packet by UDP
 360                  * - returns -EMSGSIZE if UDP would have to fragment the packet
 361                  *   to go out of the interface
 362                  *   - in which case, we'll have processed the ICMP error
 363                  *     message and update the peer record
 364                  */
 365                 ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
 366                                      iov[0].iov_len);
 367
 368                 up_read(&trans->local->defrag_sem);
 369                 if (ret == -EMSGSIZE)
 370                         goto send_fragmentable;
 371
 372                 _leave(" = %d [%u]", ret, trans->peer->maxdata);
 373                 return ret;
 374         }
 375
 376 send_fragmentable:
 377         /* attempt to send this message with fragmentation enabled */
 378         _debug("send fragment");
 379
 380         down_write(&trans->local->defrag_sem);
 381         opt = IP_PMTUDISC_DONT;
 382         ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
 383                                 (char *) &opt, sizeof(opt));
 384         if (ret == 0) {
 385                 ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
 386                                      iov[0].iov_len);
 387
 388                 opt = IP_PMTUDISC_DO;
 389                 kernel_setsockopt(trans->local->socket, SOL_IP,
 390                                   IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
 391         }
 392
 393         up_write(&trans->local->defrag_sem);
 394         _leave(" = %d [frag %u]", ret, trans->peer->maxdata);
 395         return ret;
 396 }
 397
 398 /*
 399  * wait for space to appear in the transmit/ACK window
 400  * - caller holds the socket locked
 401  */
 402 static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
 403                                     struct rxrpc_call *call,
 404                                     long *timeo)
 405 {
 406         DECLARE_WAITQUEUE(myself, current);
 407         int ret;
 408
 409         _enter(",{%d},%ld",
 410                CIRC_SPACE(call->acks_head, call->acks_tail, call->acks_winsz),
 411                *timeo);
 412
 413         add_wait_queue(&call->tx_waitq, &myself);
 414
 415         for (;;) {
 416                 set_current_state(TASK_INTERRUPTIBLE);
 417                 ret = 0;
 418                 if (CIRC_SPACE(call->acks_head, call->acks_tail,
 419                                call->acks_winsz) > 0)
 420                         break;
 421                 if (signal_pending(current)) {
 422                         ret = sock_intr_errno(*timeo);
 423                         break;
 424                 }
 425
 426                 release_sock(&rx->sk);
 427                 *timeo = schedule_timeout(*timeo);
 428                 lock_sock(&rx->sk);
 429         }
 430
 431         remove_wait_queue(&call->tx_waitq, &myself);
 432         set_current_state(TASK_RUNNING);
 433         _leave(" = %d", ret);
 434         return ret;
 435 }
 436
 437 /*
 438  * attempt to schedule an instant Tx resend
 439  */
 440 static inline void rxrpc_instant_resend(struct rxrpc_call *call)
 441 {
 442         read_lock_bh(&call->state_lock);
 443         if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
 444                 clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
 445                 if (call->state < RXRPC_CALL_COMPLETE &&
 446                     !test_and_set_bit(RXRPC_CALL_RESEND_TIMER, &call->events))
 447                         rxrpc_queue_call(call);
 448         }
 449         read_unlock_bh(&call->state_lock);
 450 }
 451
 452 /*
 453  * queue a packet for transmission, set the resend timer and attempt
 454  * to send the packet immediately
 455  */
 456 static void rxrpc_queue_packet(struct rxrpc_call *call, struct sk_buff *skb,
 457                                bool last)
 458 {
 459         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 460         int ret;
 461
 462         _net("queue skb %p [%d]", skb, call->acks_head);
 463
 464         ASSERT(call->acks_window != NULL);
 465         call->acks_window[call->acks_head] = (unsigned long) skb;
 466         smp_wmb();
 467         call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);
 468
 469         if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
 470                 _debug("________awaiting reply/ACK__________");
 471                 write_lock_bh(&call->state_lock);
 472                 switch (call->state) {
 473                 case RXRPC_CALL_CLIENT_SEND_REQUEST:
 474                         call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
 475                         break;
 476                 case RXRPC_CALL_SERVER_ACK_REQUEST:
 477                         call->state = RXRPC_CALL_SERVER_SEND_REPLY;
 478                         if (!last)
 479                                 break;
 480                 case RXRPC_CALL_SERVER_SEND_REPLY:
 481                         call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
 482                         break;
 483                 default:
 484                         break;
 485                 }
 486                 write_unlock_bh(&call->state_lock);
 487         }
 488
 489         _proto("Tx DATA %%%u { #%u }",
 490                ntohl(sp->hdr.serial), ntohl(sp->hdr.seq));
 491
 492         sp->need_resend = false;
 493         sp->resend_at = jiffies + rxrpc_resend_timeout;
 494         if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
 495                 _debug("run timer");
 496                 call->resend_timer.expires = sp->resend_at;
 497                 add_timer(&call->resend_timer);
 498         }
 499
 500         /* attempt to cancel the rx-ACK timer, deferring reply transmission if
 501          * we're ACK'ing the request phase of an incoming call */
 502         ret = -EAGAIN;
 503         if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
 504                 /* the packet may be freed by rxrpc_process_call() before this
 505                  * returns */
 506                 ret = rxrpc_send_packet(call->conn->trans, skb);
 507                 _net("sent skb %p", skb);
 508         } else {
 509                 _debug("failed to delete ACK timer");
 510         }
 511
 512         if (ret < 0) {
 513                 _debug("need instant resend %d", ret);
 514                 sp->need_resend = true;
 515                 rxrpc_instant_resend(call);
 516         }
 517
 518         _leave("");
 519 }
 520
 521 /*
 522  * send data through a socket
 523  * - must be called in process context
 524  * - caller holds the socket locked
 525  */
 526 static int rxrpc_send_data(struct kiocb *iocb,
 527                            struct rxrpc_sock *rx,
 528                            struct rxrpc_call *call,
 529                            struct msghdr *msg, size_t len)
 530 {
 531         struct rxrpc_skb_priv *sp;
 532         unsigned char __user *from;
 533         struct sk_buff *skb;
 534         const struct iovec *iov;
 535         struct sock *sk = &rx->sk;
 536         long timeo;
 537         bool more;
 538         int ret, ioc, segment, copied;
 539
 540         timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 541
 542         /* this should be in poll */
 543         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 544
 545         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 546                 return -EPIPE;
 547
 548         iov = msg->msg_iter.iov;
 549         ioc = msg->msg_iter.nr_segs - 1;
 550         from = iov->iov_base;
 551         segment = iov->iov_len;
 552         iov++;
 553         more = msg->msg_flags & MSG_MORE;
 554
 555         skb = call->tx_pending;
 556         call->tx_pending = NULL;
 557
 558         copied = 0;
 559         do {
 560                 int copy;
 561
 562                 if (segment > len)
 563                         segment = len;
 564
 565                 _debug("SEGMENT %d @%p", segment, from);
 566
 567                 if (!skb) {
 568                         size_t size, chunk, max, space;
 569
 570                         _debug("alloc");
 571
 572                         if (CIRC_SPACE(call->acks_head, call->acks_tail,
 573                                        call->acks_winsz) <= 0) {
 574                                 ret = -EAGAIN;
 575                                 if (msg->msg_flags & MSG_DONTWAIT)
 576                                         goto maybe_error;
 577                                 ret = rxrpc_wait_for_tx_window(rx, call,
 578                                                                &timeo);
 579                                 if (ret < 0)
 580                                         goto maybe_error;
 581                         }
 582
 583                         max = call->conn->trans->peer->maxdata;
 584                         max -= call->conn->security_size;
 585                         max &= ~(call->conn->size_align - 1UL);
 586
 587                         chunk = max;
 588                         if (chunk > len && !more)
 589                                 chunk = len;
 590
 591                         space = chunk + call->conn->size_align;
 592                         space &= ~(call->conn->size_align - 1UL);
 593
 594                         size = space + call->conn->header_size;
 595
 596                         _debug("SIZE: %zu/%zu/%zu", chunk, space, size);
 597
 598                         /* create a buffer that we can retain until it's ACK'd */
 599                         skb = sock_alloc_send_skb(
 600                                 sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
 601                         if (!skb)
 602                                 goto maybe_error;
 603
 604                         rxrpc_new_skb(skb);
 605
 606                         _debug("ALLOC SEND %p", skb);
 607
 608                         ASSERTCMP(skb->mark, ==, 0);
 609
 610                         _debug("HS: %u", call->conn->header_size);
 611                         skb_reserve(skb, call->conn->header_size);
 612                         skb->len += call->conn->header_size;
 613
 614                         sp = rxrpc_skb(skb);
 615                         sp->remain = chunk;
 616                         if (sp->remain > skb_tailroom(skb))
 617                                 sp->remain = skb_tailroom(skb);
 618
 619                         _net("skb: hr %d, tr %d, hl %d, rm %d",
 620                                skb_headroom(skb),
 621                                skb_tailroom(skb),
 622                                skb_headlen(skb),
 623                                sp->remain);
 624
 625                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 626                 }
 627
 628                 _debug("append");
 629                 sp = rxrpc_skb(skb);
 630
 631                 /* append next segment of data to the current buffer */
 632                 copy = skb_tailroom(skb);
 633                 ASSERTCMP(copy, >, 0);
 634                 if (copy > segment)
 635                         copy = segment;
 636                 if (copy > sp->remain)
 637                         copy = sp->remain;
 638
 639                 _debug("add");
 640                 ret = skb_add_data(skb, from, copy);
 641                 _debug("added");
 642                 if (ret < 0)
 643                         goto efault;
 644                 sp->remain -= copy;
 645                 skb->mark += copy;
 646                 copied += copy;
 647
 648                 len -= copy;
 649                 segment -= copy;
 650                 from += copy;
 651                 while (segment == 0 && ioc > 0) {
 652                         from = iov->iov_base;
 653                         segment = iov->iov_len;
 654                         iov++;
 655                         ioc--;
 656                 }
 657                 if (len == 0) {
 658                         segment = 0;
 659                         ioc = 0;
 660                 }
 661
 662                 /* check for the far side aborting the call or a network error
 663                  * occurring */
 664                 if (call->state > RXRPC_CALL_COMPLETE)
 665                         goto call_aborted;
 666
 667                 /* add the packet to the send queue if it's now full */
 668                 if (sp->remain <= 0 || (segment == 0 && !more)) {
 669                         struct rxrpc_connection *conn = call->conn;
 670                         uint32_t seq;
 671                         size_t pad;
 672
 673                         /* pad out if we're using security */
 674                         if (conn->security) {
 675                                 pad = conn->security_size + skb->mark;
 676                                 pad = conn->size_align - pad;
 677                                 pad &= conn->size_align - 1;
 678                                 _debug("pad %zu", pad);
 679                                 if (pad)
 680                                         memset(skb_put(skb, pad), 0, pad);
 681                         }
 682
 683                         seq = atomic_inc_return(&call->sequence);
 684
 685                         sp->hdr.epoch = conn->epoch;
 686                         sp->hdr.cid = call->cid;
 687                         sp->hdr.callNumber = call->call_id;
 688                         sp->hdr.seq = htonl(seq);
 689                         sp->hdr.serial =
 690                                 htonl(atomic_inc_return(&conn->serial));
 691                         sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
 692                         sp->hdr.userStatus = 0;
 693                         sp->hdr.securityIndex = conn->security_ix;
 694                         sp->hdr._rsvd = 0;
 695                         sp->hdr.serviceId = conn->service_id;
 696
 697                         sp->hdr.flags = conn->out_clientflag;
 698                         if (len == 0 && !more)
 699                                 sp->hdr.flags |= RXRPC_LAST_PACKET;
 700                         else if (CIRC_SPACE(call->acks_head, call->acks_tail,
 701                                             call->acks_winsz) > 1)
 702                                 sp->hdr.flags |= RXRPC_MORE_PACKETS;
 703                         if (more && seq & 1)
 704                                 sp->hdr.flags |= RXRPC_REQUEST_ACK;
 705
 706                         ret = rxrpc_secure_packet(
 707                                 call, skb, skb->mark,
 708                                 skb->head + sizeof(struct rxrpc_header));
 709                         if (ret < 0)
 710                                 goto out;
 711
 712                         memcpy(skb->head, &sp->hdr,
 713                                sizeof(struct rxrpc_header));
 714                         rxrpc_queue_packet(call, skb, segment == 0 && !more);
 715                         skb = NULL;
 716                 }
 717
 718         } while (segment > 0);
 719
 720 success:
 721         ret = copied;
 722 out:
 723         call->tx_pending = skb;
 724         _leave(" = %d", ret);
 725         return ret;
 726
 727 call_aborted:
 728         rxrpc_free_skb(skb);
 729         if (call->state == RXRPC_CALL_NETWORK_ERROR)
 730                 ret = call->conn->trans->peer->net_error;
 731         else
 732                 ret = -ECONNABORTED;
 733         _leave(" = %d", ret);
 734         return ret;
 735
 736 maybe_error:
 737         if (copied)
 738                 goto success;
 739         goto out;
 740
 741 efault:
 742         ret = -EFAULT;
 743         goto out;
 744 }