drivers/vhost/net.c

   1 /* Copyright (C) 2009 Red Hat, Inc.
   2  * Author: Michael S. Tsirkin <mst@redhat.com>
   3  *
   4  * This work is licensed under the terms of the GNU GPL, version 2.
   5  *
   6  * virtio-net server in host kernel.
   7  */
   8
   9 #include <linux/compat.h>
  10 #include <linux/eventfd.h>
  11 #include <linux/vhost.h>
  12 #include <linux/virtio_net.h>
  13 #include <linux/miscdevice.h>
  14 #include <linux/module.h>
  15 #include <linux/moduleparam.h>
  16 #include <linux/mutex.h>
  17 #include <linux/workqueue.h>
  18 #include <linux/file.h>
  19 #include <linux/slab.h>
  20 #include <linux/vmalloc.h>
  21
  22 #include <linux/net.h>
  23 #include <linux/if_packet.h>
  24 #include <linux/if_arp.h>
  25 #include <linux/if_tun.h>
  26 #include <linux/if_macvlan.h>
  27 #include <linux/if_vlan.h>
  28
  29 #include <net/sock.h>
  30
  31 #include "vhost.h"
  32
  33 static int experimental_zcopytx = 1;
  34 module_param(experimental_zcopytx, int, 0444);
  35 MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
  36                                        " 1 -Enable; 0 - Disable");
  37
  38 /* Max number of bytes transferred before requeueing the job.
  39  * Using this limit prevents one virtqueue from starving others. */
  40 #define VHOST_NET_WEIGHT 0x80000
  41
  42 /* MAX number of TX used buffers for outstanding zerocopy */
  43 #define VHOST_MAX_PEND 128
  44 #define VHOST_GOODCOPY_LEN 256
  45
  46 /*
  47  * For transmit, used buffer len is unused; we override it to track buffer
  48  * status internally; used for zerocopy tx only.
  49  */
  50 /* Lower device DMA failed */
  51 #define VHOST_DMA_FAILED_LEN    ((__force __virtio32)3)
  52 /* Lower device DMA done */
  53 #define VHOST_DMA_DONE_LEN      ((__force __virtio32)2)
  54 /* Lower device DMA in progress */
  55 #define VHOST_DMA_IN_PROGRESS   ((__force __virtio32)1)
  56 /* Buffer unused */
  57 #define VHOST_DMA_CLEAR_LEN     ((__force __virtio32)0)
  58
  59 #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
  60
  61 enum {
  62         VHOST_NET_FEATURES = VHOST_FEATURES |
  63                          (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
  64                          (1ULL << VIRTIO_NET_F_MRG_RXBUF)
  65 };
  66
  67 enum {
  68         VHOST_NET_VQ_RX = 0,
  69         VHOST_NET_VQ_TX = 1,
  70         VHOST_NET_VQ_MAX = 2,
  71 };
  72
  73 struct vhost_net_ubuf_ref {
  74         /* refcount follows semantics similar to kref:
  75          *  0: object is released
  76          *  1: no outstanding ubufs
  77          * >1: outstanding ubufs
  78          */
  79         atomic_t refcount;
  80         wait_queue_head_t wait;
  81         struct vhost_virtqueue *vq;
  82 };
  83
  84 struct vhost_net_virtqueue {
  85         struct vhost_virtqueue vq;
  86         size_t vhost_hlen;
  87         size_t sock_hlen;
  88         /* vhost zerocopy support fields below: */
  89         /* last used idx for outstanding DMA zerocopy buffers */
  90         int upend_idx;
  91         /* first used idx for DMA done zerocopy buffers */
  92         int done_idx;
  93         /* an array of userspace buffers info */
  94         struct ubuf_info *ubuf_info;
  95         /* Reference counting for outstanding ubufs.
  96          * Protected by vq mutex. Writers must also take device mutex. */
  97         struct vhost_net_ubuf_ref *ubufs;
  98 };
  99
 100 struct vhost_net {
 101         struct vhost_dev dev;
 102         struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
 103         struct vhost_poll poll[VHOST_NET_VQ_MAX];
 104         /* Number of TX recently submitted.
 105          * Protected by tx vq lock. */
 106         unsigned tx_packets;
 107         /* Number of times zerocopy TX recently failed.
 108          * Protected by tx vq lock. */
 109         unsigned tx_zcopy_err;
 110         /* Flush in progress. Protected by tx vq lock. */
 111         bool tx_flush;
 112 };
 113
 114 static unsigned vhost_net_zcopy_mask __read_mostly;
 115
 116 static void vhost_net_enable_zcopy(int vq)
 117 {
 118         vhost_net_zcopy_mask |= 0x1 << vq;
 119 }
 120
 121 static struct vhost_net_ubuf_ref *
 122 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
 123 {
 124         struct vhost_net_ubuf_ref *ubufs;
 125         /* No zero copy backend? Nothing to count. */
 126         if (!zcopy)
 127                 return NULL;
 128         ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
 129         if (!ubufs)
 130                 return ERR_PTR(-ENOMEM);
 131         atomic_set(&ubufs->refcount, 1);
 132         init_waitqueue_head(&ubufs->wait);
 133         ubufs->vq = vq;
 134         return ubufs;
 135 }
 136
 137 static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
 138 {
 139         int r = atomic_sub_return(1, &ubufs->refcount);
 140         if (unlikely(!r))
 141                 wake_up(&ubufs->wait);
 142         return r;
 143 }
 144
 145 static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
 146 {
 147         vhost_net_ubuf_put(ubufs);
 148         wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
 149 }
 150
 151 static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
 152 {
 153         vhost_net_ubuf_put_and_wait(ubufs);
 154         kfree(ubufs);
 155 }
 156
 157 static void vhost_net_clear_ubuf_info(struct vhost_net *n)
 158 {
 159         int i;
 160
 161         for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
 162                 kfree(n->vqs[i].ubuf_info);
 163                 n->vqs[i].ubuf_info = NULL;
 164         }
 165 }
 166
 167 static int vhost_net_set_ubuf_info(struct vhost_net *n)
 168 {
 169         bool zcopy;
 170         int i;
 171
 172         for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
 173                 zcopy = vhost_net_zcopy_mask & (0x1 << i);
 174                 if (!zcopy)
 175                         continue;
 176                 n->vqs[i].ubuf_info = kmalloc(sizeof(*n->vqs[i].ubuf_info) *
 177                                               UIO_MAXIOV, GFP_KERNEL);
 178                 if  (!n->vqs[i].ubuf_info)
 179                         goto err;
 180         }
 181         return 0;
 182
 183 err:
 184         vhost_net_clear_ubuf_info(n);
 185         return -ENOMEM;
 186 }
 187
 188 static void vhost_net_vq_reset(struct vhost_net *n)
 189 {
 190         int i;
 191
 192         vhost_net_clear_ubuf_info(n);
 193
 194         for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
 195                 n->vqs[i].done_idx = 0;
 196                 n->vqs[i].upend_idx = 0;
 197                 n->vqs[i].ubufs = NULL;
 198                 n->vqs[i].vhost_hlen = 0;
 199                 n->vqs[i].sock_hlen = 0;
 200         }
 201
 202 }
 203
 204 static void vhost_net_tx_packet(struct vhost_net *net)
 205 {
 206         ++net->tx_packets;
 207         if (net->tx_packets < 1024)
 208                 return;
 209         net->tx_packets = 0;
 210         net->tx_zcopy_err = 0;
 211 }
 212
 213 static void vhost_net_tx_err(struct vhost_net *net)
 214 {
 215         ++net->tx_zcopy_err;
 216 }
 217
 218 static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
 219 {
 220         /* TX flush waits for outstanding DMAs to be done.
 221          * Don't start new DMAs.
 222          */
 223         return !net->tx_flush &&
 224                 net->tx_packets / 64 >= net->tx_zcopy_err;
 225 }
 226
 227 static bool vhost_sock_zcopy(struct socket *sock)
 228 {
 229         return unlikely(experimental_zcopytx) &&
 230                 sock_flag(sock->sk, SOCK_ZEROCOPY);
 231 }
 232
 233 /* In case of DMA done not in order in lower device driver for some reason.
 234  * upend_idx is used to track end of used idx, done_idx is used to track head
 235  * of used idx. Once lower device DMA done contiguously, we will signal KVM
 236  * guest used idx.
 237  */
 238 static void vhost_zerocopy_signal_used(struct vhost_net *net,
 239                                        struct vhost_virtqueue *vq)
 240 {
 241         struct vhost_net_virtqueue *nvq =
 242                 container_of(vq, struct vhost_net_virtqueue, vq);
 243         int i, add;
 244         int j = 0;
 245
 246         for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
 247                 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
 248                         vhost_net_tx_err(net);
 249                 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
 250                         vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
 251                         ++j;
 252                 } else
 253                         break;
 254         }
 255         while (j) {
 256                 add = min(UIO_MAXIOV - nvq->done_idx, j);
 257                 vhost_add_used_and_signal_n(vq->dev, vq,
 258                                             &vq->heads[nvq->done_idx], add);
 259                 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
 260                 j -= add;
 261         }
 262 }
 263
 264 static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success)
 265 {
 266         struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
 267         struct vhost_virtqueue *vq = ubufs->vq;
 268         int cnt;
 269
 270         rcu_read_lock_bh();
 271
 272         /* set len to mark this desc buffers done DMA */
 273         vq->heads[ubuf->desc].len = success ?
 274                 VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
 275         cnt = vhost_net_ubuf_put(ubufs);
 276
 277         /*
 278          * Trigger polling thread if guest stopped submitting new buffers:
 279          * in this case, the refcount after decrement will eventually reach 1.
 280          * We also trigger polling periodically after each 16 packets
 281          * (the value 16 here is more or less arbitrary, it's tuned to trigger
 282          * less than 10% of times).
 283          */
 284         if (cnt <= 1 || !(cnt % 16))
 285                 vhost_poll_queue(&vq->poll);
 286
 287         rcu_read_unlock_bh();
 288 }
 289
 290 static inline unsigned long busy_clock(void)
 291 {
 292         return local_clock() >> 10;
 293 }
 294
 295 static bool vhost_can_busy_poll(struct vhost_dev *dev,
 296                                 unsigned long endtime)
 297 {
 298         return likely(!need_resched()) &&
 299                likely(!time_after(busy_clock(), endtime)) &&
 300                likely(!signal_pending(current)) &&
 301                !vhost_has_work(dev);
 302 }
 303
 304 static void vhost_net_disable_vq(struct vhost_net *n,
 305                                  struct vhost_virtqueue *vq)
 306 {
 307         struct vhost_net_virtqueue *nvq =
 308                 container_of(vq, struct vhost_net_virtqueue, vq);
 309         struct vhost_poll *poll = n->poll + (nvq - n->vqs);
 310         if (!vq->private_data)
 311                 return;
 312         vhost_poll_stop(poll);
 313 }
 314
 315 static int vhost_net_enable_vq(struct vhost_net *n,
 316                                 struct vhost_virtqueue *vq)
 317 {
 318         struct vhost_net_virtqueue *nvq =
 319                 container_of(vq, struct vhost_net_virtqueue, vq);
 320         struct vhost_poll *poll = n->poll + (nvq - n->vqs);
 321         struct socket *sock;
 322
 323         sock = vq->private_data;
 324         if (!sock)
 325                 return 0;
 326
 327         return vhost_poll_start(poll, sock->file);
 328 }
 329
 330 static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
 331                                     struct vhost_virtqueue *vq,
 332                                     struct iovec iov[], unsigned int iov_size,
 333                                     unsigned int *out_num, unsigned int *in_num)
 334 {
 335         unsigned long uninitialized_var(endtime);
 336         int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
 337                                     out_num, in_num, NULL, NULL);
 338
 339         if (r == vq->num && vq->busyloop_timeout) {
 340                 preempt_disable();
 341                 endtime = busy_clock() + vq->busyloop_timeout;
 342                 while (vhost_can_busy_poll(vq->dev, endtime) &&
 343                        vhost_vq_avail_empty(vq->dev, vq))
 344                         cpu_relax_lowlatency();
 345                 preempt_enable();
 346                 r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
 347                                         out_num, in_num, NULL, NULL);
 348         }
 349
 350         return r;
 351 }
 352
 353 /* Expects to be always run from workqueue - which acts as
 354  * read-size critical section for our kind of RCU. */
 355 static void handle_tx(struct vhost_net *net)
 356 {
 357         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
 358         struct vhost_virtqueue *vq = &nvq->vq;
 359         unsigned out, in;
 360         int head;
 361         struct msghdr msg = {
 362                 .msg_name = NULL,
 363                 .msg_namelen = 0,
 364                 .msg_control = NULL,
 365                 .msg_controllen = 0,
 366                 .msg_flags = MSG_DONTWAIT,
 367         };
 368         size_t len, total_len = 0;
 369         int err;
 370         size_t hdr_size;
 371         struct socket *sock;
 372         struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
 373         bool zcopy, zcopy_used;
 374
 375         mutex_lock(&vq->mutex);
 376         sock = vq->private_data;
 377         if (!sock)
 378                 goto out;
 379
 380         vhost_disable_notify(&net->dev, vq);
 381
 382         hdr_size = nvq->vhost_hlen;
 383         zcopy = nvq->ubufs;
 384
 385         for (;;) {
 386                 /* Release DMAs done buffers first */
 387                 if (zcopy)
 388                         vhost_zerocopy_signal_used(net, vq);
 389
 390                 /* If more outstanding DMAs, queue the work.
 391                  * Handle upend_idx wrap around
 392                  */
 393                 if (unlikely((nvq->upend_idx + vq->num - VHOST_MAX_PEND)
 394                               % UIO_MAXIOV == nvq->done_idx))
 395                         break;
 396
 397                 head = vhost_net_tx_get_vq_desc(net, vq, vq->iov,
 398                                                 ARRAY_SIZE(vq->iov),
 399                                                 &out, &in);
 400                 /* On error, stop handling until the next kick. */
 401                 if (unlikely(head < 0))
 402                         break;
 403                 /* Nothing new?  Wait for eventfd to tell us they refilled. */
 404                 if (head == vq->num) {
 405                         if (unlikely(vhost_enable_notify(&net->dev, vq))) {
 406                                 vhost_disable_notify(&net->dev, vq);
 407                                 continue;
 408                         }
 409                         break;
 410                 }
 411                 if (in) {
 412                         vq_err(vq, "Unexpected descriptor format for TX: "
 413                                "out %d, int %d\n", out, in);
 414                         break;
 415                 }
 416                 /* Skip header. TODO: support TSO. */
 417                 len = iov_length(vq->iov, out);
 418                 iov_iter_init(&msg.msg_iter, WRITE, vq->iov, out, len);
 419                 iov_iter_advance(&msg.msg_iter, hdr_size);
 420                 /* Sanity check */
 421                 if (!msg_data_left(&msg)) {
 422                         vq_err(vq, "Unexpected header len for TX: "
 423                                "%zd expected %zd\n",
 424                                len, hdr_size);
 425                         break;
 426                 }
 427                 len = msg_data_left(&msg);
 428
 429                 zcopy_used = zcopy && len >= VHOST_GOODCOPY_LEN
 430                                    && (nvq->upend_idx + 1) % UIO_MAXIOV !=
 431                                       nvq->done_idx
 432                                    && vhost_net_tx_select_zcopy(net);
 433
 434                 /* use msg_control to pass vhost zerocopy ubuf info to skb */
 435                 if (zcopy_used) {
 436                         struct ubuf_info *ubuf;
 437                         ubuf = nvq->ubuf_info + nvq->upend_idx;
 438
 439                         vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
 440                         vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
 441                         ubuf->callback = vhost_zerocopy_callback;
 442                         ubuf->ctx = nvq->ubufs;
 443                         ubuf->desc = nvq->upend_idx;
 444                         msg.msg_control = ubuf;
 445                         msg.msg_controllen = sizeof(ubuf);
 446                         ubufs = nvq->ubufs;
 447                         atomic_inc(&ubufs->refcount);
 448                         nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
 449                 } else {
 450                         msg.msg_control = NULL;
 451                         ubufs = NULL;
 452                 }
 453                 /* TODO: Check specific error and bomb out unless ENOBUFS? */
 454                 err = sock->ops->sendmsg(sock, &msg, len);
 455                 if (unlikely(err < 0)) {
 456                         if (zcopy_used) {
 457                                 vhost_net_ubuf_put(ubufs);
 458                                 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
 459                                         % UIO_MAXIOV;
 460                         }
 461                         vhost_discard_vq_desc(vq, 1);
 462                         break;
 463                 }
 464                 if (err != len)
 465                         pr_debug("Truncated TX packet: "
 466                                  " len %d != %zd\n", err, len);
 467                 if (!zcopy_used)
 468                         vhost_add_used_and_signal(&net->dev, vq, head, 0);
 469                 else
 470                         vhost_zerocopy_signal_used(net, vq);
 471                 total_len += len;
 472                 vhost_net_tx_packet(net);
 473                 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
 474                         vhost_poll_queue(&vq->poll);
 475                         break;
 476                 }
 477         }
 478 out:
 479         mutex_unlock(&vq->mutex);
 480 }
 481
 482 static int peek_head_len(struct sock *sk)
 483 {
 484         struct socket *sock = sk->sk_socket;
 485         struct sk_buff *head;
 486         int len = 0;
 487         unsigned long flags;
 488
 489         if (sock->ops->peek_len)
 490                 return sock->ops->peek_len(sock);
 491
 492         spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
 493         head = skb_peek(&sk->sk_receive_queue);
 494         if (likely(head)) {
 495                 len = head->len;
 496                 if (skb_vlan_tag_present(head))
 497                         len += VLAN_HLEN;
 498         }
 499
 500         spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
 501         return len;
 502 }
 503
 504 static int sk_has_rx_data(struct sock *sk)
 505 {
 506         struct socket *sock = sk->sk_socket;
 507
 508         if (sock->ops->peek_len)
 509                 return sock->ops->peek_len(sock);
 510
 511         return skb_queue_empty(&sk->sk_receive_queue);
 512 }
 513
 514 static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
 515 {
 516         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
 517         struct vhost_virtqueue *vq = &nvq->vq;
 518         unsigned long uninitialized_var(endtime);
 519         int len = peek_head_len(sk);
 520
 521         if (!len && vq->busyloop_timeout) {
 522                 /* Both tx vq and rx socket were polled here */
 523                 mutex_lock(&vq->mutex);
 524                 vhost_disable_notify(&net->dev, vq);
 525
 526                 preempt_disable();
 527                 endtime = busy_clock() + vq->busyloop_timeout;
 528
 529                 while (vhost_can_busy_poll(&net->dev, endtime) &&
 530                        !sk_has_rx_data(sk) &&
 531                        vhost_vq_avail_empty(&net->dev, vq))
 532                         cpu_relax_lowlatency();
 533
 534                 preempt_enable();
 535
 536                 if (vhost_enable_notify(&net->dev, vq))
 537                         vhost_poll_queue(&vq->poll);
 538                 mutex_unlock(&vq->mutex);
 539
 540                 len = peek_head_len(sk);
 541         }
 542
 543         return len;
 544 }
 545
 546 /* This is a multi-buffer version of vhost_get_desc, that works if
 547  *      vq has read descriptors only.
 548  * @vq          - the relevant virtqueue
 549  * @datalen     - data length we'll be reading
 550  * @iovcount    - returned count of io vectors we fill
 551  * @log         - vhost log
 552  * @log_num     - log offset
 553  * @quota       - headcount quota, 1 for big buffer
 554  *      returns number of buffer heads allocated, negative on error
 555  */
 556 static int get_rx_bufs(struct vhost_virtqueue *vq,
 557                        struct vring_used_elem *heads,
 558                        int datalen,
 559                        unsigned *iovcount,
 560                        struct vhost_log *log,
 561                        unsigned *log_num,
 562                        unsigned int quota)
 563 {
 564         unsigned int out, in;
 565         int seg = 0;
 566         int headcount = 0;
 567         unsigned d;
 568         int r, nlogs = 0;
 569         /* len is always initialized before use since we are always called with
 570          * datalen > 0.
 571          */
 572         u32 uninitialized_var(len);
 573
 574         while (datalen > 0 && headcount < quota) {
 575                 if (unlikely(seg >= UIO_MAXIOV)) {
 576                         r = -ENOBUFS;
 577                         goto err;
 578                 }
 579                 r = vhost_get_vq_desc(vq, vq->iov + seg,
 580                                       ARRAY_SIZE(vq->iov) - seg, &out,
 581                                       &in, log, log_num);
 582                 if (unlikely(r < 0))
 583                         goto err;
 584
 585                 d = r;
 586                 if (d == vq->num) {
 587                         r = 0;
 588                         goto err;
 589                 }
 590                 if (unlikely(out || in <= 0)) {
 591                         vq_err(vq, "unexpected descriptor format for RX: "
 592                                 "out %d, in %d\n", out, in);
 593                         r = -EINVAL;
 594                         goto err;
 595                 }
 596                 if (unlikely(log)) {
 597                         nlogs += *log_num;
 598                         log += *log_num;
 599                 }
 600                 heads[headcount].id = cpu_to_vhost32(vq, d);
 601                 len = iov_length(vq->iov + seg, in);
 602                 heads[headcount].len = cpu_to_vhost32(vq, len);
 603                 datalen -= len;
 604                 ++headcount;
 605                 seg += in;
 606         }
 607         heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
 608         *iovcount = seg;
 609         if (unlikely(log))
 610                 *log_num = nlogs;
 611
 612         /* Detect overrun */
 613         if (unlikely(datalen > 0)) {
 614                 r = UIO_MAXIOV + 1;
 615                 goto err;
 616         }
 617         return headcount;
 618 err:
 619         vhost_discard_vq_desc(vq, headcount);
 620         return r;
 621 }
 622
 623 /* Expects to be always run from workqueue - which acts as
 624  * read-size critical section for our kind of RCU. */
 625 static void handle_rx(struct vhost_net *net)
 626 {
 627         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
 628         struct vhost_virtqueue *vq = &nvq->vq;
 629         unsigned uninitialized_var(in), log;
 630         struct vhost_log *vq_log;
 631         struct msghdr msg = {
 632                 .msg_name = NULL,
 633                 .msg_namelen = 0,
 634                 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
 635                 .msg_controllen = 0,
 636                 .msg_flags = MSG_DONTWAIT,
 637         };
 638         struct virtio_net_hdr hdr = {
 639                 .flags = 0,
 640                 .gso_type = VIRTIO_NET_HDR_GSO_NONE
 641         };
 642         size_t total_len = 0;
 643         int err, mergeable;
 644         s16 headcount;
 645         size_t vhost_hlen, sock_hlen;
 646         size_t vhost_len, sock_len;
 647         struct socket *sock;
 648         struct iov_iter fixup;
 649         __virtio16 num_buffers;
 650
 651         mutex_lock(&vq->mutex);
 652         sock = vq->private_data;
 653         if (!sock)
 654                 goto out;
 655         vhost_disable_notify(&net->dev, vq);
 656         vhost_net_disable_vq(net, vq);
 657
 658         vhost_hlen = nvq->vhost_hlen;
 659         sock_hlen = nvq->sock_hlen;
 660
 661         vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
 662                 vq->log : NULL;
 663         mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
 664
 665         while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk))) {
 666                 sock_len += sock_hlen;
 667                 vhost_len = sock_len + vhost_hlen;
 668                 headcount = get_rx_bufs(vq, vq->heads, vhost_len,
 669                                         &in, vq_log, &log,
 670                                         likely(mergeable) ? UIO_MAXIOV : 1);
 671                 /* On error, stop handling until the next kick. */
 672                 if (unlikely(headcount < 0))
 673                         goto out;
 674                 /* On overrun, truncate and discard */
 675                 if (unlikely(headcount > UIO_MAXIOV)) {
 676                         iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
 677                         err = sock->ops->recvmsg(sock, &msg,
 678                                                  1, MSG_DONTWAIT | MSG_TRUNC);
 679                         pr_debug("Discarded rx packet: len %zd\n", sock_len);
 680                         continue;
 681                 }
 682                 /* OK, now we need to know about added descriptors. */
 683                 if (!headcount) {
 684                         if (unlikely(vhost_enable_notify(&net->dev, vq))) {
 685                                 /* They have slipped one in as we were
 686                                  * doing that: check again. */
 687                                 vhost_disable_notify(&net->dev, vq);
 688                                 continue;
 689                         }
 690                         /* Nothing new?  Wait for eventfd to tell us
 691                          * they refilled. */
 692                         goto out;
 693                 }
 694                 /* We don't need to be notified again. */
 695                 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
 696                 fixup = msg.msg_iter;
 697                 if (unlikely((vhost_hlen))) {
 698                         /* We will supply the header ourselves
 699                          * TODO: support TSO.
 700                          */
 701                         iov_iter_advance(&msg.msg_iter, vhost_hlen);
 702                 }
 703                 err = sock->ops->recvmsg(sock, &msg,
 704                                          sock_len, MSG_DONTWAIT | MSG_TRUNC);
 705                 /* Userspace might have consumed the packet meanwhile:
 706                  * it's not supposed to do this usually, but might be hard
 707                  * to prevent. Discard data we got (if any) and keep going. */
 708                 if (unlikely(err != sock_len)) {
 709                         pr_debug("Discarded rx packet: "
 710                                  " len %d, expected %zd\n", err, sock_len);
 711                         vhost_discard_vq_desc(vq, headcount);
 712                         continue;
 713                 }
 714                 /* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
 715                 if (unlikely(vhost_hlen)) {
 716                         if (copy_to_iter(&hdr, sizeof(hdr),
 717                                          &fixup) != sizeof(hdr)) {
 718                                 vq_err(vq, "Unable to write vnet_hdr "
 719                                        "at addr %p\n", vq->iov->iov_base);
 720                                 goto out;
 721                         }
 722                 } else {
 723                         /* Header came from socket; we'll need to patch
 724                          * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
 725                          */
 726                         iov_iter_advance(&fixup, sizeof(hdr));
 727                 }
 728                 /* TODO: Should check and handle checksum. */
 729
 730                 num_buffers = cpu_to_vhost16(vq, headcount);
 731                 if (likely(mergeable) &&
 732                     copy_to_iter(&num_buffers, sizeof num_buffers,
 733                                  &fixup) != sizeof num_buffers) {
 734                         vq_err(vq, "Failed num_buffers write");
 735                         vhost_discard_vq_desc(vq, headcount);
 736                         goto out;
 737                 }
 738                 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
 739                                             headcount);
 740                 if (unlikely(vq_log))
 741                         vhost_log_write(vq, vq_log, log, vhost_len);
 742                 total_len += vhost_len;
 743                 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
 744                         vhost_poll_queue(&vq->poll);
 745                         goto out;
 746                 }
 747         }
 748         vhost_net_enable_vq(net, vq);
 749 out:
 750         mutex_unlock(&vq->mutex);
 751 }
 752
 753 static void handle_tx_kick(struct vhost_work *work)
 754 {
 755         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 756                                                   poll.work);
 757         struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
 758
 759         handle_tx(net);
 760 }
 761
 762 static void handle_rx_kick(struct vhost_work *work)
 763 {
 764         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 765                                                   poll.work);
 766         struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
 767
 768         handle_rx(net);
 769 }
 770
 771 static void handle_tx_net(struct vhost_work *work)
 772 {
 773         struct vhost_net *net = container_of(work, struct vhost_net,
 774                                              poll[VHOST_NET_VQ_TX].work);
 775         handle_tx(net);
 776 }
 777
 778 static void handle_rx_net(struct vhost_work *work)
 779 {
 780         struct vhost_net *net = container_of(work, struct vhost_net,
 781                                              poll[VHOST_NET_VQ_RX].work);
 782         handle_rx(net);
 783 }
 784
 785 static int vhost_net_open(struct inode *inode, struct file *f)
 786 {
 787         struct vhost_net *n;
 788         struct vhost_dev *dev;
 789         struct vhost_virtqueue **vqs;
 790         int i;
 791
 792         n = kmalloc(sizeof *n, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
 793         if (!n) {
 794                 n = vmalloc(sizeof *n);
 795                 if (!n)
 796                         return -ENOMEM;
 797         }
 798         vqs = kmalloc(VHOST_NET_VQ_MAX * sizeof(*vqs), GFP_KERNEL);
 799         if (!vqs) {
 800                 kvfree(n);
 801                 return -ENOMEM;
 802         }
 803
 804         dev = &n->dev;
 805         vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
 806         vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
 807         n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
 808         n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
 809         for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
 810                 n->vqs[i].ubufs = NULL;
 811                 n->vqs[i].ubuf_info = NULL;
 812                 n->vqs[i].upend_idx = 0;
 813                 n->vqs[i].done_idx = 0;
 814                 n->vqs[i].vhost_hlen = 0;
 815                 n->vqs[i].sock_hlen = 0;
 816         }
 817         vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX);
 818
 819         vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);
 820         vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);
 821
 822         f->private_data = n;
 823
 824         return 0;
 825 }
 826
 827 static struct socket *vhost_net_stop_vq(struct vhost_net *n,
 828                                         struct vhost_virtqueue *vq)
 829 {
 830         struct socket *sock;
 831
 832         mutex_lock(&vq->mutex);
 833         sock = vq->private_data;
 834         vhost_net_disable_vq(n, vq);
 835         vq->private_data = NULL;
 836         mutex_unlock(&vq->mutex);
 837         return sock;
 838 }
 839
 840 static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
 841                            struct socket **rx_sock)
 842 {
 843         *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
 844         *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
 845 }
 846
 847 static void vhost_net_flush_vq(struct vhost_net *n, int index)
 848 {
 849         vhost_poll_flush(n->poll + index);
 850         vhost_poll_flush(&n->vqs[index].vq.poll);
 851 }
 852
 853 static void vhost_net_flush(struct vhost_net *n)
 854 {
 855         vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
 856         vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
 857         if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
 858                 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 859                 n->tx_flush = true;
 860                 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 861                 /* Wait for all lower device DMAs done. */
 862                 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
 863                 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 864                 n->tx_flush = false;
 865                 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
 866                 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 867         }
 868 }
 869
 870 static int vhost_net_release(struct inode *inode, struct file *f)
 871 {
 872         struct vhost_net *n = f->private_data;
 873         struct socket *tx_sock;
 874         struct socket *rx_sock;
 875
 876         vhost_net_stop(n, &tx_sock, &rx_sock);
 877         vhost_net_flush(n);
 878         vhost_dev_stop(&n->dev);
 879         vhost_dev_cleanup(&n->dev, false);
 880         vhost_net_vq_reset(n);
 881         if (tx_sock)
 882                 sockfd_put(tx_sock);
 883         if (rx_sock)
 884                 sockfd_put(rx_sock);
 885         /* Make sure no callbacks are outstanding */
 886         synchronize_rcu_bh();
 887         /* We do an extra flush before freeing memory,
 888          * since jobs can re-queue themselves. */
 889         vhost_net_flush(n);
 890         kfree(n->dev.vqs);
 891         kvfree(n);
 892         return 0;
 893 }
 894
 895 static struct socket *get_raw_socket(int fd)
 896 {
 897         struct {
 898                 struct sockaddr_ll sa;
 899                 char  buf[MAX_ADDR_LEN];
 900         } uaddr;
 901         int uaddr_len = sizeof uaddr, r;
 902         struct socket *sock = sockfd_lookup(fd, &r);
 903
 904         if (!sock)
 905                 return ERR_PTR(-ENOTSOCK);
 906
 907         /* Parameter checking */
 908         if (sock->sk->sk_type != SOCK_RAW) {
 909                 r = -ESOCKTNOSUPPORT;
 910                 goto err;
 911         }
 912
 913         r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
 914                                &uaddr_len, 0);
 915         if (r)
 916                 goto err;
 917
 918         if (uaddr.sa.sll_family != AF_PACKET) {
 919                 r = -EPFNOSUPPORT;
 920                 goto err;
 921         }
 922         return sock;
 923 err:
 924         sockfd_put(sock);
 925         return ERR_PTR(r);
 926 }
 927
 928 static struct socket *get_tap_socket(int fd)
 929 {
 930         struct file *file = fget(fd);
 931         struct socket *sock;
 932
 933         if (!file)
 934                 return ERR_PTR(-EBADF);
 935         sock = tun_get_socket(file);
 936         if (!IS_ERR(sock))
 937                 return sock;
 938         sock = macvtap_get_socket(file);
 939         if (IS_ERR(sock))
 940                 fput(file);
 941         return sock;
 942 }
 943
 944 static struct socket *get_socket(int fd)
 945 {
 946         struct socket *sock;
 947
 948         /* special case to disable backend */
 949         if (fd == -1)
 950                 return NULL;
 951         sock = get_raw_socket(fd);
 952         if (!IS_ERR(sock))
 953                 return sock;
 954         sock = get_tap_socket(fd);
 955         if (!IS_ERR(sock))
 956                 return sock;
 957         return ERR_PTR(-ENOTSOCK);
 958 }
 959
 960 static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
 961 {
 962         struct socket *sock, *oldsock;
 963         struct vhost_virtqueue *vq;
 964         struct vhost_net_virtqueue *nvq;
 965         struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
 966         int r;
 967
 968         mutex_lock(&n->dev.mutex);
 969         r = vhost_dev_check_owner(&n->dev);
 970         if (r)
 971                 goto err;
 972
 973         if (index >= VHOST_NET_VQ_MAX) {
 974                 r = -ENOBUFS;
 975                 goto err;
 976         }
 977         vq = &n->vqs[index].vq;
 978         nvq = &n->vqs[index];
 979         mutex_lock(&vq->mutex);
 980
 981         /* Verify that ring has been setup correctly. */
 982         if (!vhost_vq_access_ok(vq)) {
 983                 r = -EFAULT;
 984                 goto err_vq;
 985         }
 986         sock = get_socket(fd);
 987         if (IS_ERR(sock)) {
 988                 r = PTR_ERR(sock);
 989                 goto err_vq;
 990         }
 991
 992         /* start polling new socket */
 993         oldsock = vq->private_data;
 994         if (sock != oldsock) {
 995                 ubufs = vhost_net_ubuf_alloc(vq,
 996                                              sock && vhost_sock_zcopy(sock));
 997                 if (IS_ERR(ubufs)) {
 998                         r = PTR_ERR(ubufs);
 999                         goto err_ubufs;
1000                 }
1001
1002                 vhost_net_disable_vq(n, vq);
1003                 vq->private_data = sock;
1004                 r = vhost_vq_init_access(vq);
1005                 if (r)
1006                         goto err_used;
1007                 r = vhost_net_enable_vq(n, vq);
1008                 if (r)
1009                         goto err_used;
1010
1011                 oldubufs = nvq->ubufs;
1012                 nvq->ubufs = ubufs;
1013
1014                 n->tx_packets = 0;
1015                 n->tx_zcopy_err = 0;
1016                 n->tx_flush = false;
1017         }
1018
1019         mutex_unlock(&vq->mutex);
1020
1021         if (oldubufs) {
1022                 vhost_net_ubuf_put_wait_and_free(oldubufs);
1023                 mutex_lock(&vq->mutex);
1024                 vhost_zerocopy_signal_used(n, vq);
1025                 mutex_unlock(&vq->mutex);
1026         }
1027
1028         if (oldsock) {
1029                 vhost_net_flush_vq(n, index);
1030                 sockfd_put(oldsock);
1031         }
1032
1033         mutex_unlock(&n->dev.mutex);
1034         return 0;
1035
1036 err_used:
1037         vq->private_data = oldsock;
1038         vhost_net_enable_vq(n, vq);
1039         if (ubufs)
1040                 vhost_net_ubuf_put_wait_and_free(ubufs);
1041 err_ubufs:
1042         sockfd_put(sock);
1043 err_vq:
1044         mutex_unlock(&vq->mutex);
1045 err:
1046         mutex_unlock(&n->dev.mutex);
1047         return r;
1048 }
1049
1050 static long vhost_net_reset_owner(struct vhost_net *n)
1051 {
1052         struct socket *tx_sock = NULL;
1053         struct socket *rx_sock = NULL;
1054         long err;
1055         struct vhost_memory *memory;
1056
1057         mutex_lock(&n->dev.mutex);
1058         err = vhost_dev_check_owner(&n->dev);
1059         if (err)
1060                 goto done;
1061         memory = vhost_dev_reset_owner_prepare();
1062         if (!memory) {
1063                 err = -ENOMEM;
1064                 goto done;
1065         }
1066         vhost_net_stop(n, &tx_sock, &rx_sock);
1067         vhost_net_flush(n);
1068         vhost_dev_reset_owner(&n->dev, memory);
1069         vhost_net_vq_reset(n);
1070 done:
1071         mutex_unlock(&n->dev.mutex);
1072         if (tx_sock)
1073                 sockfd_put(tx_sock);
1074         if (rx_sock)
1075                 sockfd_put(rx_sock);
1076         return err;
1077 }
1078
1079 static int vhost_net_set_features(struct vhost_net *n, u64 features)
1080 {
1081         size_t vhost_hlen, sock_hlen, hdr_len;
1082         int i;
1083
1084         hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1085                                (1ULL << VIRTIO_F_VERSION_1))) ?
1086                         sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1087                         sizeof(struct virtio_net_hdr);
1088         if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
1089                 /* vhost provides vnet_hdr */
1090                 vhost_hlen = hdr_len;
1091                 sock_hlen = 0;
1092         } else {
1093                 /* socket provides vnet_hdr */
1094                 vhost_hlen = 0;
1095                 sock_hlen = hdr_len;
1096         }
1097         mutex_lock(&n->dev.mutex);
1098         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1099             !vhost_log_access_ok(&n->dev)) {
1100                 mutex_unlock(&n->dev.mutex);
1101                 return -EFAULT;
1102         }
1103         for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
1104                 mutex_lock(&n->vqs[i].vq.mutex);
1105                 n->vqs[i].vq.acked_features = features;
1106                 n->vqs[i].vhost_hlen = vhost_hlen;
1107                 n->vqs[i].sock_hlen = sock_hlen;
1108                 mutex_unlock(&n->vqs[i].vq.mutex);
1109         }
1110         mutex_unlock(&n->dev.mutex);
1111         return 0;
1112 }
1113
1114 static long vhost_net_set_owner(struct vhost_net *n)
1115 {
1116         int r;
1117
1118         mutex_lock(&n->dev.mutex);
1119         if (vhost_dev_has_owner(&n->dev)) {
1120                 r = -EBUSY;
1121                 goto out;
1122         }
1123         r = vhost_net_set_ubuf_info(n);
1124         if (r)
1125                 goto out;
1126         r = vhost_dev_set_owner(&n->dev);
1127         if (r)
1128                 vhost_net_clear_ubuf_info(n);
1129         vhost_net_flush(n);
1130 out:
1131         mutex_unlock(&n->dev.mutex);
1132         return r;
1133 }
1134
1135 static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
1136                             unsigned long arg)
1137 {
1138         struct vhost_net *n = f->private_data;
1139         void __user *argp = (void __user *)arg;
1140         u64 __user *featurep = argp;
1141         struct vhost_vring_file backend;
1142         u64 features;
1143         int r;
1144
1145         switch (ioctl) {
1146         case VHOST_NET_SET_BACKEND:
1147                 if (copy_from_user(&backend, argp, sizeof backend))
1148                         return -EFAULT;
1149                 return vhost_net_set_backend(n, backend.index, backend.fd);
1150         case VHOST_GET_FEATURES:
1151                 features = VHOST_NET_FEATURES;
1152                 if (copy_to_user(featurep, &features, sizeof features))
1153                         return -EFAULT;
1154                 return 0;
1155         case VHOST_SET_FEATURES:
1156                 if (copy_from_user(&features, featurep, sizeof features))
1157                         return -EFAULT;
1158                 if (features & ~VHOST_NET_FEATURES)
1159                         return -EOPNOTSUPP;
1160                 return vhost_net_set_features(n, features);
1161         case VHOST_RESET_OWNER:
1162                 return vhost_net_reset_owner(n);
1163         case VHOST_SET_OWNER:
1164                 return vhost_net_set_owner(n);
1165         default:
1166                 mutex_lock(&n->dev.mutex);
1167                 r = vhost_dev_ioctl(&n->dev, ioctl, argp);
1168                 if (r == -ENOIOCTLCMD)
1169                         r = vhost_vring_ioctl(&n->dev, ioctl, argp);
1170                 else
1171                         vhost_net_flush(n);
1172                 mutex_unlock(&n->dev.mutex);
1173                 return r;
1174         }
1175 }
1176
1177 #ifdef CONFIG_COMPAT
1178 static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
1179                                    unsigned long arg)
1180 {
1181         return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1182 }
1183 #endif
1184
1185 static const struct file_operations vhost_net_fops = {
1186         .owner          = THIS_MODULE,
1187         .release        = vhost_net_release,
1188         .unlocked_ioctl = vhost_net_ioctl,
1189 #ifdef CONFIG_COMPAT
1190         .compat_ioctl   = vhost_net_compat_ioctl,
1191 #endif
1192         .open           = vhost_net_open,
1193         .llseek         = noop_llseek,
1194 };
1195
1196 static struct miscdevice vhost_net_misc = {
1197         .minor = VHOST_NET_MINOR,
1198         .name = "vhost-net",
1199         .fops = &vhost_net_fops,
1200 };
1201
1202 static int vhost_net_init(void)
1203 {
1204         if (experimental_zcopytx)
1205                 vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
1206         return misc_register(&vhost_net_misc);
1207 }
1208 module_init(vhost_net_init);
1209
1210 static void vhost_net_exit(void)
1211 {
1212         misc_deregister(&vhost_net_misc);
1213 }
1214 module_exit(vhost_net_exit);
1215
1216 MODULE_VERSION("0.0.1");
1217 MODULE_LICENSE("GPL v2");
1218 MODULE_AUTHOR("Michael S. Tsirkin");
1219 MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
1220 MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
1221 MODULE_ALIAS("devname:vhost-net");