fs/fuse/dev.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/poll.h>
  14 #include <linux/uio.h>
  15 #include <linux/miscdevice.h>
  16 #include <linux/pagemap.h>
  17 #include <linux/file.h>
  18 #include <linux/slab.h>
  19 #include <linux/pipe_fs_i.h>
  20 #include <linux/swap.h>
  21 #include <linux/splice.h>
  22
  23 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
  24 MODULE_ALIAS("devname:fuse");
  25
  26 static struct kmem_cache *fuse_req_cachep;
  27
  28 static struct fuse_conn *fuse_get_conn(struct file *file)
  29 {
  30         /*
  31          * Lockless access is OK, because file->private data is set
  32          * once during mount and is valid until the file is released.
  33          */
  34         return file->private_data;
  35 }
  36
  37 static void fuse_request_init(struct fuse_req *req, struct page **pages,
  38                               struct fuse_page_desc *page_descs,
  39                               unsigned npages)
  40 {
  41         memset(req, 0, sizeof(*req));
  42         memset(pages, 0, sizeof(*pages) * npages);
  43         memset(page_descs, 0, sizeof(*page_descs) * npages);
  44         INIT_LIST_HEAD(&req->list);
  45         INIT_LIST_HEAD(&req->intr_entry);
  46         init_waitqueue_head(&req->waitq);
  47         atomic_set(&req->count, 1);
  48         req->pages = pages;
  49         req->page_descs = page_descs;
  50         req->max_pages = npages;
  51         __set_bit(FR_PENDING, &req->flags);
  52 }
  53
  54 static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
  55 {
  56         struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
  57         if (req) {
  58                 struct page **pages;
  59                 struct fuse_page_desc *page_descs;
  60
  61                 if (npages <= FUSE_REQ_INLINE_PAGES) {
  62                         pages = req->inline_pages;
  63                         page_descs = req->inline_page_descs;
  64                 } else {
  65                         pages = kmalloc(sizeof(struct page *) * npages, flags);
  66                         page_descs = kmalloc(sizeof(struct fuse_page_desc) *
  67                                              npages, flags);
  68                 }
  69
  70                 if (!pages || !page_descs) {
  71                         kfree(pages);
  72                         kfree(page_descs);
  73                         kmem_cache_free(fuse_req_cachep, req);
  74                         return NULL;
  75                 }
  76
  77                 fuse_request_init(req, pages, page_descs, npages);
  78         }
  79         return req;
  80 }
  81
  82 struct fuse_req *fuse_request_alloc(unsigned npages)
  83 {
  84         return __fuse_request_alloc(npages, GFP_KERNEL);
  85 }
  86 EXPORT_SYMBOL_GPL(fuse_request_alloc);
  87
  88 struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
  89 {
  90         return __fuse_request_alloc(npages, GFP_NOFS);
  91 }
  92
  93 void fuse_request_free(struct fuse_req *req)
  94 {
  95         if (req->pages != req->inline_pages) {
  96                 kfree(req->pages);
  97                 kfree(req->page_descs);
  98         }
  99         kmem_cache_free(fuse_req_cachep, req);
 100 }
 101
 102 static void block_sigs(sigset_t *oldset)
 103 {
 104         sigset_t mask;
 105
 106         siginitsetinv(&mask, sigmask(SIGKILL));
 107         sigprocmask(SIG_BLOCK, &mask, oldset);
 108 }
 109
 110 static void restore_sigs(sigset_t *oldset)
 111 {
 112         sigprocmask(SIG_SETMASK, oldset, NULL);
 113 }
 114
 115 void __fuse_get_request(struct fuse_req *req)
 116 {
 117         atomic_inc(&req->count);
 118 }
 119
 120 /* Must be called with > 1 refcount */
 121 static void __fuse_put_request(struct fuse_req *req)
 122 {
 123         BUG_ON(atomic_read(&req->count) < 2);
 124         atomic_dec(&req->count);
 125 }
 126
 127 static void fuse_req_init_context(struct fuse_req *req)
 128 {
 129         req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
 130         req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
 131         req->in.h.pid = current->pid;
 132 }
 133
 134 void fuse_set_initialized(struct fuse_conn *fc)
 135 {
 136         /* Make sure stores before this are seen on another CPU */
 137         smp_wmb();
 138         fc->initialized = 1;
 139 }
 140
 141 static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
 142 {
 143         return !fc->initialized || (for_background && fc->blocked);
 144 }
 145
 146 static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
 147                                        bool for_background)
 148 {
 149         struct fuse_req *req;
 150         int err;
 151         atomic_inc(&fc->num_waiting);
 152
 153         if (fuse_block_alloc(fc, for_background)) {
 154                 sigset_t oldset;
 155                 int intr;
 156
 157                 block_sigs(&oldset);
 158                 intr = wait_event_interruptible_exclusive(fc->blocked_waitq,
 159                                 !fuse_block_alloc(fc, for_background));
 160                 restore_sigs(&oldset);
 161                 err = -EINTR;
 162                 if (intr)
 163                         goto out;
 164         }
 165         /* Matches smp_wmb() in fuse_set_initialized() */
 166         smp_rmb();
 167
 168         err = -ENOTCONN;
 169         if (!fc->connected)
 170                 goto out;
 171
 172         err = -ECONNREFUSED;
 173         if (fc->conn_error)
 174                 goto out;
 175
 176         req = fuse_request_alloc(npages);
 177         err = -ENOMEM;
 178         if (!req) {
 179                 if (for_background)
 180                         wake_up(&fc->blocked_waitq);
 181                 goto out;
 182         }
 183
 184         fuse_req_init_context(req);
 185         __set_bit(FR_WAITING, &req->flags);
 186         if (for_background)
 187                 __set_bit(FR_BACKGROUND, &req->flags);
 188
 189         return req;
 190
 191  out:
 192         atomic_dec(&fc->num_waiting);
 193         return ERR_PTR(err);
 194 }
 195
 196 struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
 197 {
 198         return __fuse_get_req(fc, npages, false);
 199 }
 200 EXPORT_SYMBOL_GPL(fuse_get_req);
 201
 202 struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
 203                                              unsigned npages)
 204 {
 205         return __fuse_get_req(fc, npages, true);
 206 }
 207 EXPORT_SYMBOL_GPL(fuse_get_req_for_background);
 208
 209 /*
 210  * Return request in fuse_file->reserved_req.  However that may
 211  * currently be in use.  If that is the case, wait for it to become
 212  * available.
 213  */
 214 static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
 215                                          struct file *file)
 216 {
 217         struct fuse_req *req = NULL;
 218         struct fuse_file *ff = file->private_data;
 219
 220         do {
 221                 wait_event(fc->reserved_req_waitq, ff->reserved_req);
 222                 spin_lock(&fc->lock);
 223                 if (ff->reserved_req) {
 224                         req = ff->reserved_req;
 225                         ff->reserved_req = NULL;
 226                         req->stolen_file = get_file(file);
 227                 }
 228                 spin_unlock(&fc->lock);
 229         } while (!req);
 230
 231         return req;
 232 }
 233
 234 /*
 235  * Put stolen request back into fuse_file->reserved_req
 236  */
 237 static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
 238 {
 239         struct file *file = req->stolen_file;
 240         struct fuse_file *ff = file->private_data;
 241
 242         spin_lock(&fc->lock);
 243         fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
 244         BUG_ON(ff->reserved_req);
 245         ff->reserved_req = req;
 246         wake_up_all(&fc->reserved_req_waitq);
 247         spin_unlock(&fc->lock);
 248         fput(file);
 249 }
 250
 251 /*
 252  * Gets a requests for a file operation, always succeeds
 253  *
 254  * This is used for sending the FLUSH request, which must get to
 255  * userspace, due to POSIX locks which may need to be unlocked.
 256  *
 257  * If allocation fails due to OOM, use the reserved request in
 258  * fuse_file.
 259  *
 260  * This is very unlikely to deadlock accidentally, since the
 261  * filesystem should not have it's own file open.  If deadlock is
 262  * intentional, it can still be broken by "aborting" the filesystem.
 263  */
 264 struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
 265                                              struct file *file)
 266 {
 267         struct fuse_req *req;
 268
 269         atomic_inc(&fc->num_waiting);
 270         wait_event(fc->blocked_waitq, fc->initialized);
 271         /* Matches smp_wmb() in fuse_set_initialized() */
 272         smp_rmb();
 273         req = fuse_request_alloc(0);
 274         if (!req)
 275                 req = get_reserved_req(fc, file);
 276
 277         fuse_req_init_context(req);
 278         __set_bit(FR_WAITING, &req->flags);
 279         __clear_bit(FR_BACKGROUND, &req->flags);
 280         return req;
 281 }
 282
 283 void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
 284 {
 285         if (atomic_dec_and_test(&req->count)) {
 286                 if (test_bit(FR_BACKGROUND, &req->flags)) {
 287                         /*
 288                          * We get here in the unlikely case that a background
 289                          * request was allocated but not sent
 290                          */
 291                         spin_lock(&fc->lock);
 292                         if (!fc->blocked)
 293                                 wake_up(&fc->blocked_waitq);
 294                         spin_unlock(&fc->lock);
 295                 }
 296
 297                 if (test_bit(FR_WAITING, &req->flags)) {
 298                         __clear_bit(FR_WAITING, &req->flags);
 299                         atomic_dec(&fc->num_waiting);
 300                 }
 301
 302                 if (req->stolen_file)
 303                         put_reserved_req(fc, req);
 304                 else
 305                         fuse_request_free(req);
 306         }
 307 }
 308 EXPORT_SYMBOL_GPL(fuse_put_request);
 309
 310 static unsigned len_args(unsigned numargs, struct fuse_arg *args)
 311 {
 312         unsigned nbytes = 0;
 313         unsigned i;
 314
 315         for (i = 0; i < numargs; i++)
 316                 nbytes += args[i].size;
 317
 318         return nbytes;
 319 }
 320
 321 static u64 fuse_get_unique(struct fuse_iqueue *fiq)
 322 {
 323         return ++fiq->reqctr;
 324 }
 325
 326 static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
 327 {
 328         req->in.h.len = sizeof(struct fuse_in_header) +
 329                 len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
 330         list_add_tail(&req->list, &fiq->pending);
 331         wake_up_locked(&fiq->waitq);
 332         kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
 333 }
 334
 335 void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
 336                        u64 nodeid, u64 nlookup)
 337 {
 338         struct fuse_iqueue *fiq = &fc->iq;
 339
 340         forget->forget_one.nodeid = nodeid;
 341         forget->forget_one.nlookup = nlookup;
 342
 343         spin_lock(&fc->lock);
 344         spin_lock(&fiq->waitq.lock);
 345         if (fiq->connected) {
 346                 fiq->forget_list_tail->next = forget;
 347                 fiq->forget_list_tail = forget;
 348                 wake_up_locked(&fiq->waitq);
 349                 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
 350         } else {
 351                 kfree(forget);
 352         }
 353         spin_unlock(&fiq->waitq.lock);
 354         spin_unlock(&fc->lock);
 355 }
 356
 357 static void flush_bg_queue(struct fuse_conn *fc)
 358 {
 359         while (fc->active_background < fc->max_background &&
 360                !list_empty(&fc->bg_queue)) {
 361                 struct fuse_req *req;
 362                 struct fuse_iqueue *fiq = &fc->iq;
 363
 364                 req = list_entry(fc->bg_queue.next, struct fuse_req, list);
 365                 list_del(&req->list);
 366                 fc->active_background++;
 367                 spin_lock(&fiq->waitq.lock);
 368                 req->in.h.unique = fuse_get_unique(fiq);
 369                 queue_request(fiq, req);
 370                 spin_unlock(&fiq->waitq.lock);
 371         }
 372 }
 373
 374 /*
 375  * This function is called when a request is finished.  Either a reply
 376  * has arrived or it was aborted (and not yet sent) or some error
 377  * occurred during communication with userspace, or the device file
 378  * was closed.  The requester thread is woken up (if still waiting),
 379  * the 'end' callback is called if given, else the reference to the
 380  * request is released
 381  *
 382  * Called with fc->lock, unlocks it
 383  */
 384 static void request_end(struct fuse_conn *fc, struct fuse_req *req)
 385 __releases(fc->lock)
 386 {
 387         struct fuse_iqueue *fiq = &fc->iq;
 388         void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
 389         req->end = NULL;
 390         list_del_init(&req->list);
 391         spin_lock(&fiq->waitq.lock);
 392         list_del_init(&req->intr_entry);
 393         spin_unlock(&fiq->waitq.lock);
 394         WARN_ON(test_bit(FR_PENDING, &req->flags));
 395         WARN_ON(test_bit(FR_SENT, &req->flags));
 396         smp_wmb();
 397         set_bit(FR_FINISHED, &req->flags);
 398         if (test_bit(FR_BACKGROUND, &req->flags)) {
 399                 clear_bit(FR_BACKGROUND, &req->flags);
 400                 if (fc->num_background == fc->max_background)
 401                         fc->blocked = 0;
 402
 403                 /* Wake up next waiter, if any */
 404                 if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
 405                         wake_up(&fc->blocked_waitq);
 406
 407                 if (fc->num_background == fc->congestion_threshold &&
 408                     fc->connected && fc->bdi_initialized) {
 409                         clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
 410                         clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
 411                 }
 412                 fc->num_background--;
 413                 fc->active_background--;
 414                 flush_bg_queue(fc);
 415         }
 416         spin_unlock(&fc->lock);
 417         wake_up(&req->waitq);
 418         if (end)
 419                 end(fc, req);
 420         fuse_put_request(fc, req);
 421 }
 422
 423 static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
 424 {
 425         spin_lock(&fiq->waitq.lock);
 426         list_add_tail(&req->intr_entry, &fiq->interrupts);
 427         wake_up_locked(&fiq->waitq);
 428         spin_unlock(&fiq->waitq.lock);
 429         kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
 430 }
 431
 432 static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
 433 {
 434         struct fuse_iqueue *fiq = &fc->iq;
 435         int err;
 436
 437         if (!fc->no_interrupt) {
 438                 /* Any signal may interrupt this */
 439                 err = wait_event_interruptible(req->waitq,
 440                                         test_bit(FR_FINISHED, &req->flags));
 441                 if (!err)
 442                         return;
 443
 444                 spin_lock(&fc->lock);
 445                 set_bit(FR_INTERRUPTED, &req->flags);
 446                 if (test_bit(FR_SENT, &req->flags))
 447                         queue_interrupt(fiq, req);
 448                 spin_unlock(&fc->lock);
 449         }
 450
 451         if (!test_bit(FR_FORCE, &req->flags)) {
 452                 sigset_t oldset;
 453
 454                 /* Only fatal signals may interrupt this */
 455                 block_sigs(&oldset);
 456                 err = wait_event_interruptible(req->waitq,
 457                                         test_bit(FR_FINISHED, &req->flags));
 458                 restore_sigs(&oldset);
 459
 460                 if (!err)
 461                         return;
 462
 463                 spin_lock(&fc->lock);
 464                 spin_lock(&fiq->waitq.lock);
 465                 /* Request is not yet in userspace, bail out */
 466                 if (test_bit(FR_PENDING, &req->flags)) {
 467                         list_del(&req->list);
 468                         spin_unlock(&fiq->waitq.lock);
 469                         spin_unlock(&fc->lock);
 470                         __fuse_put_request(req);
 471                         req->out.h.error = -EINTR;
 472                         return;
 473                 }
 474                 spin_unlock(&fiq->waitq.lock);
 475                 spin_unlock(&fc->lock);
 476         }
 477
 478         /*
 479          * Either request is already in userspace, or it was forced.
 480          * Wait it out.
 481          */
 482         wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
 483 }
 484
 485 static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
 486 {
 487         struct fuse_iqueue *fiq = &fc->iq;
 488
 489         BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
 490         spin_lock(&fc->lock);
 491         spin_lock(&fiq->waitq.lock);
 492         if (!fiq->connected) {
 493                 spin_unlock(&fc->lock);
 494                 spin_unlock(&fiq->waitq.lock);
 495                 req->out.h.error = -ENOTCONN;
 496         } else {
 497                 req->in.h.unique = fuse_get_unique(fiq);
 498                 queue_request(fiq, req);
 499                 /* acquire extra reference, since request is still needed
 500                    after request_end() */
 501                 __fuse_get_request(req);
 502                 spin_unlock(&fiq->waitq.lock);
 503                 spin_unlock(&fc->lock);
 504
 505                 request_wait_answer(fc, req);
 506                 /* Pairs with smp_wmb() in request_end() */
 507                 smp_rmb();
 508         }
 509 }
 510
 511 void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
 512 {
 513         __set_bit(FR_ISREPLY, &req->flags);
 514         if (!test_bit(FR_WAITING, &req->flags)) {
 515                 __set_bit(FR_WAITING, &req->flags);
 516                 atomic_inc(&fc->num_waiting);
 517         }
 518         __fuse_request_send(fc, req);
 519 }
 520 EXPORT_SYMBOL_GPL(fuse_request_send);
 521
 522 static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
 523 {
 524         if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
 525                 args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;
 526
 527         if (fc->minor < 9) {
 528                 switch (args->in.h.opcode) {
 529                 case FUSE_LOOKUP:
 530                 case FUSE_CREATE:
 531                 case FUSE_MKNOD:
 532                 case FUSE_MKDIR:
 533                 case FUSE_SYMLINK:
 534                 case FUSE_LINK:
 535                         args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
 536                         break;
 537                 case FUSE_GETATTR:
 538                 case FUSE_SETATTR:
 539                         args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
 540                         break;
 541                 }
 542         }
 543         if (fc->minor < 12) {
 544                 switch (args->in.h.opcode) {
 545                 case FUSE_CREATE:
 546                         args->in.args[0].size = sizeof(struct fuse_open_in);
 547                         break;
 548                 case FUSE_MKNOD:
 549                         args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
 550                         break;
 551                 }
 552         }
 553 }
 554
 555 ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
 556 {
 557         struct fuse_req *req;
 558         ssize_t ret;
 559
 560         req = fuse_get_req(fc, 0);
 561         if (IS_ERR(req))
 562                 return PTR_ERR(req);
 563
 564         /* Needs to be done after fuse_get_req() so that fc->minor is valid */
 565         fuse_adjust_compat(fc, args);
 566
 567         req->in.h.opcode = args->in.h.opcode;
 568         req->in.h.nodeid = args->in.h.nodeid;
 569         req->in.numargs = args->in.numargs;
 570         memcpy(req->in.args, args->in.args,
 571                args->in.numargs * sizeof(struct fuse_in_arg));
 572         req->out.argvar = args->out.argvar;
 573         req->out.numargs = args->out.numargs;
 574         memcpy(req->out.args, args->out.args,
 575                args->out.numargs * sizeof(struct fuse_arg));
 576         fuse_request_send(fc, req);
 577         ret = req->out.h.error;
 578         if (!ret && args->out.argvar) {
 579                 BUG_ON(args->out.numargs != 1);
 580                 ret = req->out.args[0].size;
 581         }
 582         fuse_put_request(fc, req);
 583
 584         return ret;
 585 }
 586
 587 /*
 588  * Called under fc->lock
 589  *
 590  * fc->connected must have been checked previously
 591  */
 592 void fuse_request_send_background_locked(struct fuse_conn *fc,
 593                                          struct fuse_req *req)
 594 {
 595         BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
 596         if (!test_bit(FR_WAITING, &req->flags)) {
 597                 __set_bit(FR_WAITING, &req->flags);
 598                 atomic_inc(&fc->num_waiting);
 599         }
 600         __set_bit(FR_ISREPLY, &req->flags);
 601         fc->num_background++;
 602         if (fc->num_background == fc->max_background)
 603                 fc->blocked = 1;
 604         if (fc->num_background == fc->congestion_threshold &&
 605             fc->bdi_initialized) {
 606                 set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
 607                 set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
 608         }
 609         list_add_tail(&req->list, &fc->bg_queue);
 610         flush_bg_queue(fc);
 611 }
 612
 613 void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
 614 {
 615         BUG_ON(!req->end);
 616         spin_lock(&fc->lock);
 617         if (fc->connected) {
 618                 fuse_request_send_background_locked(fc, req);
 619                 spin_unlock(&fc->lock);
 620         } else {
 621                 spin_unlock(&fc->lock);
 622                 req->out.h.error = -ENOTCONN;
 623                 req->end(fc, req);
 624                 fuse_put_request(fc, req);
 625         }
 626 }
 627 EXPORT_SYMBOL_GPL(fuse_request_send_background);
 628
 629 static int fuse_request_send_notify_reply(struct fuse_conn *fc,
 630                                           struct fuse_req *req, u64 unique)
 631 {
 632         int err = -ENODEV;
 633         struct fuse_iqueue *fiq = &fc->iq;
 634
 635         __clear_bit(FR_ISREPLY, &req->flags);
 636         req->in.h.unique = unique;
 637         spin_lock(&fc->lock);
 638         spin_lock(&fiq->waitq.lock);
 639         if (fiq->connected) {
 640                 queue_request(fiq, req);
 641                 err = 0;
 642         }
 643         spin_unlock(&fiq->waitq.lock);
 644         spin_unlock(&fc->lock);
 645
 646         return err;
 647 }
 648
 649 void fuse_force_forget(struct file *file, u64 nodeid)
 650 {
 651         struct inode *inode = file_inode(file);
 652         struct fuse_conn *fc = get_fuse_conn(inode);
 653         struct fuse_req *req;
 654         struct fuse_forget_in inarg;
 655
 656         memset(&inarg, 0, sizeof(inarg));
 657         inarg.nlookup = 1;
 658         req = fuse_get_req_nofail_nopages(fc, file);
 659         req->in.h.opcode = FUSE_FORGET;
 660         req->in.h.nodeid = nodeid;
 661         req->in.numargs = 1;
 662         req->in.args[0].size = sizeof(inarg);
 663         req->in.args[0].value = &inarg;
 664         __clear_bit(FR_ISREPLY, &req->flags);
 665         __fuse_request_send(fc, req);
 666         /* ignore errors */
 667         fuse_put_request(fc, req);
 668 }
 669
 670 /*
 671  * Lock the request.  Up to the next unlock_request() there mustn't be
 672  * anything that could cause a page-fault.  If the request was already
 673  * aborted bail out.
 674  */
 675 static int lock_request(struct fuse_req *req)
 676 {
 677         int err = 0;
 678         if (req) {
 679                 spin_lock(&req->waitq.lock);
 680                 if (test_bit(FR_ABORTED, &req->flags))
 681                         err = -ENOENT;
 682                 else
 683                         set_bit(FR_LOCKED, &req->flags);
 684                 spin_unlock(&req->waitq.lock);
 685         }
 686         return err;
 687 }
 688
 689 /*
 690  * Unlock request.  If it was aborted while locked, caller is responsible
 691  * for unlocking and ending the request.
 692  */
 693 static int unlock_request(struct fuse_req *req)
 694 {
 695         int err = 0;
 696         if (req) {
 697                 spin_lock(&req->waitq.lock);
 698                 if (test_bit(FR_ABORTED, &req->flags))
 699                         err = -ENOENT;
 700                 else
 701                         clear_bit(FR_LOCKED, &req->flags);
 702                 spin_unlock(&req->waitq.lock);
 703         }
 704         return err;
 705 }
 706
 707 struct fuse_copy_state {
 708         int write;
 709         struct fuse_req *req;
 710         struct iov_iter *iter;
 711         struct pipe_buffer *pipebufs;
 712         struct pipe_buffer *currbuf;
 713         struct pipe_inode_info *pipe;
 714         unsigned long nr_segs;
 715         struct page *pg;
 716         unsigned len;
 717         unsigned offset;
 718         unsigned move_pages:1;
 719 };
 720
 721 static void fuse_copy_init(struct fuse_copy_state *cs, int write,
 722                            struct iov_iter *iter)
 723 {
 724         memset(cs, 0, sizeof(*cs));
 725         cs->write = write;
 726         cs->iter = iter;
 727 }
 728
 729 /* Unmap and put previous page of userspace buffer */
 730 static void fuse_copy_finish(struct fuse_copy_state *cs)
 731 {
 732         if (cs->currbuf) {
 733                 struct pipe_buffer *buf = cs->currbuf;
 734
 735                 if (cs->write)
 736                         buf->len = PAGE_SIZE - cs->len;
 737                 cs->currbuf = NULL;
 738         } else if (cs->pg) {
 739                 if (cs->write) {
 740                         flush_dcache_page(cs->pg);
 741                         set_page_dirty_lock(cs->pg);
 742                 }
 743                 put_page(cs->pg);
 744         }
 745         cs->pg = NULL;
 746 }
 747
 748 /*
 749  * Get another pagefull of userspace buffer, and map it to kernel
 750  * address space, and lock request
 751  */
 752 static int fuse_copy_fill(struct fuse_copy_state *cs)
 753 {
 754         struct page *page;
 755         int err;
 756
 757         err = unlock_request(cs->req);
 758         if (err)
 759                 return err;
 760
 761         fuse_copy_finish(cs);
 762         if (cs->pipebufs) {
 763                 struct pipe_buffer *buf = cs->pipebufs;
 764
 765                 if (!cs->write) {
 766                         err = buf->ops->confirm(cs->pipe, buf);
 767                         if (err)
 768                                 return err;
 769
 770                         BUG_ON(!cs->nr_segs);
 771                         cs->currbuf = buf;
 772                         cs->pg = buf->page;
 773                         cs->offset = buf->offset;
 774                         cs->len = buf->len;
 775                         cs->pipebufs++;
 776                         cs->nr_segs--;
 777                 } else {
 778                         if (cs->nr_segs == cs->pipe->buffers)
 779                                 return -EIO;
 780
 781                         page = alloc_page(GFP_HIGHUSER);
 782                         if (!page)
 783                                 return -ENOMEM;
 784
 785                         buf->page = page;
 786                         buf->offset = 0;
 787                         buf->len = 0;
 788
 789                         cs->currbuf = buf;
 790                         cs->pg = page;
 791                         cs->offset = 0;
 792                         cs->len = PAGE_SIZE;
 793                         cs->pipebufs++;
 794                         cs->nr_segs++;
 795                 }
 796         } else {
 797                 size_t off;
 798                 err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
 799                 if (err < 0)
 800                         return err;
 801                 BUG_ON(!err);
 802                 cs->len = err;
 803                 cs->offset = off;
 804                 cs->pg = page;
 805                 cs->offset = off;
 806                 iov_iter_advance(cs->iter, err);
 807         }
 808
 809         return lock_request(cs->req);
 810 }
 811
 812 /* Do as much copy to/from userspace buffer as we can */
 813 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
 814 {
 815         unsigned ncpy = min(*size, cs->len);
 816         if (val) {
 817                 void *pgaddr = kmap_atomic(cs->pg);
 818                 void *buf = pgaddr + cs->offset;
 819
 820                 if (cs->write)
 821                         memcpy(buf, *val, ncpy);
 822                 else
 823                         memcpy(*val, buf, ncpy);
 824
 825                 kunmap_atomic(pgaddr);
 826                 *val += ncpy;
 827         }
 828         *size -= ncpy;
 829         cs->len -= ncpy;
 830         cs->offset += ncpy;
 831         return ncpy;
 832 }
 833
 834 static int fuse_check_page(struct page *page)
 835 {
 836         if (page_mapcount(page) ||
 837             page->mapping != NULL ||
 838             page_count(page) != 1 ||
 839             (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
 840              ~(1 << PG_locked |
 841                1 << PG_referenced |
 842                1 << PG_uptodate |
 843                1 << PG_lru |
 844                1 << PG_active |
 845                1 << PG_reclaim))) {
 846                 printk(KERN_WARNING "fuse: trying to steal weird page\n");
 847                 printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
 848                 return 1;
 849         }
 850         return 0;
 851 }
 852
 853 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 854 {
 855         int err;
 856         struct page *oldpage = *pagep;
 857         struct page *newpage;
 858         struct pipe_buffer *buf = cs->pipebufs;
 859
 860         err = unlock_request(cs->req);
 861         if (err)
 862                 return err;
 863
 864         fuse_copy_finish(cs);
 865
 866         err = buf->ops->confirm(cs->pipe, buf);
 867         if (err)
 868                 return err;
 869
 870         BUG_ON(!cs->nr_segs);
 871         cs->currbuf = buf;
 872         cs->len = buf->len;
 873         cs->pipebufs++;
 874         cs->nr_segs--;
 875
 876         if (cs->len != PAGE_SIZE)
 877                 goto out_fallback;
 878
 879         if (buf->ops->steal(cs->pipe, buf) != 0)
 880                 goto out_fallback;
 881
 882         newpage = buf->page;
 883
 884         if (!PageUptodate(newpage))
 885                 SetPageUptodate(newpage);
 886
 887         ClearPageMappedToDisk(newpage);
 888
 889         if (fuse_check_page(newpage) != 0)
 890                 goto out_fallback_unlock;
 891
 892         /*
 893          * This is a new and locked page, it shouldn't be mapped or
 894          * have any special flags on it
 895          */
 896         if (WARN_ON(page_mapped(oldpage)))
 897                 goto out_fallback_unlock;
 898         if (WARN_ON(page_has_private(oldpage)))
 899                 goto out_fallback_unlock;
 900         if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
 901                 goto out_fallback_unlock;
 902         if (WARN_ON(PageMlocked(oldpage)))
 903                 goto out_fallback_unlock;
 904
 905         err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
 906         if (err) {
 907                 unlock_page(newpage);
 908                 return err;
 909         }
 910
 911         page_cache_get(newpage);
 912
 913         if (!(buf->flags & PIPE_BUF_FLAG_LRU))
 914                 lru_cache_add_file(newpage);
 915
 916         err = 0;
 917         spin_lock(&cs->req->waitq.lock);
 918         if (test_bit(FR_ABORTED, &cs->req->flags))
 919                 err = -ENOENT;
 920         else
 921                 *pagep = newpage;
 922         spin_unlock(&cs->req->waitq.lock);
 923
 924         if (err) {
 925                 unlock_page(newpage);
 926                 page_cache_release(newpage);
 927                 return err;
 928         }
 929
 930         unlock_page(oldpage);
 931         page_cache_release(oldpage);
 932         cs->len = 0;
 933
 934         return 0;
 935
 936 out_fallback_unlock:
 937         unlock_page(newpage);
 938 out_fallback:
 939         cs->pg = buf->page;
 940         cs->offset = buf->offset;
 941
 942         err = lock_request(cs->req);
 943         if (err)
 944                 return err;
 945
 946         return 1;
 947 }
 948
 949 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
 950                          unsigned offset, unsigned count)
 951 {
 952         struct pipe_buffer *buf;
 953         int err;
 954
 955         if (cs->nr_segs == cs->pipe->buffers)
 956                 return -EIO;
 957
 958         err = unlock_request(cs->req);
 959         if (err)
 960                 return err;
 961
 962         fuse_copy_finish(cs);
 963
 964         buf = cs->pipebufs;
 965         page_cache_get(page);
 966         buf->page = page;
 967         buf->offset = offset;
 968         buf->len = count;
 969
 970         cs->pipebufs++;
 971         cs->nr_segs++;
 972         cs->len = 0;
 973
 974         return 0;
 975 }
 976
 977 /*
 978  * Copy a page in the request to/from the userspace buffer.  Must be
 979  * done atomically
 980  */
 981 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
 982                           unsigned offset, unsigned count, int zeroing)
 983 {
 984         int err;
 985         struct page *page = *pagep;
 986
 987         if (page && zeroing && count < PAGE_SIZE)
 988                 clear_highpage(page);
 989
 990         while (count) {
 991                 if (cs->write && cs->pipebufs && page) {
 992                         return fuse_ref_page(cs, page, offset, count);
 993                 } else if (!cs->len) {
 994                         if (cs->move_pages && page &&
 995                             offset == 0 && count == PAGE_SIZE) {
 996                                 err = fuse_try_move_page(cs, pagep);
 997                                 if (err <= 0)
 998                                         return err;
 999                         } else {
1000                                 err = fuse_copy_fill(cs);
1001                                 if (err)
1002                                         return err;
1003                         }
1004                 }
1005                 if (page) {
1006                         void *mapaddr = kmap_atomic(page);
1007                         void *buf = mapaddr + offset;
1008                         offset += fuse_copy_do(cs, &buf, &count);
1009                         kunmap_atomic(mapaddr);
1010                 } else
1011                         offset += fuse_copy_do(cs, NULL, &count);
1012         }
1013         if (page && !cs->write)
1014                 flush_dcache_page(page);
1015         return 0;
1016 }
1017
1018 /* Copy pages in the request to/from userspace buffer */
1019 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
1020                            int zeroing)
1021 {
1022         unsigned i;
1023         struct fuse_req *req = cs->req;
1024
1025         for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
1026                 int err;
1027                 unsigned offset = req->page_descs[i].offset;
1028                 unsigned count = min(nbytes, req->page_descs[i].length);
1029
1030                 err = fuse_copy_page(cs, &req->pages[i], offset, count,
1031                                      zeroing);
1032                 if (err)
1033                         return err;
1034
1035                 nbytes -= count;
1036         }
1037         return 0;
1038 }
1039
1040 /* Copy a single argument in the request to/from userspace buffer */
1041 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1042 {
1043         while (size) {
1044                 if (!cs->len) {
1045                         int err = fuse_copy_fill(cs);
1046                         if (err)
1047                                 return err;
1048                 }
1049                 fuse_copy_do(cs, &val, &size);
1050         }
1051         return 0;
1052 }
1053
1054 /* Copy request arguments to/from userspace buffer */
1055 static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1056                           unsigned argpages, struct fuse_arg *args,
1057                           int zeroing)
1058 {
1059         int err = 0;
1060         unsigned i;
1061
1062         for (i = 0; !err && i < numargs; i++)  {
1063                 struct fuse_arg *arg = &args[i];
1064                 if (i == numargs - 1 && argpages)
1065                         err = fuse_copy_pages(cs, arg->size, zeroing);
1066                 else
1067                         err = fuse_copy_one(cs, arg->value, arg->size);
1068         }
1069         return err;
1070 }
1071
1072 static int forget_pending(struct fuse_iqueue *fiq)
1073 {
1074         return fiq->forget_list_head.next != NULL;
1075 }
1076
1077 static int request_pending(struct fuse_iqueue *fiq)
1078 {
1079         return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1080                 forget_pending(fiq);
1081 }
1082
1083 /* Wait until a request is available on the pending list */
1084 static void request_wait(struct fuse_conn *fc)
1085 __releases(fc->iq.waitq.lock)
1086 __releases(fc->lock)
1087 __acquires(fc->lock)
1088 __acquires(fc->iq.waitq.lock)
1089 {
1090         struct fuse_iqueue *fiq = &fc->iq;
1091         DECLARE_WAITQUEUE(wait, current);
1092
1093         add_wait_queue_exclusive(&fiq->waitq, &wait);
1094         while (fiq->connected && !request_pending(fiq)) {
1095                 set_current_state(TASK_INTERRUPTIBLE);
1096                 if (signal_pending(current))
1097                         break;
1098
1099                 spin_unlock(&fiq->waitq.lock);
1100                 spin_unlock(&fc->lock);
1101                 schedule();
1102                 spin_lock(&fc->lock);
1103                 spin_lock(&fiq->waitq.lock);
1104         }
1105         set_current_state(TASK_RUNNING);
1106         remove_wait_queue(&fiq->waitq, &wait);
1107 }
1108
1109 /*
1110  * Transfer an interrupt request to userspace
1111  *
1112  * Unlike other requests this is assembled on demand, without a need
1113  * to allocate a separate fuse_req structure.
1114  *
1115  * Called with fc->lock held, releases it
1116  */
1117 static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_copy_state *cs,
1118                                size_t nbytes, struct fuse_req *req)
1119 __releases(fc->iq.waitq.lock)
1120 __releases(fc->lock)
1121 {
1122         struct fuse_iqueue *fiq = &fc->iq;
1123         struct fuse_in_header ih;
1124         struct fuse_interrupt_in arg;
1125         unsigned reqsize = sizeof(ih) + sizeof(arg);
1126         int err;
1127
1128         list_del_init(&req->intr_entry);
1129         req->intr_unique = fuse_get_unique(fiq);
1130         memset(&ih, 0, sizeof(ih));
1131         memset(&arg, 0, sizeof(arg));
1132         ih.len = reqsize;
1133         ih.opcode = FUSE_INTERRUPT;
1134         ih.unique = req->intr_unique;
1135         arg.unique = req->in.h.unique;
1136
1137         spin_unlock(&fiq->waitq.lock);
1138         spin_unlock(&fc->lock);
1139         if (nbytes < reqsize)
1140                 return -EINVAL;
1141
1142         err = fuse_copy_one(cs, &ih, sizeof(ih));
1143         if (!err)
1144                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1145         fuse_copy_finish(cs);
1146
1147         return err ? err : reqsize;
1148 }
1149
1150 static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
1151                                                unsigned max,
1152                                                unsigned *countp)
1153 {
1154         struct fuse_forget_link *head = fiq->forget_list_head.next;
1155         struct fuse_forget_link **newhead = &head;
1156         unsigned count;
1157
1158         for (count = 0; *newhead != NULL && count < max; count++)
1159                 newhead = &(*newhead)->next;
1160
1161         fiq->forget_list_head.next = *newhead;
1162         *newhead = NULL;
1163         if (fiq->forget_list_head.next == NULL)
1164                 fiq->forget_list_tail = &fiq->forget_list_head;
1165
1166         if (countp != NULL)
1167                 *countp = count;
1168
1169         return head;
1170 }
1171
1172 static int fuse_read_single_forget(struct fuse_conn *fc,
1173                                    struct fuse_copy_state *cs,
1174                                    size_t nbytes)
1175 __releases(fc->iq.waitq.lock)
1176 __releases(fc->lock)
1177 {
1178         int err;
1179         struct fuse_iqueue *fiq = &fc->iq;
1180         struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
1181         struct fuse_forget_in arg = {
1182                 .nlookup = forget->forget_one.nlookup,
1183         };
1184         struct fuse_in_header ih = {
1185                 .opcode = FUSE_FORGET,
1186                 .nodeid = forget->forget_one.nodeid,
1187                 .unique = fuse_get_unique(fiq),
1188                 .len = sizeof(ih) + sizeof(arg),
1189         };
1190
1191         spin_unlock(&fiq->waitq.lock);
1192         spin_unlock(&fc->lock);
1193         kfree(forget);
1194         if (nbytes < ih.len)
1195                 return -EINVAL;
1196
1197         err = fuse_copy_one(cs, &ih, sizeof(ih));
1198         if (!err)
1199                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1200         fuse_copy_finish(cs);
1201
1202         if (err)
1203                 return err;
1204
1205         return ih.len;
1206 }
1207
1208 static int fuse_read_batch_forget(struct fuse_conn *fc,
1209                                    struct fuse_copy_state *cs, size_t nbytes)
1210 __releases(fc->iq.waitq.lock)
1211 __releases(fc->lock)
1212 {
1213         int err;
1214         unsigned max_forgets;
1215         unsigned count;
1216         struct fuse_forget_link *head;
1217         struct fuse_iqueue *fiq = &fc->iq;
1218         struct fuse_batch_forget_in arg = { .count = 0 };
1219         struct fuse_in_header ih = {
1220                 .opcode = FUSE_BATCH_FORGET,
1221                 .unique = fuse_get_unique(fiq),
1222                 .len = sizeof(ih) + sizeof(arg),
1223         };
1224
1225         if (nbytes < ih.len) {
1226                 spin_unlock(&fiq->waitq.lock);
1227                 spin_unlock(&fc->lock);
1228                 return -EINVAL;
1229         }
1230
1231         max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1232         head = dequeue_forget(fiq, max_forgets, &count);
1233         spin_unlock(&fiq->waitq.lock);
1234         spin_unlock(&fc->lock);
1235
1236         arg.count = count;
1237         ih.len += count * sizeof(struct fuse_forget_one);
1238         err = fuse_copy_one(cs, &ih, sizeof(ih));
1239         if (!err)
1240                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1241
1242         while (head) {
1243                 struct fuse_forget_link *forget = head;
1244
1245                 if (!err) {
1246                         err = fuse_copy_one(cs, &forget->forget_one,
1247                                             sizeof(forget->forget_one));
1248                 }
1249                 head = forget->next;
1250                 kfree(forget);
1251         }
1252
1253         fuse_copy_finish(cs);
1254
1255         if (err)
1256                 return err;
1257
1258         return ih.len;
1259 }
1260
1261 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_copy_state *cs,
1262                             size_t nbytes)
1263 __releases(fc->iq.waitq.lock)
1264 __releases(fc->lock)
1265 {
1266         struct fuse_iqueue *fiq = &fc->iq;
1267
1268         if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
1269                 return fuse_read_single_forget(fc, cs, nbytes);
1270         else
1271                 return fuse_read_batch_forget(fc, cs, nbytes);
1272 }
1273
1274 /*
1275  * Read a single request into the userspace filesystem's buffer.  This
1276  * function waits until a request is available, then removes it from
1277  * the pending list and copies request data to userspace buffer.  If
1278  * no reply is needed (FORGET) or request has been aborted or there
1279  * was an error during the copying then it's finished by calling
1280  * request_end().  Otherwise add it to the processing list, and set
1281  * the 'sent' flag.
1282  */
1283 static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file,
1284                                 struct fuse_copy_state *cs, size_t nbytes)
1285 {
1286         int err;
1287         struct fuse_iqueue *fiq = &fc->iq;
1288         struct fuse_req *req;
1289         struct fuse_in *in;
1290         unsigned reqsize;
1291
1292  restart:
1293         spin_lock(&fc->lock);
1294         spin_lock(&fiq->waitq.lock);
1295         err = -EAGAIN;
1296         if ((file->f_flags & O_NONBLOCK) && fiq->connected &&
1297             !request_pending(fiq))
1298                 goto err_unlock;
1299
1300         request_wait(fc);
1301         err = -ENODEV;
1302         if (!fiq->connected)
1303                 goto err_unlock;
1304         err = -ERESTARTSYS;
1305         if (!request_pending(fiq))
1306                 goto err_unlock;
1307
1308         if (!list_empty(&fiq->interrupts)) {
1309                 req = list_entry(fiq->interrupts.next, struct fuse_req,
1310                                  intr_entry);
1311                 return fuse_read_interrupt(fc, cs, nbytes, req);
1312         }
1313
1314         if (forget_pending(fiq)) {
1315                 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
1316                         return fuse_read_forget(fc, cs, nbytes);
1317
1318                 if (fiq->forget_batch <= -8)
1319                         fiq->forget_batch = 16;
1320         }
1321
1322         req = list_entry(fiq->pending.next, struct fuse_req, list);
1323         clear_bit(FR_PENDING, &req->flags);
1324         list_del_init(&req->list);
1325         spin_unlock(&fiq->waitq.lock);
1326
1327         list_add(&req->list, &fc->io);
1328
1329         in = &req->in;
1330         reqsize = in->h.len;
1331         /* If request is too large, reply with an error and restart the read */
1332         if (nbytes < reqsize) {
1333                 req->out.h.error = -EIO;
1334                 /* SETXATTR is special, since it may contain too large data */
1335                 if (in->h.opcode == FUSE_SETXATTR)
1336                         req->out.h.error = -E2BIG;
1337                 request_end(fc, req);
1338                 goto restart;
1339         }
1340         spin_unlock(&fc->lock);
1341         cs->req = req;
1342         err = fuse_copy_one(cs, &in->h, sizeof(in->h));
1343         if (!err)
1344                 err = fuse_copy_args(cs, in->numargs, in->argpages,
1345                                      (struct fuse_arg *) in->args, 0);
1346         fuse_copy_finish(cs);
1347         spin_lock(&fc->lock);
1348         clear_bit(FR_LOCKED, &req->flags);
1349         if (!fc->connected) {
1350                 request_end(fc, req);
1351                 return -ENODEV;
1352         }
1353         if (err) {
1354                 req->out.h.error = -EIO;
1355                 request_end(fc, req);
1356                 return err;
1357         }
1358         if (!test_bit(FR_ISREPLY, &req->flags)) {
1359                 request_end(fc, req);
1360         } else {
1361                 set_bit(FR_SENT, &req->flags);
1362                 list_move_tail(&req->list, &fc->processing);
1363                 if (test_bit(FR_INTERRUPTED, &req->flags))
1364                         queue_interrupt(fiq, req);
1365                 spin_unlock(&fc->lock);
1366         }
1367         return reqsize;
1368
1369  err_unlock:
1370         spin_unlock(&fiq->waitq.lock);
1371         spin_unlock(&fc->lock);
1372         return err;
1373 }
1374
1375 static int fuse_dev_open(struct inode *inode, struct file *file)
1376 {
1377         /*
1378          * The fuse device's file's private_data is used to hold
1379          * the fuse_conn(ection) when it is mounted, and is used to
1380          * keep track of whether the file has been mounted already.
1381          */
1382         file->private_data = NULL;
1383         return 0;
1384 }
1385
1386 static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
1387 {
1388         struct fuse_copy_state cs;
1389         struct file *file = iocb->ki_filp;
1390         struct fuse_conn *fc = fuse_get_conn(file);
1391         if (!fc)
1392                 return -EPERM;
1393
1394         if (!iter_is_iovec(to))
1395                 return -EINVAL;
1396
1397         fuse_copy_init(&cs, 1, to);
1398
1399         return fuse_dev_do_read(fc, file, &cs, iov_iter_count(to));
1400 }
1401
1402 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1403                                     struct pipe_inode_info *pipe,
1404                                     size_t len, unsigned int flags)
1405 {
1406         int ret;
1407         int page_nr = 0;
1408         int do_wakeup = 0;
1409         struct pipe_buffer *bufs;
1410         struct fuse_copy_state cs;
1411         struct fuse_conn *fc = fuse_get_conn(in);
1412         if (!fc)
1413                 return -EPERM;
1414
1415         bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1416         if (!bufs)
1417                 return -ENOMEM;
1418
1419         fuse_copy_init(&cs, 1, NULL);
1420         cs.pipebufs = bufs;
1421         cs.pipe = pipe;
1422         ret = fuse_dev_do_read(fc, in, &cs, len);
1423         if (ret < 0)
1424                 goto out;
1425
1426         ret = 0;
1427         pipe_lock(pipe);
1428
1429         if (!pipe->readers) {
1430                 send_sig(SIGPIPE, current, 0);
1431                 if (!ret)
1432                         ret = -EPIPE;
1433                 goto out_unlock;
1434         }
1435
1436         if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1437                 ret = -EIO;
1438                 goto out_unlock;
1439         }
1440
1441         while (page_nr < cs.nr_segs) {
1442                 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1443                 struct pipe_buffer *buf = pipe->bufs + newbuf;
1444
1445                 buf->page = bufs[page_nr].page;
1446                 buf->offset = bufs[page_nr].offset;
1447                 buf->len = bufs[page_nr].len;
1448                 /*
1449                  * Need to be careful about this.  Having buf->ops in module
1450                  * code can Oops if the buffer persists after module unload.
1451                  */
1452                 buf->ops = &nosteal_pipe_buf_ops;
1453
1454                 pipe->nrbufs++;
1455                 page_nr++;
1456                 ret += buf->len;
1457
1458                 if (pipe->files)
1459                         do_wakeup = 1;
1460         }
1461
1462 out_unlock:
1463         pipe_unlock(pipe);
1464
1465         if (do_wakeup) {
1466                 smp_mb();
1467                 if (waitqueue_active(&pipe->wait))
1468                         wake_up_interruptible(&pipe->wait);
1469                 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
1470         }
1471
1472 out:
1473         for (; page_nr < cs.nr_segs; page_nr++)
1474                 page_cache_release(bufs[page_nr].page);
1475
1476         kfree(bufs);
1477         return ret;
1478 }
1479
1480 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1481                             struct fuse_copy_state *cs)
1482 {
1483         struct fuse_notify_poll_wakeup_out outarg;
1484         int err = -EINVAL;
1485
1486         if (size != sizeof(outarg))
1487                 goto err;
1488
1489         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1490         if (err)
1491                 goto err;
1492
1493         fuse_copy_finish(cs);
1494         return fuse_notify_poll_wakeup(fc, &outarg);
1495
1496 err:
1497         fuse_copy_finish(cs);
1498         return err;
1499 }
1500
1501 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1502                                    struct fuse_copy_state *cs)
1503 {
1504         struct fuse_notify_inval_inode_out outarg;
1505         int err = -EINVAL;
1506
1507         if (size != sizeof(outarg))
1508                 goto err;
1509
1510         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1511         if (err)
1512                 goto err;
1513         fuse_copy_finish(cs);
1514
1515         down_read(&fc->killsb);
1516         err = -ENOENT;
1517         if (fc->sb) {
1518                 err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1519                                                outarg.off, outarg.len);
1520         }
1521         up_read(&fc->killsb);
1522         return err;
1523
1524 err:
1525         fuse_copy_finish(cs);
1526         return err;
1527 }
1528
1529 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1530                                    struct fuse_copy_state *cs)
1531 {
1532         struct fuse_notify_inval_entry_out outarg;
1533         int err = -ENOMEM;
1534         char *buf;
1535         struct qstr name;
1536
1537         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1538         if (!buf)
1539                 goto err;
1540
1541         err = -EINVAL;
1542         if (size < sizeof(outarg))
1543                 goto err;
1544
1545         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1546         if (err)
1547                 goto err;
1548
1549         err = -ENAMETOOLONG;
1550         if (outarg.namelen > FUSE_NAME_MAX)
1551                 goto err;
1552
1553         err = -EINVAL;
1554         if (size != sizeof(outarg) + outarg.namelen + 1)
1555                 goto err;
1556
1557         name.name = buf;
1558         name.len = outarg.namelen;
1559         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1560         if (err)
1561                 goto err;
1562         fuse_copy_finish(cs);
1563         buf[outarg.namelen] = 0;
1564         name.hash = full_name_hash(name.name, name.len);
1565
1566         down_read(&fc->killsb);
1567         err = -ENOENT;
1568         if (fc->sb)
1569                 err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
1570         up_read(&fc->killsb);
1571         kfree(buf);
1572         return err;
1573
1574 err:
1575         kfree(buf);
1576         fuse_copy_finish(cs);
1577         return err;
1578 }
1579
1580 static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1581                               struct fuse_copy_state *cs)
1582 {
1583         struct fuse_notify_delete_out outarg;
1584         int err = -ENOMEM;
1585         char *buf;
1586         struct qstr name;
1587
1588         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1589         if (!buf)
1590                 goto err;
1591
1592         err = -EINVAL;
1593         if (size < sizeof(outarg))
1594                 goto err;
1595
1596         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1597         if (err)
1598                 goto err;
1599
1600         err = -ENAMETOOLONG;
1601         if (outarg.namelen > FUSE_NAME_MAX)
1602                 goto err;
1603
1604         err = -EINVAL;
1605         if (size != sizeof(outarg) + outarg.namelen + 1)
1606                 goto err;
1607
1608         name.name = buf;
1609         name.len = outarg.namelen;
1610         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1611         if (err)
1612                 goto err;
1613         fuse_copy_finish(cs);
1614         buf[outarg.namelen] = 0;
1615         name.hash = full_name_hash(name.name, name.len);
1616
1617         down_read(&fc->killsb);
1618         err = -ENOENT;
1619         if (fc->sb)
1620                 err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
1621                                                outarg.child, &name);
1622         up_read(&fc->killsb);
1623         kfree(buf);
1624         return err;
1625
1626 err:
1627         kfree(buf);
1628         fuse_copy_finish(cs);
1629         return err;
1630 }
1631
1632 static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1633                              struct fuse_copy_state *cs)
1634 {
1635         struct fuse_notify_store_out outarg;
1636         struct inode *inode;
1637         struct address_space *mapping;
1638         u64 nodeid;
1639         int err;
1640         pgoff_t index;
1641         unsigned int offset;
1642         unsigned int num;
1643         loff_t file_size;
1644         loff_t end;
1645
1646         err = -EINVAL;
1647         if (size < sizeof(outarg))
1648                 goto out_finish;
1649
1650         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1651         if (err)
1652                 goto out_finish;
1653
1654         err = -EINVAL;
1655         if (size - sizeof(outarg) != outarg.size)
1656                 goto out_finish;
1657
1658         nodeid = outarg.nodeid;
1659
1660         down_read(&fc->killsb);
1661
1662         err = -ENOENT;
1663         if (!fc->sb)
1664                 goto out_up_killsb;
1665
1666         inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1667         if (!inode)
1668                 goto out_up_killsb;
1669
1670         mapping = inode->i_mapping;
1671         index = outarg.offset >> PAGE_CACHE_SHIFT;
1672         offset = outarg.offset & ~PAGE_CACHE_MASK;
1673         file_size = i_size_read(inode);
1674         end = outarg.offset + outarg.size;
1675         if (end > file_size) {
1676                 file_size = end;
1677                 fuse_write_update_size(inode, file_size);
1678         }
1679
1680         num = outarg.size;
1681         while (num) {
1682                 struct page *page;
1683                 unsigned int this_num;
1684
1685                 err = -ENOMEM;
1686                 page = find_or_create_page(mapping, index,
1687                                            mapping_gfp_mask(mapping));
1688                 if (!page)
1689                         goto out_iput;
1690
1691                 this_num = min_t(unsigned, num, PAGE_CACHE_SIZE - offset);
1692                 err = fuse_copy_page(cs, &page, offset, this_num, 0);
1693                 if (!err && offset == 0 &&
1694                     (this_num == PAGE_CACHE_SIZE || file_size == end))
1695                         SetPageUptodate(page);
1696                 unlock_page(page);
1697                 page_cache_release(page);
1698
1699                 if (err)
1700                         goto out_iput;
1701
1702                 num -= this_num;
1703                 offset = 0;
1704                 index++;
1705         }
1706
1707         err = 0;
1708
1709 out_iput:
1710         iput(inode);
1711 out_up_killsb:
1712         up_read(&fc->killsb);
1713 out_finish:
1714         fuse_copy_finish(cs);
1715         return err;
1716 }
1717
1718 static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
1719 {
1720         release_pages(req->pages, req->num_pages, false);
1721 }
1722
1723 static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
1724                          struct fuse_notify_retrieve_out *outarg)
1725 {
1726         int err;
1727         struct address_space *mapping = inode->i_mapping;
1728         struct fuse_req *req;
1729         pgoff_t index;
1730         loff_t file_size;
1731         unsigned int num;
1732         unsigned int offset;
1733         size_t total_len = 0;
1734         int num_pages;
1735
1736         offset = outarg->offset & ~PAGE_CACHE_MASK;
1737         file_size = i_size_read(inode);
1738
1739         num = outarg->size;
1740         if (outarg->offset > file_size)
1741                 num = 0;
1742         else if (outarg->offset + num > file_size)
1743                 num = file_size - outarg->offset;
1744
1745         num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1746         num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);
1747
1748         req = fuse_get_req(fc, num_pages);
1749         if (IS_ERR(req))
1750                 return PTR_ERR(req);
1751
1752         req->in.h.opcode = FUSE_NOTIFY_REPLY;
1753         req->in.h.nodeid = outarg->nodeid;
1754         req->in.numargs = 2;
1755         req->in.argpages = 1;
1756         req->page_descs[0].offset = offset;
1757         req->end = fuse_retrieve_end;
1758
1759         index = outarg->offset >> PAGE_CACHE_SHIFT;
1760
1761         while (num && req->num_pages < num_pages) {
1762                 struct page *page;
1763                 unsigned int this_num;
1764
1765                 page = find_get_page(mapping, index);
1766                 if (!page)
1767                         break;
1768
1769                 this_num = min_t(unsigned, num, PAGE_CACHE_SIZE - offset);
1770                 req->pages[req->num_pages] = page;
1771                 req->page_descs[req->num_pages].length = this_num;
1772                 req->num_pages++;
1773
1774                 offset = 0;
1775                 num -= this_num;
1776                 total_len += this_num;
1777                 index++;
1778         }
1779         req->misc.retrieve_in.offset = outarg->offset;
1780         req->misc.retrieve_in.size = total_len;
1781         req->in.args[0].size = sizeof(req->misc.retrieve_in);
1782         req->in.args[0].value = &req->misc.retrieve_in;
1783         req->in.args[1].size = total_len;
1784
1785         err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
1786         if (err)
1787                 fuse_retrieve_end(fc, req);
1788
1789         return err;
1790 }
1791
1792 static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1793                                 struct fuse_copy_state *cs)
1794 {
1795         struct fuse_notify_retrieve_out outarg;
1796         struct inode *inode;
1797         int err;
1798
1799         err = -EINVAL;
1800         if (size != sizeof(outarg))
1801                 goto copy_finish;
1802
1803         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1804         if (err)
1805                 goto copy_finish;
1806
1807         fuse_copy_finish(cs);
1808
1809         down_read(&fc->killsb);
1810         err = -ENOENT;
1811         if (fc->sb) {
1812                 u64 nodeid = outarg.nodeid;
1813
1814                 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1815                 if (inode) {
1816                         err = fuse_retrieve(fc, inode, &outarg);
1817                         iput(inode);
1818                 }
1819         }
1820         up_read(&fc->killsb);
1821
1822         return err;
1823
1824 copy_finish:
1825         fuse_copy_finish(cs);
1826         return err;
1827 }
1828
1829 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1830                        unsigned int size, struct fuse_copy_state *cs)
1831 {
1832         /* Don't try to move pages (yet) */
1833         cs->move_pages = 0;
1834
1835         switch (code) {
1836         case FUSE_NOTIFY_POLL:
1837                 return fuse_notify_poll(fc, size, cs);
1838
1839         case FUSE_NOTIFY_INVAL_INODE:
1840                 return fuse_notify_inval_inode(fc, size, cs);
1841
1842         case FUSE_NOTIFY_INVAL_ENTRY:
1843                 return fuse_notify_inval_entry(fc, size, cs);
1844
1845         case FUSE_NOTIFY_STORE:
1846                 return fuse_notify_store(fc, size, cs);
1847
1848         case FUSE_NOTIFY_RETRIEVE:
1849                 return fuse_notify_retrieve(fc, size, cs);
1850
1851         case FUSE_NOTIFY_DELETE:
1852                 return fuse_notify_delete(fc, size, cs);
1853
1854         default:
1855                 fuse_copy_finish(cs);
1856                 return -EINVAL;
1857         }
1858 }
1859
1860 /* Look up request on processing list by unique ID */
1861 static struct fuse_req *request_find(struct fuse_conn *fc, u64 unique)
1862 {
1863         struct fuse_req *req;
1864
1865         list_for_each_entry(req, &fc->processing, list) {
1866                 if (req->in.h.unique == unique || req->intr_unique == unique)
1867                         return req;
1868         }
1869         return NULL;
1870 }
1871
1872 static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1873                          unsigned nbytes)
1874 {
1875         unsigned reqsize = sizeof(struct fuse_out_header);
1876
1877         if (out->h.error)
1878                 return nbytes != reqsize ? -EINVAL : 0;
1879
1880         reqsize += len_args(out->numargs, out->args);
1881
1882         if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1883                 return -EINVAL;
1884         else if (reqsize > nbytes) {
1885                 struct fuse_arg *lastarg = &out->args[out->numargs-1];
1886                 unsigned diffsize = reqsize - nbytes;
1887                 if (diffsize > lastarg->size)
1888                         return -EINVAL;
1889                 lastarg->size -= diffsize;
1890         }
1891         return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1892                               out->page_zeroing);
1893 }
1894
1895 /*
1896  * Write a single reply to a request.  First the header is copied from
1897  * the write buffer.  The request is then searched on the processing
1898  * list by the unique ID found in the header.  If found, then remove
1899  * it from the list and copy the rest of the buffer to the request.
1900  * The request is finished by calling request_end()
1901  */
1902 static ssize_t fuse_dev_do_write(struct fuse_conn *fc,
1903                                  struct fuse_copy_state *cs, size_t nbytes)
1904 {
1905         int err;
1906         struct fuse_req *req;
1907         struct fuse_out_header oh;
1908
1909         if (nbytes < sizeof(struct fuse_out_header))
1910                 return -EINVAL;
1911
1912         err = fuse_copy_one(cs, &oh, sizeof(oh));
1913         if (err)
1914                 goto err_finish;
1915
1916         err = -EINVAL;
1917         if (oh.len != nbytes)
1918                 goto err_finish;
1919
1920         /*
1921          * Zero oh.unique indicates unsolicited notification message
1922          * and error contains notification code.
1923          */
1924         if (!oh.unique) {
1925                 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1926                 return err ? err : nbytes;
1927         }
1928
1929         err = -EINVAL;
1930         if (oh.error <= -1000 || oh.error > 0)
1931                 goto err_finish;
1932
1933         spin_lock(&fc->lock);
1934         err = -ENOENT;
1935         if (!fc->connected)
1936                 goto err_unlock;
1937
1938         req = request_find(fc, oh.unique);
1939         if (!req)
1940                 goto err_unlock;
1941
1942         /* Is it an interrupt reply? */
1943         if (req->intr_unique == oh.unique) {
1944                 err = -EINVAL;
1945                 if (nbytes != sizeof(struct fuse_out_header))
1946                         goto err_unlock;
1947
1948                 if (oh.error == -ENOSYS)
1949                         fc->no_interrupt = 1;
1950                 else if (oh.error == -EAGAIN)
1951                         queue_interrupt(&fc->iq, req);
1952
1953                 spin_unlock(&fc->lock);
1954                 fuse_copy_finish(cs);
1955                 return nbytes;
1956         }
1957
1958         clear_bit(FR_SENT, &req->flags);
1959         list_move(&req->list, &fc->io);
1960         req->out.h = oh;
1961         set_bit(FR_LOCKED, &req->flags);
1962         cs->req = req;
1963         if (!req->out.page_replace)
1964                 cs->move_pages = 0;
1965         spin_unlock(&fc->lock);
1966
1967         err = copy_out_args(cs, &req->out, nbytes);
1968         fuse_copy_finish(cs);
1969
1970         spin_lock(&fc->lock);
1971         clear_bit(FR_LOCKED, &req->flags);
1972         if (!fc->connected)
1973                 err = -ENOENT;
1974         else if (err)
1975                 req->out.h.error = -EIO;
1976         request_end(fc, req);
1977
1978         return err ? err : nbytes;
1979
1980  err_unlock:
1981         spin_unlock(&fc->lock);
1982  err_finish:
1983         fuse_copy_finish(cs);
1984         return err;
1985 }
1986
1987 static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
1988 {
1989         struct fuse_copy_state cs;
1990         struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp);
1991         if (!fc)
1992                 return -EPERM;
1993
1994         if (!iter_is_iovec(from))
1995                 return -EINVAL;
1996
1997         fuse_copy_init(&cs, 0, from);
1998
1999         return fuse_dev_do_write(fc, &cs, iov_iter_count(from));
2000 }
2001
2002 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
2003                                      struct file *out, loff_t *ppos,
2004                                      size_t len, unsigned int flags)
2005 {
2006         unsigned nbuf;
2007         unsigned idx;
2008         struct pipe_buffer *bufs;
2009         struct fuse_copy_state cs;
2010         struct fuse_conn *fc;
2011         size_t rem;
2012         ssize_t ret;
2013
2014         fc = fuse_get_conn(out);
2015         if (!fc)
2016                 return -EPERM;
2017
2018         bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
2019         if (!bufs)
2020                 return -ENOMEM;
2021
2022         pipe_lock(pipe);
2023         nbuf = 0;
2024         rem = 0;
2025         for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
2026                 rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
2027
2028         ret = -EINVAL;
2029         if (rem < len) {
2030                 pipe_unlock(pipe);
2031                 goto out;
2032         }
2033
2034         rem = len;
2035         while (rem) {
2036                 struct pipe_buffer *ibuf;
2037                 struct pipe_buffer *obuf;
2038
2039                 BUG_ON(nbuf >= pipe->buffers);
2040                 BUG_ON(!pipe->nrbufs);
2041                 ibuf = &pipe->bufs[pipe->curbuf];
2042                 obuf = &bufs[nbuf];
2043
2044                 if (rem >= ibuf->len) {
2045                         *obuf = *ibuf;
2046                         ibuf->ops = NULL;
2047                         pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
2048                         pipe->nrbufs--;
2049                 } else {
2050                         ibuf->ops->get(pipe, ibuf);
2051                         *obuf = *ibuf;
2052                         obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2053                         obuf->len = rem;
2054                         ibuf->offset += obuf->len;
2055                         ibuf->len -= obuf->len;
2056                 }
2057                 nbuf++;
2058                 rem -= obuf->len;
2059         }
2060         pipe_unlock(pipe);
2061
2062         fuse_copy_init(&cs, 0, NULL);
2063         cs.pipebufs = bufs;
2064         cs.nr_segs = nbuf;
2065         cs.pipe = pipe;
2066
2067         if (flags & SPLICE_F_MOVE)
2068                 cs.move_pages = 1;
2069
2070         ret = fuse_dev_do_write(fc, &cs, len);
2071
2072         for (idx = 0; idx < nbuf; idx++) {
2073                 struct pipe_buffer *buf = &bufs[idx];
2074                 buf->ops->release(pipe, buf);
2075         }
2076 out:
2077         kfree(bufs);
2078         return ret;
2079 }
2080
2081 static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
2082 {
2083         unsigned mask = POLLOUT | POLLWRNORM;
2084         struct fuse_iqueue *fiq;
2085         struct fuse_conn *fc = fuse_get_conn(file);
2086         if (!fc)
2087                 return POLLERR;
2088
2089         fiq = &fc->iq;
2090         poll_wait(file, &fiq->waitq, wait);
2091
2092         spin_lock(&fc->lock);
2093         spin_lock(&fiq->waitq.lock);
2094         if (!fiq->connected)
2095                 mask = POLLERR;
2096         else if (request_pending(fiq))
2097                 mask |= POLLIN | POLLRDNORM;
2098         spin_unlock(&fiq->waitq.lock);
2099         spin_unlock(&fc->lock);
2100
2101         return mask;
2102 }
2103
2104 /*
2105  * Abort all requests on the given list (pending or processing)
2106  *
2107  * This function releases and reacquires fc->lock
2108  */
2109 static void end_requests(struct fuse_conn *fc, struct list_head *head)
2110 __releases(fc->lock)
2111 __acquires(fc->lock)
2112 {
2113         while (!list_empty(head)) {
2114                 struct fuse_req *req;
2115                 req = list_entry(head->next, struct fuse_req, list);
2116                 req->out.h.error = -ECONNABORTED;
2117                 clear_bit(FR_PENDING, &req->flags);
2118                 clear_bit(FR_SENT, &req->flags);
2119                 request_end(fc, req);
2120                 spin_lock(&fc->lock);
2121         }
2122 }
2123
2124 static void end_polls(struct fuse_conn *fc)
2125 {
2126         struct rb_node *p;
2127
2128         p = rb_first(&fc->polled_files);
2129
2130         while (p) {
2131                 struct fuse_file *ff;
2132                 ff = rb_entry(p, struct fuse_file, polled_node);
2133                 wake_up_interruptible_all(&ff->poll_wait);
2134
2135                 p = rb_next(p);
2136         }
2137 }
2138
2139 /*
2140  * Abort all requests.
2141  *
2142  * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2143  * filesystem.
2144  *
2145  * The same effect is usually achievable through killing the filesystem daemon
2146  * and all users of the filesystem.  The exception is the combination of an
2147  * asynchronous request and the tricky deadlock (see
2148  * Documentation/filesystems/fuse.txt).
2149  *
2150  * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2151  * requests, they should be finished off immediately.  Locked requests will be
2152  * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2153  * requests.  It is possible that some request will finish before we can.  This
2154  * is OK, the request will in that case be removed from the list before we touch
2155  * it.
2156  */
2157 void fuse_abort_conn(struct fuse_conn *fc)
2158 {
2159         struct fuse_iqueue *fiq = &fc->iq;
2160
2161         spin_lock(&fc->lock);
2162         if (fc->connected) {
2163                 struct fuse_req *req, *next;
2164                 LIST_HEAD(to_end1);
2165                 LIST_HEAD(to_end2);
2166
2167                 fc->connected = 0;
2168                 fc->blocked = 0;
2169                 fuse_set_initialized(fc);
2170                 list_for_each_entry_safe(req, next, &fc->io, list) {
2171                         req->out.h.error = -ECONNABORTED;
2172                         spin_lock(&req->waitq.lock);
2173                         set_bit(FR_ABORTED, &req->flags);
2174                         if (!test_bit(FR_LOCKED, &req->flags))
2175                                 list_move(&req->list, &to_end1);
2176                         spin_unlock(&req->waitq.lock);
2177                 }
2178                 fc->max_background = UINT_MAX;
2179                 flush_bg_queue(fc);
2180
2181                 spin_lock(&fiq->waitq.lock);
2182                 fiq->connected = 0;
2183                 list_splice_init(&fiq->pending, &to_end2);
2184                 while (forget_pending(fiq))
2185                         kfree(dequeue_forget(fiq, 1, NULL));
2186                 wake_up_all_locked(&fiq->waitq);
2187                 spin_unlock(&fiq->waitq.lock);
2188                 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
2189
2190                 list_splice_init(&fc->processing, &to_end2);
2191                 while (!list_empty(&to_end1)) {
2192                         req = list_first_entry(&to_end1, struct fuse_req, list);
2193                         __fuse_get_request(req);
2194                         request_end(fc, req);
2195                         spin_lock(&fc->lock);
2196                 }
2197                 end_requests(fc, &to_end2);
2198                 end_polls(fc);
2199                 wake_up_all(&fc->blocked_waitq);
2200         }
2201         spin_unlock(&fc->lock);
2202 }
2203 EXPORT_SYMBOL_GPL(fuse_abort_conn);
2204
2205 int fuse_dev_release(struct inode *inode, struct file *file)
2206 {
2207         struct fuse_conn *fc = fuse_get_conn(file);
2208         if (fc) {
2209                 WARN_ON(!list_empty(&fc->io));
2210                 WARN_ON(fc->iq.fasync != NULL);
2211                 fuse_abort_conn(fc);
2212                 fuse_conn_put(fc);
2213         }
2214
2215         return 0;
2216 }
2217 EXPORT_SYMBOL_GPL(fuse_dev_release);
2218
2219 static int fuse_dev_fasync(int fd, struct file *file, int on)
2220 {
2221         struct fuse_conn *fc = fuse_get_conn(file);
2222         if (!fc)
2223                 return -EPERM;
2224
2225         /* No locking - fasync_helper does its own locking */
2226         return fasync_helper(fd, file, on, &fc->iq.fasync);
2227 }
2228
2229 const struct file_operations fuse_dev_operations = {
2230         .owner          = THIS_MODULE,
2231         .open           = fuse_dev_open,
2232         .llseek         = no_llseek,
2233         .read_iter      = fuse_dev_read,
2234         .splice_read    = fuse_dev_splice_read,
2235         .write_iter     = fuse_dev_write,
2236         .splice_write   = fuse_dev_splice_write,
2237         .poll           = fuse_dev_poll,
2238         .release        = fuse_dev_release,
2239         .fasync         = fuse_dev_fasync,
2240 };
2241 EXPORT_SYMBOL_GPL(fuse_dev_operations);
2242
2243 static struct miscdevice fuse_miscdevice = {
2244         .minor = FUSE_MINOR,
2245         .name  = "fuse",
2246         .fops = &fuse_dev_operations,
2247 };
2248
2249 int __init fuse_dev_init(void)
2250 {
2251         int err = -ENOMEM;
2252         fuse_req_cachep = kmem_cache_create("fuse_request",
2253                                             sizeof(struct fuse_req),
2254                                             0, 0, NULL);
2255         if (!fuse_req_cachep)
2256                 goto out;
2257
2258         err = misc_register(&fuse_miscdevice);
2259         if (err)
2260                 goto out_cache_clean;
2261
2262         return 0;
2263
2264  out_cache_clean:
2265         kmem_cache_destroy(fuse_req_cachep);
2266  out:
2267         return err;
2268 }
2269
2270 void fuse_dev_cleanup(void)
2271 {
2272         misc_deregister(&fuse_miscdevice);
2273         kmem_cache_destroy(fuse_req_cachep);
2274 }