drivers/xen/grant-table.c

   1 /******************************************************************************
   2  * grant_table.c
   3  *
   4  * Granting foreign access to our memory reservation.
   5  *
   6  * Copyright (c) 2005-2006, Christopher Clark
   7  * Copyright (c) 2004-2005, K A Fraser
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License version 2
  11  * as published by the Free Software Foundation; or, when distributed
  12  * separately from the Linux kernel or incorporated into other
  13  * software packages, subject to the following license:
  14  *
  15  * Permission is hereby granted, free of charge, to any person obtaining a copy
  16  * of this source file (the "Software"), to deal in the Software without
  17  * restriction, including without limitation the rights to use, copy, modify,
  18  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19  * and to permit persons to whom the Software is furnished to do so, subject to
  20  * the following conditions:
  21  *
  22  * The above copyright notice and this permission notice shall be included in
  23  * all copies or substantial portions of the Software.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31  * IN THE SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/mm.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/io.h>
  41
  42 #include <xen/xen.h>
  43 #include <xen/interface/xen.h>
  44 #include <xen/page.h>
  45 #include <xen/grant_table.h>
  46 #include <xen/interface/memory.h>
  47 #include <asm/xen/hypercall.h>
  48
  49 #include <asm/pgtable.h>
  50 #include <asm/sync_bitops.h>
  51
  52
  53 /* External tools reserve first few grant table entries. */
  54 #define NR_RESERVED_ENTRIES 8
  55 #define GNTTAB_LIST_END 0xffffffff
  56 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry_v1))
  57
  58 static grant_ref_t **gnttab_list;
  59 static unsigned int nr_grant_frames;
  60 static unsigned int boot_max_nr_grant_frames;
  61 static int gnttab_free_count;
  62 static grant_ref_t gnttab_free_head;
  63 static DEFINE_SPINLOCK(gnttab_list_lock);
  64 unsigned long xen_hvm_resume_frames;
  65 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
  66
  67 static union {
  68         struct grant_entry_v1 *v1;
  69         void *addr;
  70 } gnttab_shared;
  71
  72 /*This is a structure of function pointers for grant table*/
  73 struct gnttab_ops {
  74         /*
  75          * Mapping a list of frames for storing grant entries. First input
  76          * parameter is used to storing grant table address when grant table
  77          * being setup, second parameter is the number of frames to map grant
  78          * table. Returning GNTST_okay means success and negative value means
  79          * failure.
  80          */
  81         int (*map_frames)(unsigned long *, unsigned int);
  82         /*
  83          * Release a list of frames which are mapped in map_frames for grant
  84          * entry status.
  85          */
  86         void (*unmap_frames)(void);
  87         /*
  88          * Introducing a valid entry into the grant table, granting the frame
  89          * of this grant entry to domain for accessing, or transfering, or
  90          * transitively accessing. First input parameter is reference of this
  91          * introduced grant entry, second one is domid of granted domain, third
  92          * one is the frame to be granted, and the last one is status of the
  93          * grant entry to be updated.
  94          */
  95         void (*update_entry)(grant_ref_t, domid_t, unsigned long, unsigned);
  96         /*
  97          * Stop granting a grant entry to domain for accessing. First input
  98          * parameter is reference of a grant entry whose grant access will be
  99          * stopped, second one is not in use now. If the grant entry is
 100          * currently mapped for reading or writing, just return failure(==0)
 101          * directly and don't tear down the grant access. Otherwise, stop grant
 102          * access for this entry and return success(==1).
 103          */
 104         int (*end_foreign_access_ref)(grant_ref_t, int);
 105         /*
 106          * Stop granting a grant entry to domain for transfer. If tranfer has
 107          * not started, just reclaim the grant entry and return failure(==0).
 108          * Otherwise, wait for the transfer to complete and then return the
 109          * frame.
 110          */
 111         unsigned long (*end_foreign_transfer_ref)(grant_ref_t);
 112         /*
 113          * Query the status of a grant entry. Input parameter is reference of
 114          * queried grant entry, return value is the status of queried entry.
 115          * Detailed status(writing/reading) can be gotten from the return value
 116          * by bit operations.
 117          */
 118         int (*query_foreign_access)(grant_ref_t);
 119 };
 120
 121 static struct gnttab_ops *gnttab_interface;
 122
 123 static int grant_table_version;
 124
 125 static struct gnttab_free_callback *gnttab_free_callback_list;
 126
 127 static int gnttab_expand(unsigned int req_entries);
 128
 129 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 130
 131 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 132 {
 133         return &gnttab_list[(entry) / RPP][(entry) % RPP];
 134 }
 135 /* This can be used as an l-value */
 136 #define gnttab_entry(entry) (*__gnttab_entry(entry))
 137
 138 static int get_free_entries(unsigned count)
 139 {
 140         unsigned long flags;
 141         int ref, rc = 0;
 142         grant_ref_t head;
 143
 144         spin_lock_irqsave(&gnttab_list_lock, flags);
 145
 146         if ((gnttab_free_count < count) &&
 147             ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 148                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 149                 return rc;
 150         }
 151
 152         ref = head = gnttab_free_head;
 153         gnttab_free_count -= count;
 154         while (count-- > 1)
 155                 head = gnttab_entry(head);
 156         gnttab_free_head = gnttab_entry(head);
 157         gnttab_entry(head) = GNTTAB_LIST_END;
 158
 159         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 160
 161         return ref;
 162 }
 163
 164 static void do_free_callbacks(void)
 165 {
 166         struct gnttab_free_callback *callback, *next;
 167
 168         callback = gnttab_free_callback_list;
 169         gnttab_free_callback_list = NULL;
 170
 171         while (callback != NULL) {
 172                 next = callback->next;
 173                 if (gnttab_free_count >= callback->count) {
 174                         callback->next = NULL;
 175                         callback->fn(callback->arg);
 176                 } else {
 177                         callback->next = gnttab_free_callback_list;
 178                         gnttab_free_callback_list = callback;
 179                 }
 180                 callback = next;
 181         }
 182 }
 183
 184 static inline void check_free_callbacks(void)
 185 {
 186         if (unlikely(gnttab_free_callback_list))
 187                 do_free_callbacks();
 188 }
 189
 190 static void put_free_entry(grant_ref_t ref)
 191 {
 192         unsigned long flags;
 193         spin_lock_irqsave(&gnttab_list_lock, flags);
 194         gnttab_entry(ref) = gnttab_free_head;
 195         gnttab_free_head = ref;
 196         gnttab_free_count++;
 197         check_free_callbacks();
 198         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 199 }
 200
 201 /*
 202  * Introducing a valid entry into the grant table:
 203  *  1. Write ent->domid.
 204  *  2. Write ent->frame:
 205  *      GTF_permit_access:   Frame to which access is permitted.
 206  *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 207  *                           frame, or zero if none.
 208  *  3. Write memory barrier (WMB).
 209  *  4. Write ent->flags, inc. valid type.
 210  */
 211 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 212                                    unsigned long frame, unsigned flags)
 213 {
 214         gnttab_shared.v1[ref].domid = domid;
 215         gnttab_shared.v1[ref].frame = frame;
 216         wmb();
 217         gnttab_shared.v1[ref].flags = flags;
 218 }
 219
 220 /*
 221  * Public grant-issuing interface functions
 222  */
 223 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 224                                      unsigned long frame, int readonly)
 225 {
 226         gnttab_interface->update_entry(ref, domid, frame,
 227                            GTF_permit_access | (readonly ? GTF_readonly : 0));
 228 }
 229 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 230
 231 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 232                                 int readonly)
 233 {
 234         int ref;
 235
 236         ref = get_free_entries(1);
 237         if (unlikely(ref < 0))
 238                 return -ENOSPC;
 239
 240         gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 241
 242         return ref;
 243 }
 244 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 245
 246 static int gnttab_query_foreign_access_v1(grant_ref_t ref)
 247 {
 248         return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
 249 }
 250
 251 int gnttab_query_foreign_access(grant_ref_t ref)
 252 {
 253         return gnttab_interface->query_foreign_access(ref);
 254 }
 255 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 256
 257 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
 258 {
 259         u16 flags, nflags;
 260         u16 *pflags;
 261
 262         pflags = &gnttab_shared.v1[ref].flags;
 263         nflags = *pflags;
 264         do {
 265                 flags = nflags;
 266                 if (flags & (GTF_reading|GTF_writing)) {
 267                         printk(KERN_ALERT "WARNING: g.e. still in use!\n");
 268                         return 0;
 269                 }
 270         } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
 271
 272         return 1;
 273 }
 274
 275 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 276 {
 277         return gnttab_interface->end_foreign_access_ref(ref, readonly);
 278 }
 279 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 280
 281 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 282                                unsigned long page)
 283 {
 284         if (gnttab_end_foreign_access_ref(ref, readonly)) {
 285                 put_free_entry(ref);
 286                 if (page != 0)
 287                         free_page(page);
 288         } else {
 289                 /* XXX This needs to be fixed so that the ref and page are
 290                    placed on a list to be freed up later. */
 291                 printk(KERN_WARNING
 292                        "WARNING: leaking g.e. and page still in use!\n");
 293         }
 294 }
 295 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 296
 297 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 298 {
 299         int ref;
 300
 301         ref = get_free_entries(1);
 302         if (unlikely(ref < 0))
 303                 return -ENOSPC;
 304         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 305
 306         return ref;
 307 }
 308 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 309
 310 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 311                                        unsigned long pfn)
 312 {
 313         gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
 314 }
 315 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 316
 317 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
 318 {
 319         unsigned long frame;
 320         u16           flags;
 321         u16          *pflags;
 322
 323         pflags = &gnttab_shared.v1[ref].flags;
 324
 325         /*
 326          * If a transfer is not even yet started, try to reclaim the grant
 327          * reference and return failure (== 0).
 328          */
 329         while (!((flags = *pflags) & GTF_transfer_committed)) {
 330                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 331                         return 0;
 332                 cpu_relax();
 333         }
 334
 335         /* If a transfer is in progress then wait until it is completed. */
 336         while (!(flags & GTF_transfer_completed)) {
 337                 flags = *pflags;
 338                 cpu_relax();
 339         }
 340
 341         rmb();  /* Read the frame number /after/ reading completion status. */
 342         frame = gnttab_shared.v1[ref].frame;
 343         BUG_ON(frame == 0);
 344
 345         return frame;
 346 }
 347
 348 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 349 {
 350         return gnttab_interface->end_foreign_transfer_ref(ref);
 351 }
 352 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 353
 354 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 355 {
 356         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 357         put_free_entry(ref);
 358         return frame;
 359 }
 360 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 361
 362 void gnttab_free_grant_reference(grant_ref_t ref)
 363 {
 364         put_free_entry(ref);
 365 }
 366 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 367
 368 void gnttab_free_grant_references(grant_ref_t head)
 369 {
 370         grant_ref_t ref;
 371         unsigned long flags;
 372         int count = 1;
 373         if (head == GNTTAB_LIST_END)
 374                 return;
 375         spin_lock_irqsave(&gnttab_list_lock, flags);
 376         ref = head;
 377         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 378                 ref = gnttab_entry(ref);
 379                 count++;
 380         }
 381         gnttab_entry(ref) = gnttab_free_head;
 382         gnttab_free_head = head;
 383         gnttab_free_count += count;
 384         check_free_callbacks();
 385         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 386 }
 387 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 388
 389 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 390 {
 391         int h = get_free_entries(count);
 392
 393         if (h < 0)
 394                 return -ENOSPC;
 395
 396         *head = h;
 397
 398         return 0;
 399 }
 400 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 401
 402 int gnttab_empty_grant_references(const grant_ref_t *private_head)
 403 {
 404         return (*private_head == GNTTAB_LIST_END);
 405 }
 406 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 407
 408 int gnttab_claim_grant_reference(grant_ref_t *private_head)
 409 {
 410         grant_ref_t g = *private_head;
 411         if (unlikely(g == GNTTAB_LIST_END))
 412                 return -ENOSPC;
 413         *private_head = gnttab_entry(g);
 414         return g;
 415 }
 416 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 417
 418 void gnttab_release_grant_reference(grant_ref_t *private_head,
 419                                     grant_ref_t release)
 420 {
 421         gnttab_entry(release) = *private_head;
 422         *private_head = release;
 423 }
 424 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 425
 426 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 427                                   void (*fn)(void *), void *arg, u16 count)
 428 {
 429         unsigned long flags;
 430         spin_lock_irqsave(&gnttab_list_lock, flags);
 431         if (callback->next)
 432                 goto out;
 433         callback->fn = fn;
 434         callback->arg = arg;
 435         callback->count = count;
 436         callback->next = gnttab_free_callback_list;
 437         gnttab_free_callback_list = callback;
 438         check_free_callbacks();
 439 out:
 440         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 441 }
 442 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 443
 444 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 445 {
 446         struct gnttab_free_callback **pcb;
 447         unsigned long flags;
 448
 449         spin_lock_irqsave(&gnttab_list_lock, flags);
 450         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 451                 if (*pcb == callback) {
 452                         *pcb = callback->next;
 453                         break;
 454                 }
 455         }
 456         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 457 }
 458 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 459
 460 static int grow_gnttab_list(unsigned int more_frames)
 461 {
 462         unsigned int new_nr_grant_frames, extra_entries, i;
 463         unsigned int nr_glist_frames, new_nr_glist_frames;
 464
 465         new_nr_grant_frames = nr_grant_frames + more_frames;
 466         extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
 467
 468         nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 469         new_nr_glist_frames =
 470                 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 471         for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 472                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 473                 if (!gnttab_list[i])
 474                         goto grow_nomem;
 475         }
 476
 477
 478         for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 479              i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
 480                 gnttab_entry(i) = i + 1;
 481
 482         gnttab_entry(i) = gnttab_free_head;
 483         gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 484         gnttab_free_count += extra_entries;
 485
 486         nr_grant_frames = new_nr_grant_frames;
 487
 488         check_free_callbacks();
 489
 490         return 0;
 491
 492 grow_nomem:
 493         for ( ; i >= nr_glist_frames; i--)
 494                 free_page((unsigned long) gnttab_list[i]);
 495         return -ENOMEM;
 496 }
 497
 498 static unsigned int __max_nr_grant_frames(void)
 499 {
 500         struct gnttab_query_size query;
 501         int rc;
 502
 503         query.dom = DOMID_SELF;
 504
 505         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 506         if ((rc < 0) || (query.status != GNTST_okay))
 507                 return 4; /* Legacy max supported number of frames */
 508
 509         return query.max_nr_frames;
 510 }
 511
 512 unsigned int gnttab_max_grant_frames(void)
 513 {
 514         unsigned int xen_max = __max_nr_grant_frames();
 515
 516         if (xen_max > boot_max_nr_grant_frames)
 517                 return boot_max_nr_grant_frames;
 518         return xen_max;
 519 }
 520 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 521
 522 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 523                         struct gnttab_map_grant_ref *kmap_ops,
 524                         struct page **pages, unsigned int count)
 525 {
 526         int i, ret;
 527         pte_t *pte;
 528         unsigned long mfn;
 529
 530         ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
 531         if (ret)
 532                 return ret;
 533
 534         if (xen_feature(XENFEAT_auto_translated_physmap))
 535                 return ret;
 536
 537         for (i = 0; i < count; i++) {
 538                 /* Do not add to override if the map failed. */
 539                 if (map_ops[i].status)
 540                         continue;
 541
 542                 if (map_ops[i].flags & GNTMAP_contains_pte) {
 543                         pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
 544                                 (map_ops[i].host_addr & ~PAGE_MASK));
 545                         mfn = pte_mfn(*pte);
 546                 } else {
 547                         /* If you really wanted to do this:
 548                          * mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
 549                          *
 550                          * The reason we do not implement it is b/c on the
 551                          * unmap path (gnttab_unmap_refs) we have no means of
 552                          * checking whether the page is !GNTMAP_contains_pte.
 553                          *
 554                          * That is without some extra data-structure to carry
 555                          * the struct page, bool clear_pte, and list_head next
 556                          * tuples and deal with allocation/delallocation, etc.
 557                          *
 558                          * The users of this API set the GNTMAP_contains_pte
 559                          * flag so lets just return not supported until it
 560                          * becomes neccessary to implement.
 561                          */
 562                         return -EOPNOTSUPP;
 563                 }
 564                 ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]);
 565                 if (ret)
 566                         return ret;
 567         }
 568
 569         return ret;
 570 }
 571 EXPORT_SYMBOL_GPL(gnttab_map_refs);
 572
 573 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 574                 struct page **pages, unsigned int count)
 575 {
 576         int i, ret;
 577
 578         ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
 579         if (ret)
 580                 return ret;
 581
 582         if (xen_feature(XENFEAT_auto_translated_physmap))
 583                 return ret;
 584
 585         for (i = 0; i < count; i++) {
 586                 ret = m2p_remove_override(pages[i], true /* clear the PTE */);
 587                 if (ret)
 588                         return ret;
 589         }
 590
 591         return ret;
 592 }
 593 EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
 594
 595 static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
 596 {
 597         int rc;
 598
 599         rc = arch_gnttab_map_shared(frames, nr_gframes,
 600                                     gnttab_max_grant_frames(),
 601                                     &gnttab_shared.addr);
 602         BUG_ON(rc);
 603
 604         return 0;
 605 }
 606
 607 static void gnttab_unmap_frames_v1(void)
 608 {
 609         arch_gnttab_unmap_shared(gnttab_shared.addr, nr_grant_frames);
 610 }
 611
 612 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 613 {
 614         struct gnttab_setup_table setup;
 615         unsigned long *frames;
 616         unsigned int nr_gframes = end_idx + 1;
 617         int rc;
 618
 619         if (xen_hvm_domain()) {
 620                 struct xen_add_to_physmap xatp;
 621                 unsigned int i = end_idx;
 622                 rc = 0;
 623                 /*
 624                  * Loop backwards, so that the first hypercall has the largest
 625                  * index, ensuring that the table will grow only once.
 626                  */
 627                 do {
 628                         xatp.domid = DOMID_SELF;
 629                         xatp.idx = i;
 630                         xatp.space = XENMAPSPACE_grant_table;
 631                         xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
 632                         rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 633                         if (rc != 0) {
 634                                 printk(KERN_WARNING
 635                                                 "grant table add_to_physmap failed, err=%d\n", rc);
 636                                 break;
 637                         }
 638                 } while (i-- > start_idx);
 639
 640                 return rc;
 641         }
 642
 643         frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
 644         if (!frames)
 645                 return -ENOMEM;
 646
 647         setup.dom        = DOMID_SELF;
 648         setup.nr_frames  = nr_gframes;
 649         set_xen_guest_handle(setup.frame_list, frames);
 650
 651         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
 652         if (rc == -ENOSYS) {
 653                 kfree(frames);
 654                 return -ENOSYS;
 655         }
 656
 657         BUG_ON(rc || setup.status);
 658
 659         rc = gnttab_interface->map_frames(frames, nr_gframes);
 660
 661         kfree(frames);
 662
 663         return rc;
 664 }
 665
 666 static struct gnttab_ops gnttab_v1_ops = {
 667         .map_frames                     = gnttab_map_frames_v1,
 668         .unmap_frames                   = gnttab_unmap_frames_v1,
 669         .update_entry                   = gnttab_update_entry_v1,
 670         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v1,
 671         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v1,
 672         .query_foreign_access           = gnttab_query_foreign_access_v1,
 673 };
 674
 675 static void gnttab_request_version(void)
 676 {
 677         grant_table_version = 1;
 678         gnttab_interface = &gnttab_v1_ops;
 679         printk(KERN_INFO "Grant tables using version %d layout.\n",
 680                 grant_table_version);
 681 }
 682
 683 int gnttab_resume(void)
 684 {
 685         unsigned int max_nr_gframes;
 686
 687         gnttab_request_version();
 688         max_nr_gframes = gnttab_max_grant_frames();
 689         if (max_nr_gframes < nr_grant_frames)
 690                 return -ENOSYS;
 691
 692         if (xen_pv_domain())
 693                 return gnttab_map(0, nr_grant_frames - 1);
 694
 695         if (gnttab_shared.addr == NULL) {
 696                 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
 697                                                 PAGE_SIZE * max_nr_gframes);
 698                 if (gnttab_shared.addr == NULL) {
 699                         printk(KERN_WARNING
 700                                         "Failed to ioremap gnttab share frames!");
 701                         return -ENOMEM;
 702                 }
 703         }
 704
 705         gnttab_map(0, nr_grant_frames - 1);
 706
 707         return 0;
 708 }
 709
 710 int gnttab_suspend(void)
 711 {
 712         gnttab_interface->unmap_frames();
 713         return 0;
 714 }
 715
 716 static int gnttab_expand(unsigned int req_entries)
 717 {
 718         int rc;
 719         unsigned int cur, extra;
 720
 721         cur = nr_grant_frames;
 722         extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
 723                  GREFS_PER_GRANT_FRAME);
 724         if (cur + extra > gnttab_max_grant_frames())
 725                 return -ENOSPC;
 726
 727         rc = gnttab_map(cur, cur + extra - 1);
 728         if (rc == 0)
 729                 rc = grow_gnttab_list(extra);
 730
 731         return rc;
 732 }
 733
 734 int gnttab_init(void)
 735 {
 736         int i;
 737         unsigned int max_nr_glist_frames, nr_glist_frames;
 738         unsigned int nr_init_grefs;
 739
 740         nr_grant_frames = 1;
 741         boot_max_nr_grant_frames = __max_nr_grant_frames();
 742
 743         /* Determine the maximum number of frames required for the
 744          * grant reference free list on the current hypervisor.
 745          */
 746         max_nr_glist_frames = (boot_max_nr_grant_frames *
 747                                GREFS_PER_GRANT_FRAME / RPP);
 748
 749         gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
 750                               GFP_KERNEL);
 751         if (gnttab_list == NULL)
 752                 return -ENOMEM;
 753
 754         nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 755         for (i = 0; i < nr_glist_frames; i++) {
 756                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
 757                 if (gnttab_list[i] == NULL)
 758                         goto ini_nomem;
 759         }
 760
 761         if (gnttab_resume() < 0)
 762                 return -ENODEV;
 763
 764         nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
 765
 766         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
 767                 gnttab_entry(i) = i + 1;
 768
 769         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
 770         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
 771         gnttab_free_head  = NR_RESERVED_ENTRIES;
 772
 773         printk("Grant table initialized\n");
 774         return 0;
 775
 776  ini_nomem:
 777         for (i--; i >= 0; i--)
 778                 free_page((unsigned long)gnttab_list[i]);
 779         kfree(gnttab_list);
 780         return -ENOMEM;
 781 }
 782 EXPORT_SYMBOL_GPL(gnttab_init);
 783
 784 static int __devinit __gnttab_init(void)
 785 {
 786         /* Delay grant-table initialization in the PV on HVM case */
 787         if (xen_hvm_domain())
 788                 return 0;
 789
 790         if (!xen_pv_domain())
 791                 return -ENODEV;
 792
 793         return gnttab_init();
 794 }
 795
 796 core_initcall(__gnttab_init);