arch/powerpc/kernel/vdso.c

   1
   2 /*
   3  *    Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
   4  *                       <benh@kernel.crashing.org>
   5  *
   6  *  This program is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU General Public License
   8  *  as published by the Free Software Foundation; either version
   9  *  2 of the License, or (at your option) any later version.
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/sched.h>
  14 #include <linux/kernel.h>
  15 #include <linux/mm.h>
  16 #include <linux/smp.h>
  17 #include <linux/stddef.h>
  18 #include <linux/unistd.h>
  19 #include <linux/slab.h>
  20 #include <linux/user.h>
  21 #include <linux/elf.h>
  22 #include <linux/security.h>
  23 #include <linux/bootmem.h>
  24 #include <linux/memblock.h>
  25
  26 #include <asm/pgtable.h>
  27 #include <asm/processor.h>
  28 #include <asm/mmu.h>
  29 #include <asm/mmu_context.h>
  30 #include <asm/prom.h>
  31 #include <asm/machdep.h>
  32 #include <asm/cputable.h>
  33 #include <asm/sections.h>
  34 #include <asm/firmware.h>
  35 #include <asm/vdso.h>
  36 #include <asm/vdso_datapage.h>
  37 #include <asm/setup.h>
  38
  39 #undef DEBUG
  40
  41 #ifdef DEBUG
  42 #define DBG(fmt...) printk(fmt)
  43 #else
  44 #define DBG(fmt...)
  45 #endif
  46
  47 /* Max supported size for symbol names */
  48 #define MAX_SYMNAME     64
  49
  50 /* The alignment of the vDSO */
  51 #define VDSO_ALIGNMENT  (1 << 16)
  52
  53 extern char vdso32_start, vdso32_end;
  54 static void *vdso32_kbase = &vdso32_start;
  55 static unsigned int vdso32_pages;
  56 static struct page **vdso32_pagelist;
  57 unsigned long vdso32_sigtramp;
  58 unsigned long vdso32_rt_sigtramp;
  59
  60 #ifdef CONFIG_PPC64
  61 extern char vdso64_start, vdso64_end;
  62 static void *vdso64_kbase = &vdso64_start;
  63 static unsigned int vdso64_pages;
  64 static struct page **vdso64_pagelist;
  65 unsigned long vdso64_rt_sigtramp;
  66 #endif /* CONFIG_PPC64 */
  67
  68 static int vdso_ready;
  69
  70 /*
  71  * The vdso data page (aka. systemcfg for old ppc64 fans) is here.
  72  * Once the early boot kernel code no longer needs to muck around
  73  * with it, it will become dynamically allocated
  74  */
  75 static union {
  76         struct vdso_data        data;
  77         u8                      page[PAGE_SIZE];
  78 } vdso_data_store __page_aligned_data;
  79 struct vdso_data *vdso_data = &vdso_data_store.data;
  80
  81 /* Format of the patch table */
  82 struct vdso_patch_def
  83 {
  84         unsigned long   ftr_mask, ftr_value;
  85         const char      *gen_name;
  86         const char      *fix_name;
  87 };
  88
  89 /* Table of functions to patch based on the CPU type/revision
  90  *
  91  * Currently, we only change sync_dicache to do nothing on processors
  92  * with a coherent icache
  93  */
  94 static struct vdso_patch_def vdso_patches[] = {
  95         {
  96                 CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE,
  97                 "__kernel_sync_dicache", "__kernel_sync_dicache_p5"
  98         },
  99         {
 100                 CPU_FTR_USE_TB, 0,
 101                 "__kernel_gettimeofday", NULL
 102         },
 103         {
 104                 CPU_FTR_USE_TB, 0,
 105                 "__kernel_clock_gettime", NULL
 106         },
 107         {
 108                 CPU_FTR_USE_TB, 0,
 109                 "__kernel_clock_getres", NULL
 110         },
 111         {
 112                 CPU_FTR_USE_TB, 0,
 113                 "__kernel_get_tbfreq", NULL
 114         },
 115         {
 116                 CPU_FTR_USE_TB, 0,
 117                 "__kernel_time", NULL
 118         },
 119 };
 120
 121 /*
 122  * Some infos carried around for each of them during parsing at
 123  * boot time.
 124  */
 125 struct lib32_elfinfo
 126 {
 127         Elf32_Ehdr      *hdr;           /* ptr to ELF */
 128         Elf32_Sym       *dynsym;        /* ptr to .dynsym section */
 129         unsigned long   dynsymsize;     /* size of .dynsym section */
 130         char            *dynstr;        /* ptr to .dynstr section */
 131         unsigned long   text;           /* offset of .text section in .so */
 132 };
 133
 134 struct lib64_elfinfo
 135 {
 136         Elf64_Ehdr      *hdr;
 137         Elf64_Sym       *dynsym;
 138         unsigned long   dynsymsize;
 139         char            *dynstr;
 140         unsigned long   text;
 141 };
 142
 143
 144 #ifdef __DEBUG
 145 static void dump_one_vdso_page(struct page *pg, struct page *upg)
 146 {
 147         printk("kpg: %p (c:%d,f:%08lx)", __va(page_to_pfn(pg) << PAGE_SHIFT),
 148                page_count(pg),
 149                pg->flags);
 150         if (upg && !IS_ERR(upg) /* && pg != upg*/) {
 151                 printk(" upg: %p (c:%d,f:%08lx)", __va(page_to_pfn(upg)
 152                                                        << PAGE_SHIFT),
 153                        page_count(upg),
 154                        upg->flags);
 155         }
 156         printk("\n");
 157 }
 158
 159 static void dump_vdso_pages(struct vm_area_struct * vma)
 160 {
 161         int i;
 162
 163         if (!vma || is_32bit_task()) {
 164                 printk("vDSO32 @ %016lx:\n", (unsigned long)vdso32_kbase);
 165                 for (i=0; i<vdso32_pages; i++) {
 166                         struct page *pg = virt_to_page(vdso32_kbase +
 167                                                        i*PAGE_SIZE);
 168                         struct page *upg = (vma && vma->vm_mm) ?
 169                                 follow_page(vma, vma->vm_start + i*PAGE_SIZE, 0)
 170                                 : NULL;
 171                         dump_one_vdso_page(pg, upg);
 172                 }
 173         }
 174         if (!vma || !is_32bit_task()) {
 175                 printk("vDSO64 @ %016lx:\n", (unsigned long)vdso64_kbase);
 176                 for (i=0; i<vdso64_pages; i++) {
 177                         struct page *pg = virt_to_page(vdso64_kbase +
 178                                                        i*PAGE_SIZE);
 179                         struct page *upg = (vma && vma->vm_mm) ?
 180                                 follow_page(vma, vma->vm_start + i*PAGE_SIZE, 0)
 181                                 : NULL;
 182                         dump_one_vdso_page(pg, upg);
 183                 }
 184         }
 185 }
 186 #endif /* DEBUG */
 187
 188 /*
 189  * This is called from binfmt_elf, we create the special vma for the
 190  * vDSO and insert it into the mm struct tree
 191  */
 192 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 193 {
 194         struct mm_struct *mm = current->mm;
 195         struct page **vdso_pagelist;
 196         unsigned long vdso_pages;
 197         unsigned long vdso_base;
 198         int rc;
 199
 200         if (!vdso_ready)
 201                 return 0;
 202
 203 #ifdef CONFIG_PPC64
 204         if (is_32bit_task()) {
 205                 vdso_pagelist = vdso32_pagelist;
 206                 vdso_pages = vdso32_pages;
 207                 vdso_base = VDSO32_MBASE;
 208         } else {
 209                 vdso_pagelist = vdso64_pagelist;
 210                 vdso_pages = vdso64_pages;
 211                 /*
 212                  * On 64bit we don't have a preferred map address. This
 213                  * allows get_unmapped_area to find an area near other mmaps
 214                  * and most likely share a SLB entry.
 215                  */
 216                 vdso_base = 0;
 217         }
 218 #else
 219         vdso_pagelist = vdso32_pagelist;
 220         vdso_pages = vdso32_pages;
 221         vdso_base = VDSO32_MBASE;
 222 #endif
 223
 224         current->mm->context.vdso_base = 0;
 225
 226         /* vDSO has a problem and was disabled, just don't "enable" it for the
 227          * process
 228          */
 229         if (vdso_pages == 0)
 230                 return 0;
 231         /* Add a page to the vdso size for the data page */
 232         vdso_pages ++;
 233
 234         /*
 235          * pick a base address for the vDSO in process space. We try to put it
 236          * at vdso_base which is the "natural" base for it, but we might fail
 237          * and end up putting it elsewhere.
 238          * Add enough to the size so that the result can be aligned.
 239          */
 240         down_write(&mm->mmap_sem);
 241         vdso_base = get_unmapped_area(NULL, vdso_base,
 242                                       (vdso_pages << PAGE_SHIFT) +
 243                                       ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
 244                                       0, 0);
 245         if (IS_ERR_VALUE(vdso_base)) {
 246                 rc = vdso_base;
 247                 goto fail_mmapsem;
 248         }
 249
 250         /* Add required alignment. */
 251         vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
 252
 253         /*
 254          * Put vDSO base into mm struct. We need to do this before calling
 255          * install_special_mapping or the perf counter mmap tracking code
 256          * will fail to recognise it as a vDSO (since arch_vma_name fails).
 257          */
 258         current->mm->context.vdso_base = vdso_base;
 259
 260         /*
 261          * our vma flags don't have VM_WRITE so by default, the process isn't
 262          * allowed to write those pages.
 263          * gdb can break that with ptrace interface, and thus trigger COW on
 264          * those pages but it's then your responsibility to never do that on
 265          * the "data" page of the vDSO or you'll stop getting kernel updates
 266          * and your nice userland gettimeofday will be totally dead.
 267          * It's fine to use that for setting breakpoints in the vDSO code
 268          * pages though.
 269          */
 270         rc = install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT,
 271                                      VM_READ|VM_EXEC|
 272                                      VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
 273                                      vdso_pagelist);
 274         if (rc) {
 275                 current->mm->context.vdso_base = 0;
 276                 goto fail_mmapsem;
 277         }
 278
 279         up_write(&mm->mmap_sem);
 280         return 0;
 281
 282  fail_mmapsem:
 283         up_write(&mm->mmap_sem);
 284         return rc;
 285 }
 286
 287 const char *arch_vma_name(struct vm_area_struct *vma)
 288 {
 289         if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base)
 290                 return "[vdso]";
 291         return NULL;
 292 }
 293
 294
 295
 296 static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname,
 297                                   unsigned long *size)
 298 {
 299         Elf32_Shdr *sechdrs;
 300         unsigned int i;
 301         char *secnames;
 302
 303         /* Grab section headers and strings so we can tell who is who */
 304         sechdrs = (void *)ehdr + ehdr->e_shoff;
 305         secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
 306
 307         /* Find the section they want */
 308         for (i = 1; i < ehdr->e_shnum; i++) {
 309                 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
 310                         if (size)
 311                                 *size = sechdrs[i].sh_size;
 312                         return (void *)ehdr + sechdrs[i].sh_offset;
 313                 }
 314         }
 315         *size = 0;
 316         return NULL;
 317 }
 318
 319 static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib,
 320                                         const char *symname)
 321 {
 322         unsigned int i;
 323         char name[MAX_SYMNAME], *c;
 324
 325         for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) {
 326                 if (lib->dynsym[i].st_name == 0)
 327                         continue;
 328                 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
 329                         MAX_SYMNAME);
 330                 c = strchr(name, '@');
 331                 if (c)
 332                         *c = 0;
 333                 if (strcmp(symname, name) == 0)
 334                         return &lib->dynsym[i];
 335         }
 336         return NULL;
 337 }
 338
 339 /* Note that we assume the section is .text and the symbol is relative to
 340  * the library base
 341  */
 342 static unsigned long __init find_function32(struct lib32_elfinfo *lib,
 343                                             const char *symname)
 344 {
 345         Elf32_Sym *sym = find_symbol32(lib, symname);
 346
 347         if (sym == NULL) {
 348                 printk(KERN_WARNING "vDSO32: function %s not found !\n",
 349                        symname);
 350                 return 0;
 351         }
 352         return sym->st_value - VDSO32_LBASE;
 353 }
 354
 355 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
 356                                        struct lib64_elfinfo *v64,
 357                                        const char *orig, const char *fix)
 358 {
 359         Elf32_Sym *sym32_gen, *sym32_fix;
 360
 361         sym32_gen = find_symbol32(v32, orig);
 362         if (sym32_gen == NULL) {
 363                 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig);
 364                 return -1;
 365         }
 366         if (fix == NULL) {
 367                 sym32_gen->st_name = 0;
 368                 return 0;
 369         }
 370         sym32_fix = find_symbol32(v32, fix);
 371         if (sym32_fix == NULL) {
 372                 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix);
 373                 return -1;
 374         }
 375         sym32_gen->st_value = sym32_fix->st_value;
 376         sym32_gen->st_size = sym32_fix->st_size;
 377         sym32_gen->st_info = sym32_fix->st_info;
 378         sym32_gen->st_other = sym32_fix->st_other;
 379         sym32_gen->st_shndx = sym32_fix->st_shndx;
 380
 381         return 0;
 382 }
 383
 384
 385 #ifdef CONFIG_PPC64
 386
 387 static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname,
 388                                   unsigned long *size)
 389 {
 390         Elf64_Shdr *sechdrs;
 391         unsigned int i;
 392         char *secnames;
 393
 394         /* Grab section headers and strings so we can tell who is who */
 395         sechdrs = (void *)ehdr + ehdr->e_shoff;
 396         secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
 397
 398         /* Find the section they want */
 399         for (i = 1; i < ehdr->e_shnum; i++) {
 400                 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
 401                         if (size)
 402                                 *size = sechdrs[i].sh_size;
 403                         return (void *)ehdr + sechdrs[i].sh_offset;
 404                 }
 405         }
 406         if (size)
 407                 *size = 0;
 408         return NULL;
 409 }
 410
 411 static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib,
 412                                         const char *symname)
 413 {
 414         unsigned int i;
 415         char name[MAX_SYMNAME], *c;
 416
 417         for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) {
 418                 if (lib->dynsym[i].st_name == 0)
 419                         continue;
 420                 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
 421                         MAX_SYMNAME);
 422                 c = strchr(name, '@');
 423                 if (c)
 424                         *c = 0;
 425                 if (strcmp(symname, name) == 0)
 426                         return &lib->dynsym[i];
 427         }
 428         return NULL;
 429 }
 430
 431 /* Note that we assume the section is .text and the symbol is relative to
 432  * the library base
 433  */
 434 static unsigned long __init find_function64(struct lib64_elfinfo *lib,
 435                                             const char *symname)
 436 {
 437         Elf64_Sym *sym = find_symbol64(lib, symname);
 438
 439         if (sym == NULL) {
 440                 printk(KERN_WARNING "vDSO64: function %s not found !\n",
 441                        symname);
 442                 return 0;
 443         }
 444 #ifdef VDS64_HAS_DESCRIPTORS
 445         return *((u64 *)(vdso64_kbase + sym->st_value - VDSO64_LBASE)) -
 446                 VDSO64_LBASE;
 447 #else
 448         return sym->st_value - VDSO64_LBASE;
 449 #endif
 450 }
 451
 452 static int __init vdso_do_func_patch64(struct lib32_elfinfo *v32,
 453                                        struct lib64_elfinfo *v64,
 454                                        const char *orig, const char *fix)
 455 {
 456         Elf64_Sym *sym64_gen, *sym64_fix;
 457
 458         sym64_gen = find_symbol64(v64, orig);
 459         if (sym64_gen == NULL) {
 460                 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig);
 461                 return -1;
 462         }
 463         if (fix == NULL) {
 464                 sym64_gen->st_name = 0;
 465                 return 0;
 466         }
 467         sym64_fix = find_symbol64(v64, fix);
 468         if (sym64_fix == NULL) {
 469                 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix);
 470                 return -1;
 471         }
 472         sym64_gen->st_value = sym64_fix->st_value;
 473         sym64_gen->st_size = sym64_fix->st_size;
 474         sym64_gen->st_info = sym64_fix->st_info;
 475         sym64_gen->st_other = sym64_fix->st_other;
 476         sym64_gen->st_shndx = sym64_fix->st_shndx;
 477
 478         return 0;
 479 }
 480
 481 #endif /* CONFIG_PPC64 */
 482
 483
 484 static __init int vdso_do_find_sections(struct lib32_elfinfo *v32,
 485                                         struct lib64_elfinfo *v64)
 486 {
 487         void *sect;
 488
 489         /*
 490          * Locate symbol tables & text section
 491          */
 492
 493         v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize);
 494         v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL);
 495         if (v32->dynsym == NULL || v32->dynstr == NULL) {
 496                 printk(KERN_ERR "vDSO32: required symbol section not found\n");
 497                 return -1;
 498         }
 499         sect = find_section32(v32->hdr, ".text", NULL);
 500         if (sect == NULL) {
 501                 printk(KERN_ERR "vDSO32: the .text section was not found\n");
 502                 return -1;
 503         }
 504         v32->text = sect - vdso32_kbase;
 505
 506 #ifdef CONFIG_PPC64
 507         v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize);
 508         v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL);
 509         if (v64->dynsym == NULL || v64->dynstr == NULL) {
 510                 printk(KERN_ERR "vDSO64: required symbol section not found\n");
 511                 return -1;
 512         }
 513         sect = find_section64(v64->hdr, ".text", NULL);
 514         if (sect == NULL) {
 515                 printk(KERN_ERR "vDSO64: the .text section was not found\n");
 516                 return -1;
 517         }
 518         v64->text = sect - vdso64_kbase;
 519 #endif /* CONFIG_PPC64 */
 520
 521         return 0;
 522 }
 523
 524 static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32,
 525                                           struct lib64_elfinfo *v64)
 526 {
 527         /*
 528          * Find signal trampolines
 529          */
 530
 531 #ifdef CONFIG_PPC64
 532         vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64");
 533 #endif
 534         vdso32_sigtramp    = find_function32(v32, "__kernel_sigtramp32");
 535         vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32");
 536 }
 537
 538 static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32,
 539                                        struct lib64_elfinfo *v64)
 540 {
 541         Elf32_Sym *sym32;
 542 #ifdef CONFIG_PPC64
 543         Elf64_Sym *sym64;
 544
 545         sym64 = find_symbol64(v64, "__kernel_datapage_offset");
 546         if (sym64 == NULL) {
 547                 printk(KERN_ERR "vDSO64: Can't find symbol "
 548                        "__kernel_datapage_offset !\n");
 549                 return -1;
 550         }
 551         *((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) =
 552                 (vdso64_pages << PAGE_SHIFT) -
 553                 (sym64->st_value - VDSO64_LBASE);
 554 #endif /* CONFIG_PPC64 */
 555
 556         sym32 = find_symbol32(v32, "__kernel_datapage_offset");
 557         if (sym32 == NULL) {
 558                 printk(KERN_ERR "vDSO32: Can't find symbol "
 559                        "__kernel_datapage_offset !\n");
 560                 return -1;
 561         }
 562         *((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) =
 563                 (vdso32_pages << PAGE_SHIFT) -
 564                 (sym32->st_value - VDSO32_LBASE);
 565
 566         return 0;
 567 }
 568
 569
 570 static __init int vdso_fixup_features(struct lib32_elfinfo *v32,
 571                                       struct lib64_elfinfo *v64)
 572 {
 573         void *start32;
 574         unsigned long size32;
 575
 576 #ifdef CONFIG_PPC64
 577         void *start64;
 578         unsigned long size64;
 579
 580         start64 = find_section64(v64->hdr, "__ftr_fixup", &size64);
 581         if (start64)
 582                 do_feature_fixups(cur_cpu_spec->cpu_features,
 583                                   start64, start64 + size64);
 584
 585         start64 = find_section64(v64->hdr, "__mmu_ftr_fixup", &size64);
 586         if (start64)
 587                 do_feature_fixups(cur_cpu_spec->mmu_features,
 588                                   start64, start64 + size64);
 589
 590         start64 = find_section64(v64->hdr, "__fw_ftr_fixup", &size64);
 591         if (start64)
 592                 do_feature_fixups(powerpc_firmware_features,
 593                                   start64, start64 + size64);
 594
 595         start64 = find_section64(v64->hdr, "__lwsync_fixup", &size64);
 596         if (start64)
 597                 do_lwsync_fixups(cur_cpu_spec->cpu_features,
 598                                  start64, start64 + size64);
 599 #endif /* CONFIG_PPC64 */
 600
 601         start32 = find_section32(v32->hdr, "__ftr_fixup", &size32);
 602         if (start32)
 603                 do_feature_fixups(cur_cpu_spec->cpu_features,
 604                                   start32, start32 + size32);
 605
 606         start32 = find_section32(v32->hdr, "__mmu_ftr_fixup", &size32);
 607         if (start32)
 608                 do_feature_fixups(cur_cpu_spec->mmu_features,
 609                                   start32, start32 + size32);
 610
 611 #ifdef CONFIG_PPC64
 612         start32 = find_section32(v32->hdr, "__fw_ftr_fixup", &size32);
 613         if (start32)
 614                 do_feature_fixups(powerpc_firmware_features,
 615                                   start32, start32 + size32);
 616 #endif /* CONFIG_PPC64 */
 617
 618         start32 = find_section32(v32->hdr, "__lwsync_fixup", &size32);
 619         if (start32)
 620                 do_lwsync_fixups(cur_cpu_spec->cpu_features,
 621                                  start32, start32 + size32);
 622
 623         return 0;
 624 }
 625
 626 static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32,
 627                                        struct lib64_elfinfo *v64)
 628 {
 629         int i;
 630
 631         for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) {
 632                 struct vdso_patch_def *patch = &vdso_patches[i];
 633                 int match = (cur_cpu_spec->cpu_features & patch->ftr_mask)
 634                         == patch->ftr_value;
 635                 if (!match)
 636                         continue;
 637
 638                 DBG("replacing %s with %s...\n", patch->gen_name,
 639                     patch->fix_name ? "NONE" : patch->fix_name);
 640
 641                 /*
 642                  * Patch the 32 bits and 64 bits symbols. Note that we do not
 643                  * patch the "." symbol on 64 bits.
 644                  * It would be easy to do, but doesn't seem to be necessary,
 645                  * patching the OPD symbol is enough.
 646                  */
 647                 vdso_do_func_patch32(v32, v64, patch->gen_name,
 648                                      patch->fix_name);
 649 #ifdef CONFIG_PPC64
 650                 vdso_do_func_patch64(v32, v64, patch->gen_name,
 651                                      patch->fix_name);
 652 #endif /* CONFIG_PPC64 */
 653         }
 654
 655         return 0;
 656 }
 657
 658
 659 static __init int vdso_setup(void)
 660 {
 661         struct lib32_elfinfo    v32;
 662         struct lib64_elfinfo    v64;
 663
 664         v32.hdr = vdso32_kbase;
 665 #ifdef CONFIG_PPC64
 666         v64.hdr = vdso64_kbase;
 667 #endif
 668         if (vdso_do_find_sections(&v32, &v64))
 669                 return -1;
 670
 671         if (vdso_fixup_datapage(&v32, &v64))
 672                 return -1;
 673
 674         if (vdso_fixup_features(&v32, &v64))
 675                 return -1;
 676
 677         if (vdso_fixup_alt_funcs(&v32, &v64))
 678                 return -1;
 679
 680         vdso_setup_trampolines(&v32, &v64);
 681
 682         return 0;
 683 }
 684
 685 /*
 686  * Called from setup_arch to initialize the bitmap of available
 687  * syscalls in the systemcfg page
 688  */
 689 static void __init vdso_setup_syscall_map(void)
 690 {
 691         unsigned int i;
 692         extern unsigned long *sys_call_table;
 693         extern unsigned long sys_ni_syscall;
 694
 695
 696         for (i = 0; i < __NR_syscalls; i++) {
 697 #ifdef CONFIG_PPC64
 698                 if (sys_call_table[i*2] != sys_ni_syscall)
 699                         vdso_data->syscall_map_64[i >> 5] |=
 700                                 0x80000000UL >> (i & 0x1f);
 701                 if (sys_call_table[i*2+1] != sys_ni_syscall)
 702                         vdso_data->syscall_map_32[i >> 5] |=
 703                                 0x80000000UL >> (i & 0x1f);
 704 #else /* CONFIG_PPC64 */
 705                 if (sys_call_table[i] != sys_ni_syscall)
 706                         vdso_data->syscall_map_32[i >> 5] |=
 707                                 0x80000000UL >> (i & 0x1f);
 708 #endif /* CONFIG_PPC64 */
 709         }
 710 }
 711
 712 #ifdef CONFIG_PPC64
 713 int vdso_getcpu_init(void)
 714 {
 715         unsigned long cpu, node, val;
 716
 717         /*
 718          * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node
 719          * in the next 16 bits.  The VDSO uses this to implement getcpu().
 720          */
 721         cpu = get_cpu();
 722         WARN_ON_ONCE(cpu > 0xffff);
 723
 724         node = cpu_to_node(cpu);
 725         WARN_ON_ONCE(node > 0xffff);
 726
 727         val = (cpu & 0xfff) | ((node & 0xffff) << 16);
 728         mtspr(SPRN_SPRG_VDSO_WRITE, val);
 729         get_paca()->sprg_vdso = val;
 730
 731         put_cpu();
 732
 733         return 0;
 734 }
 735 /* We need to call this before SMP init */
 736 early_initcall(vdso_getcpu_init);
 737 #endif
 738
 739 static int __init vdso_init(void)
 740 {
 741         int i;
 742
 743 #ifdef CONFIG_PPC64
 744         /*
 745          * Fill up the "systemcfg" stuff for backward compatibility
 746          */
 747         strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64");
 748         vdso_data->version.major = SYSTEMCFG_MAJOR;
 749         vdso_data->version.minor = SYSTEMCFG_MINOR;
 750         vdso_data->processor = mfspr(SPRN_PVR);
 751         /*
 752          * Fake the old platform number for pSeries and add
 753          * in LPAR bit if necessary
 754          */
 755         vdso_data->platform = 0x100;
 756         if (firmware_has_feature(FW_FEATURE_LPAR))
 757                 vdso_data->platform |= 1;
 758         vdso_data->physicalMemorySize = memblock_phys_mem_size();
 759         vdso_data->dcache_size = ppc64_caches.dsize;
 760         vdso_data->dcache_line_size = ppc64_caches.dline_size;
 761         vdso_data->icache_size = ppc64_caches.isize;
 762         vdso_data->icache_line_size = ppc64_caches.iline_size;
 763
 764         /* XXXOJN: Blocks should be added to ppc64_caches and used instead */
 765         vdso_data->dcache_block_size = ppc64_caches.dline_size;
 766         vdso_data->icache_block_size = ppc64_caches.iline_size;
 767         vdso_data->dcache_log_block_size = ppc64_caches.log_dline_size;
 768         vdso_data->icache_log_block_size = ppc64_caches.log_iline_size;
 769
 770         /*
 771          * Calculate the size of the 64 bits vDSO
 772          */
 773         vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT;
 774         DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages);
 775 #else
 776         vdso_data->dcache_block_size = L1_CACHE_BYTES;
 777         vdso_data->dcache_log_block_size = L1_CACHE_SHIFT;
 778         vdso_data->icache_block_size = L1_CACHE_BYTES;
 779         vdso_data->icache_log_block_size = L1_CACHE_SHIFT;
 780 #endif /* CONFIG_PPC64 */
 781
 782
 783         /*
 784          * Calculate the size of the 32 bits vDSO
 785          */
 786         vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT;
 787         DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages);
 788
 789
 790         /*
 791          * Setup the syscall map in the vDOS
 792          */
 793         vdso_setup_syscall_map();
 794
 795         /*
 796          * Initialize the vDSO images in memory, that is do necessary
 797          * fixups of vDSO symbols, locate trampolines, etc...
 798          */
 799         if (vdso_setup()) {
 800                 printk(KERN_ERR "vDSO setup failure, not enabled !\n");
 801                 vdso32_pages = 0;
 802 #ifdef CONFIG_PPC64
 803                 vdso64_pages = 0;
 804 #endif
 805                 return 0;
 806         }
 807
 808         /* Make sure pages are in the correct state */
 809         vdso32_pagelist = kzalloc(sizeof(struct page *) * (vdso32_pages + 2),
 810                                   GFP_KERNEL);
 811         BUG_ON(vdso32_pagelist == NULL);
 812         for (i = 0; i < vdso32_pages; i++) {
 813                 struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE);
 814                 ClearPageReserved(pg);
 815                 get_page(pg);
 816                 vdso32_pagelist[i] = pg;
 817         }
 818         vdso32_pagelist[i++] = virt_to_page(vdso_data);
 819         vdso32_pagelist[i] = NULL;
 820
 821 #ifdef CONFIG_PPC64
 822         vdso64_pagelist = kzalloc(sizeof(struct page *) * (vdso64_pages + 2),
 823                                   GFP_KERNEL);
 824         BUG_ON(vdso64_pagelist == NULL);
 825         for (i = 0; i < vdso64_pages; i++) {
 826                 struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE);
 827                 ClearPageReserved(pg);
 828                 get_page(pg);
 829                 vdso64_pagelist[i] = pg;
 830         }
 831         vdso64_pagelist[i++] = virt_to_page(vdso_data);
 832         vdso64_pagelist[i] = NULL;
 833 #endif /* CONFIG_PPC64 */
 834
 835         get_page(virt_to_page(vdso_data));
 836
 837         smp_wmb();
 838         vdso_ready = 1;
 839
 840         return 0;
 841 }
 842 arch_initcall(vdso_init);