Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

[cascardo/linux.git] / arch / s390 / mm / pgtable.c

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 293130b..5f09201 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -27,40 +27,37 @@
 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
                                      unsigned long addr, pte_t *ptep)
 {
-       int active, count;
        pte_t old;
 
        old = *ptep;
        if (unlikely(pte_val(old) & _PAGE_INVALID))
                return old;
-       active = (mm == current->active_mm) ? 1 : 0;
-       count = atomic_add_return(0x10000, &mm->context.attach_count);
-       if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
+       atomic_inc(&mm->context.flush_count);
+       if (MACHINE_HAS_TLB_LC &&
            cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
                __ptep_ipte_local(addr, ptep);
        else
                __ptep_ipte(addr, ptep);
-       atomic_sub(0x10000, &mm->context.attach_count);
+       atomic_dec(&mm->context.flush_count);
        return old;
 }
 
 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
                                    unsigned long addr, pte_t *ptep)
 {
-       int active, count;
        pte_t old;
 
        old = *ptep;
        if (unlikely(pte_val(old) & _PAGE_INVALID))
                return old;
-       active = (mm == current->active_mm) ? 1 : 0;
-       count = atomic_add_return(0x10000, &mm->context.attach_count);
-       if ((count & 0xffff) <= active) {
+       atomic_inc(&mm->context.flush_count);
+       if (cpumask_equal(&mm->context.cpu_attach_mask,
+                         cpumask_of(smp_processor_id()))) {
                pte_val(*ptep) |= _PAGE_INVALID;
                mm->context.flush_mm = 1;
        } else
                __ptep_ipte(addr, ptep);
-       atomic_sub(0x10000, &mm->context.attach_count);
+       atomic_dec(&mm->context.flush_count);
        return old;
 }
 
@@ -70,7 +67,6 @@ static inline pgste_t pgste_get_lock(pte_t *ptep)
 #ifdef CONFIG_PGSTE
        unsigned long old;
 
-       preempt_disable();
        asm(
                "       lg      %0,%2\n"
                "0:     lgr     %1,%0\n"
@@ -93,7 +89,6 @@ static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
                : "=Q" (ptep[PTRS_PER_PTE])
                : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
                : "cc", "memory");
-       preempt_enable();
 #endif
 }
 
@@ -233,9 +228,11 @@ pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
        pgste_t pgste;
        pte_t old;
 
+       preempt_disable();
        pgste = ptep_xchg_start(mm, addr, ptep);
        old = ptep_flush_direct(mm, addr, ptep);
        ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
+       preempt_enable();
        return old;
 }
 EXPORT_SYMBOL(ptep_xchg_direct);
@@ -246,9 +243,11 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
        pgste_t pgste;
        pte_t old;
 
+       preempt_disable();
        pgste = ptep_xchg_start(mm, addr, ptep);
        old = ptep_flush_lazy(mm, addr, ptep);
        ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
+       preempt_enable();
        return old;
 }
 EXPORT_SYMBOL(ptep_xchg_lazy);
@@ -259,6 +258,7 @@ pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
        pgste_t pgste;
        pte_t old;
 
+       preempt_disable();
        pgste = ptep_xchg_start(mm, addr, ptep);
        old = ptep_flush_lazy(mm, addr, ptep);
        if (mm_has_pgste(mm)) {
@@ -282,13 +282,13 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
        } else {
                *ptep = pte;
        }
+       preempt_enable();
 }
 EXPORT_SYMBOL(ptep_modify_prot_commit);
 
 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
                                      unsigned long addr, pmd_t *pmdp)
 {
-       int active, count;
        pmd_t old;
 
        old = *pmdp;
@@ -298,36 +298,34 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
                __pmdp_csp(pmdp);
                return old;
        }
-       active = (mm == current->active_mm) ? 1 : 0;
-       count = atomic_add_return(0x10000, &mm->context.attach_count);
-       if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
+       atomic_inc(&mm->context.flush_count);
+       if (MACHINE_HAS_TLB_LC &&
            cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
                __pmdp_idte_local(addr, pmdp);
        else
                __pmdp_idte(addr, pmdp);
-       atomic_sub(0x10000, &mm->context.attach_count);
+       atomic_dec(&mm->context.flush_count);
        return old;
 }
 
 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
                                    unsigned long addr, pmd_t *pmdp)
 {
-       int active, count;
        pmd_t old;
 
        old = *pmdp;
        if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
                return old;
-       active = (mm == current->active_mm) ? 1 : 0;
-       count = atomic_add_return(0x10000, &mm->context.attach_count);
-       if ((count & 0xffff) <= active) {
+       atomic_inc(&mm->context.flush_count);
+       if (cpumask_equal(&mm->context.cpu_attach_mask,
+                         cpumask_of(smp_processor_id()))) {
                pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
                mm->context.flush_mm = 1;
        } else if (MACHINE_HAS_IDTE)
                __pmdp_idte(addr, pmdp);
        else
                __pmdp_csp(pmdp);
-       atomic_sub(0x10000, &mm->context.attach_count);
+       atomic_dec(&mm->context.flush_count);
        return old;
 }
 
@@ -336,8 +334,10 @@ pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
 {
        pmd_t old;
 
+       preempt_disable();
        old = pmdp_flush_direct(mm, addr, pmdp);
        *pmdp = new;
+       preempt_enable();
        return old;
 }
 EXPORT_SYMBOL(pmdp_xchg_direct);
@@ -347,12 +347,53 @@ pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
 {
        pmd_t old;
 
+       preempt_disable();
        old = pmdp_flush_lazy(mm, addr, pmdp);
        *pmdp = new;
+       preempt_enable();
        return old;
 }
 EXPORT_SYMBOL(pmdp_xchg_lazy);
 
+static inline pud_t pudp_flush_direct(struct mm_struct *mm,
+                                     unsigned long addr, pud_t *pudp)
+{
+       pud_t old;
+
+       old = *pudp;
+       if (pud_val(old) & _REGION_ENTRY_INVALID)
+               return old;
+       if (!MACHINE_HAS_IDTE) {
+               /*
+                * Invalid bit position is the same for pmd and pud, so we can
+                * re-use _pmd_csp() here
+                */
+               __pmdp_csp((pmd_t *) pudp);
+               return old;
+       }
+       atomic_inc(&mm->context.flush_count);
+       if (MACHINE_HAS_TLB_LC &&
+           cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
+               __pudp_idte_local(addr, pudp);
+       else
+               __pudp_idte(addr, pudp);
+       atomic_dec(&mm->context.flush_count);
+       return old;
+}
+
+pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr,
+                      pud_t *pudp, pud_t new)
+{
+       pud_t old;
+
+       preempt_disable();
+       old = pudp_flush_direct(mm, addr, pudp);
+       *pudp = new;
+       preempt_enable();
+       return old;
+}
+EXPORT_SYMBOL(pudp_xchg_direct);
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
                                pgtable_t pgtable)
@@ -401,20 +442,24 @@ void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
        pgste_t pgste;
 
        /* the mm_has_pgste() check is done in set_pte_at() */
+       preempt_disable();
        pgste = pgste_get_lock(ptep);
        pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
        pgste_set_key(ptep, pgste, entry, mm);
        pgste = pgste_set_pte(ptep, pgste, entry);
        pgste_set_unlock(ptep, pgste);
+       preempt_enable();
 }
 
 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
        pgste_t pgste;
 
+       preempt_disable();
        pgste = pgste_get_lock(ptep);
        pgste_val(pgste) |= PGSTE_IN_BIT;
        pgste_set_unlock(ptep, pgste);
+       preempt_enable();
 }
 
 /**
@@ -521,10 +566,11 @@ void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
        pte_t pte;
 
        /* Zap unused and logically-zero pages */
+       preempt_disable();
        pgste = pgste_get_lock(ptep);
        pgstev = pgste_val(pgste);
        pte = *ptep;
-       if (pte_swap(pte) &&
+       if (!reset && pte_swap(pte) &&
            ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
             (pgstev & _PGSTE_GPS_ZERO))) {
                ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
@@ -533,6 +579,7 @@ void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
        if (reset)
                pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
        pgste_set_unlock(ptep, pgste);
+       preempt_enable();
 }
 
 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
@@ -541,6 +588,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
        pgste_t pgste;
 
        /* Clear storage key */
+       preempt_disable();
        pgste = pgste_get_lock(ptep);
        pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT |
                              PGSTE_GR_BIT | PGSTE_GC_BIT);
@@ -548,6 +596,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
        if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
                page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
        pgste_set_unlock(ptep, pgste);
+       preempt_enable();
 }
 
 /*