mm: gup: FOLL_MM_SYNC: zeropage and MAP_PRIVATE pagecache

This removes the need of FOLL_FORCE|FOLL_WRITE for MAP_PRIVATE
pagecache and zeropages on anonymous mappings.

If compared to FOLL_FORCE|FOLL_WRITE, FOLL_MM_SYNC will avoid
unnecessary COWs and it'll retain MM coherency also for MAP_SHARED
PROT_READ.

Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>

5 files changed, 27 insertions, 13 deletions

diff --git a/include/linux/mm.h b/include/linux/mm.h
index bddd2495b0c614..46ce49b9189c8e 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2954,7 +2954,7 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
 }
 
 extern bool gup_must_unshare(unsigned int flags, struct page *page,
-			     bool is_head);
+			     bool is_head, struct vm_area_struct *vma);
 extern bool gup_must_unshare_irqsafe(unsigned int flags, struct page *page,
 				     bool is_head);
 
diff --git a/mm/gup.c b/mm/gup.c
index b1b57aee6f4025..6026e44ec522e1 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -101,7 +101,8 @@ static bool gup_must_unshare_hugetlbfs_slowpath(struct page *page)
  */
 static __always_inline bool __gup_must_unshare(unsigned int flags,
 					       struct page *page,
-					       bool is_head, bool irq_safe)
+					       bool is_head, bool irq_safe,
+					       struct vm_area_struct *vma)
 {
 	if (flags & (FOLL_WRITE|FOLL_NOUNSHARE))
 		return false;
@@ -109,7 +110,8 @@ static __always_inline bool __gup_must_unshare(unsigned int flags,
 	if (!(flags & (FOLL_GET|FOLL_PIN)))
 		return false;
 	if (!PageAnon(page))
-		return false;
+		return (flags & FOLL_MM_SYNC) &&
+			(irq_safe || !(vma->vm_flags & VM_SHARED));
 	if (PageKsm(page))
 		return gup_must_unshare_ksm(flags);
 	if (PageHuge(page)) {
@@ -138,16 +140,17 @@ static __always_inline bool __gup_must_unshare(unsigned int flags,
 }
 
 /* requires full accuracy */
-bool gup_must_unshare(unsigned int flags, struct page *page, bool is_head)
+bool gup_must_unshare(unsigned int flags, struct page *page, bool is_head,
+		      struct vm_area_struct *vma)
 {
-	return __gup_must_unshare(flags, page, is_head, false);
+	return __gup_must_unshare(flags, page, is_head, false, vma);
 }
 
 /* false positives are allowed, false negatives not allowed */
 bool gup_must_unshare_irqsafe(unsigned int flags, struct page *page,
 			      bool is_head)
 {
-	return __gup_must_unshare(flags, page, is_head, true);
+	return __gup_must_unshare(flags, page, is_head, true, NULL);
 }
 
 static void hpage_pincount_add(struct page *page, int refs)
@@ -712,7 +715,7 @@ retry:
 	 * exclusive.
 	 */
 	if (!pte_write(pte) &&
-	    gup_must_unshare(flags, page, false)) {
+	    gup_must_unshare(flags, page, false, vma)) {
 		page = ERR_PTR(-EMLINK);
 		goto out;
 	}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index b85c8f43becf56..777e567fff7fe9 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1418,7 +1418,7 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
 
 	/* see comments of the gup_must_unshare() callers in mm/gup.c */
 	if (!pmd_write(*pmd) &&
-	    gup_must_unshare(flags, page, true))
+	    gup_must_unshare(flags, page, true, vma))
 		return ERR_PTR(-EMLINK);
 
 	if (!try_grab_page(page, flags))
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 326d212c523bf5..8ee2f9b4eb2bc0 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5553,7 +5553,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
 		    (!huge_pte_write(pteval) &&
 		     ((flags & FOLL_WRITE) ||
 		      (unshare = gup_must_unshare(flags, pte_page(pteval),
-						  true))))) {
+						  true, vma))))) {
 			vm_fault_t ret;
 			unsigned int fault_flags = 0;
 
@@ -6255,6 +6255,7 @@ follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address, int flags
 			 (FOLL_PIN | FOLL_GET)))
 		return NULL;
 
+	mm = vma->vm_mm;
 retry:
 	ptep = huge_pte_offset(mm, address, huge_page_size(h));
 	if (!ptep)
@@ -6275,7 +6276,7 @@ retry:
 		 * check here is just in case.
 		 */
 		if (!huge_pte_write(pte) &&
-		    gup_must_unshare(flags, head_page, true)) {
+		    gup_must_unshare(flags, head_page, true, vma)) {
 			page = NULL;
 			goto out;
 		}
diff --git a/mm/memory.c b/mm/memory.c
index 9c4b2ed9e5445e..44e84b3014ccd1 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3252,7 +3252,12 @@ static vm_fault_t wp_page_unshare(struct vm_fault *vmf)
 	bool mm_sync = !!(vmf->flags & FAULT_FLAG_UNSHARE_MM_SYNC);
 	vmf->page = vm_normal_page(vmf->vma, vmf->address, vmf->orig_pte);
 	if (!vmf->page) {
-		goto out_unlock;
+		if (!mm_sync || !is_zero_pfn(pte_pfn(vmf->orig_pte)))
+			goto out_unlock;
+		if (vmf->vma->vm_flags & VM_SHARED) {
+			WARN_ON_ONCE(1);
+			goto out_unlock;
+		}
 	} else if (PageKsm(vmf->page)) {
 		if (!mm_sync)
 			goto out_unlock;
@@ -3318,12 +3323,17 @@ static vm_fault_t wp_page_unshare(struct vm_fault *vmf)
 		if (must_unshare)
 			return __wp_page_unshare(vmf);
 		goto out_unlock;
+	} else {
+		if (!mm_sync || vmf->vma->vm_flags & VM_SHARED)
+			goto out_unlock;
 	}
 
 	/*
-	 * Here the page can only be PageKsm.
+	 * Here the page can only be PageKsm a zeropage or a
+	 * MAP_PRIVATE pagecache.
 	 */
-	get_page(vmf->page);
+	if (vmf->page)
+		get_page(vmf->page);
 	pte_unmap_unlock(vmf->pte, vmf->ptl);
 	return wp_page_unshare_copy(vmf);