diff --git a/mm/migrate.c b/mm/migrate.c
index 8f91463eab4e..96b9546e69e0 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -164,7 +164,6 @@ static void remove_migration_pte(struct vm_area_struct *vma, unsigned long addr,
 	if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
 		goto out;
 
-	inc_mm_counter(mm, anon_rss);
 	get_page(new);
 	pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
 	if (is_write_migration_entry(entry))
diff --git a/mm/rmap.c b/mm/rmap.c
index a53a10b93ecf..05d6d73a692d 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -595,6 +595,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 					list_add(&mm->mmlist, &init_mm.mmlist);
 				spin_unlock(&mmlist_lock);
 			}
+			dec_mm_counter(mm, anon_rss);
 		} else {
 			/*
 			 * Store the pfn of the page in a special migration
@@ -606,7 +607,6 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 		}
 		set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
 		BUG_ON(pte_file(*pte));
-		dec_mm_counter(mm, anon_rss);
 	} else
 		dec_mm_counter(mm, file_rss);