khugepaged.c 54.5 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
3
4
5
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/mm.h>
#include <linux/sched.h>
6
#include <linux/sched/mm.h>
7
#include <linux/sched/coredump.h>
8
9
10
11
12
13
14
15
16
17
18
19
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/mm_inline.h>
#include <linux/kthread.h>
#include <linux/khugepaged.h>
#include <linux/freezer.h>
#include <linux/mman.h>
#include <linux/hashtable.h>
#include <linux/userfaultfd_k.h>
#include <linux/page_idle.h>
#include <linux/swapops.h>
20
#include <linux/shmem_fs.h>
21
22
23
24
25
26
27
28
29
30
31
32

#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"

enum scan_result {
	SCAN_FAIL,
	SCAN_SUCCEED,
	SCAN_PMD_NULL,
	SCAN_EXCEED_NONE_PTE,
	SCAN_PTE_NON_PRESENT,
	SCAN_PAGE_RO,
33
	SCAN_LACK_REFERENCED_PAGE,
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
	SCAN_PAGE_NULL,
	SCAN_SCAN_ABORT,
	SCAN_PAGE_COUNT,
	SCAN_PAGE_LRU,
	SCAN_PAGE_LOCK,
	SCAN_PAGE_ANON,
	SCAN_PAGE_COMPOUND,
	SCAN_ANY_PROCESS,
	SCAN_VMA_NULL,
	SCAN_VMA_CHECK,
	SCAN_ADDRESS_RANGE,
	SCAN_SWAP_CACHE_PAGE,
	SCAN_DEL_PAGE_LRU,
	SCAN_ALLOC_HUGE_PAGE_FAIL,
	SCAN_CGROUP_CHARGE_FAIL,
49
50
	SCAN_EXCEED_SWAP_PTE,
	SCAN_TRUNCATED,
51
	SCAN_PAGE_HAS_PRIVATE,
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
};

#define CREATE_TRACE_POINTS
#include <trace/events/huge_memory.h>

/* default scan 8*512 pte (or vmas) every 30 second */
static unsigned int khugepaged_pages_to_scan __read_mostly;
static unsigned int khugepaged_pages_collapsed;
static unsigned int khugepaged_full_scans;
static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
/* during fragmentation poll the hugepage allocator once every minute */
static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
static unsigned long khugepaged_sleep_expire;
static DEFINE_SPINLOCK(khugepaged_mm_lock);
static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
/*
 * default collapse hugepages if there is at least one pte mapped like
 * it would have happened if the vma was large enough during page
 * fault.
 */
static unsigned int khugepaged_max_ptes_none __read_mostly;
static unsigned int khugepaged_max_ptes_swap __read_mostly;

#define MM_SLOTS_HASH_BITS 10
static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);

static struct kmem_cache *mm_slot_cache __read_mostly;

80
81
#define MAX_PTE_MAPPED_THP 8

82
83
84
85
86
87
88
89
90
91
/**
 * struct mm_slot - hash lookup from mm to mm_slot
 * @hash: hash collision list
 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
 * @mm: the mm that this information is valid for
 */
struct mm_slot {
	struct hlist_node hash;
	struct list_head mm_node;
	struct mm_struct *mm;
92
93
94
95

	/* pte-mapped THP in this mm */
	int nr_pte_mapped_thp;
	unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP];
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
};

/**
 * struct khugepaged_scan - cursor for scanning
 * @mm_head: the head of the mm list to scan
 * @mm_slot: the current mm_slot we are scanning
 * @address: the next address inside that to be scanned
 *
 * There is only the one khugepaged_scan instance of this cursor structure.
 */
struct khugepaged_scan {
	struct list_head mm_head;
	struct mm_slot *mm_slot;
	unsigned long address;
};

static struct khugepaged_scan khugepaged_scan = {
	.mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
};

116
#ifdef CONFIG_SYSFS
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
					 struct kobj_attribute *attr,
					 char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
}

static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
					  struct kobj_attribute *attr,
					  const char *buf, size_t count)
{
	unsigned long msecs;
	int err;

	err = kstrtoul(buf, 10, &msecs);
	if (err || msecs > UINT_MAX)
		return -EINVAL;

	khugepaged_scan_sleep_millisecs = msecs;
	khugepaged_sleep_expire = 0;
	wake_up_interruptible(&khugepaged_wait);

	return count;
}
static struct kobj_attribute scan_sleep_millisecs_attr =
	__ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
	       scan_sleep_millisecs_store);

static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
					  struct kobj_attribute *attr,
					  char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
}

static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
					   struct kobj_attribute *attr,
					   const char *buf, size_t count)
{
	unsigned long msecs;
	int err;

	err = kstrtoul(buf, 10, &msecs);
	if (err || msecs > UINT_MAX)
		return -EINVAL;

	khugepaged_alloc_sleep_millisecs = msecs;
	khugepaged_sleep_expire = 0;
	wake_up_interruptible(&khugepaged_wait);

	return count;
}
static struct kobj_attribute alloc_sleep_millisecs_attr =
	__ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
	       alloc_sleep_millisecs_store);

static ssize_t pages_to_scan_show(struct kobject *kobj,
				  struct kobj_attribute *attr,
				  char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
}
static ssize_t pages_to_scan_store(struct kobject *kobj,
				   struct kobj_attribute *attr,
				   const char *buf, size_t count)
{
	int err;
	unsigned long pages;

	err = kstrtoul(buf, 10, &pages);
	if (err || !pages || pages > UINT_MAX)
		return -EINVAL;

	khugepaged_pages_to_scan = pages;

	return count;
}
static struct kobj_attribute pages_to_scan_attr =
	__ATTR(pages_to_scan, 0644, pages_to_scan_show,
	       pages_to_scan_store);

static ssize_t pages_collapsed_show(struct kobject *kobj,
				    struct kobj_attribute *attr,
				    char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
}
static struct kobj_attribute pages_collapsed_attr =
	__ATTR_RO(pages_collapsed);

static ssize_t full_scans_show(struct kobject *kobj,
			       struct kobj_attribute *attr,
			       char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_full_scans);
}
static struct kobj_attribute full_scans_attr =
	__ATTR_RO(full_scans);

static ssize_t khugepaged_defrag_show(struct kobject *kobj,
				      struct kobj_attribute *attr, char *buf)
{
	return single_hugepage_flag_show(kobj, attr, buf,
				TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
}
static ssize_t khugepaged_defrag_store(struct kobject *kobj,
				       struct kobj_attribute *attr,
				       const char *buf, size_t count)
{
	return single_hugepage_flag_store(kobj, attr, buf, count,
				 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
}
static struct kobj_attribute khugepaged_defrag_attr =
	__ATTR(defrag, 0644, khugepaged_defrag_show,
	       khugepaged_defrag_store);

/*
 * max_ptes_none controls if khugepaged should collapse hugepages over
 * any unmapped ptes in turn potentially increasing the memory
 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
 * reduce the available free memory in the system as it
 * runs. Increasing max_ptes_none will instead potentially reduce the
 * free memory in the system during the khugepaged scan.
 */
static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
					     struct kobj_attribute *attr,
					     char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
}
static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
					      struct kobj_attribute *attr,
					      const char *buf, size_t count)
{
	int err;
	unsigned long max_ptes_none;

	err = kstrtoul(buf, 10, &max_ptes_none);
	if (err || max_ptes_none > HPAGE_PMD_NR-1)
		return -EINVAL;

	khugepaged_max_ptes_none = max_ptes_none;

	return count;
}
static struct kobj_attribute khugepaged_max_ptes_none_attr =
	__ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
	       khugepaged_max_ptes_none_store);

static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj,
					     struct kobj_attribute *attr,
					     char *buf)
{
	return sprintf(buf, "%u\n", khugepaged_max_ptes_swap);
}

static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj,
					      struct kobj_attribute *attr,
					      const char *buf, size_t count)
{
	int err;
	unsigned long max_ptes_swap;

	err  = kstrtoul(buf, 10, &max_ptes_swap);
	if (err || max_ptes_swap > HPAGE_PMD_NR-1)
		return -EINVAL;

	khugepaged_max_ptes_swap = max_ptes_swap;

	return count;
}

static struct kobj_attribute khugepaged_max_ptes_swap_attr =
	__ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show,
	       khugepaged_max_ptes_swap_store);

static struct attribute *khugepaged_attr[] = {
	&khugepaged_defrag_attr.attr,
	&khugepaged_max_ptes_none_attr.attr,
	&pages_to_scan_attr.attr,
	&pages_collapsed_attr.attr,
	&full_scans_attr.attr,
	&scan_sleep_millisecs_attr.attr,
	&alloc_sleep_millisecs_attr.attr,
	&khugepaged_max_ptes_swap_attr.attr,
	NULL,
};

struct attribute_group khugepaged_attr_group = {
	.attrs = khugepaged_attr,
	.name = "khugepaged",
};
309
#endif /* CONFIG_SYSFS */
310

311
#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406

int hugepage_madvise(struct vm_area_struct *vma,
		     unsigned long *vm_flags, int advice)
{
	switch (advice) {
	case MADV_HUGEPAGE:
#ifdef CONFIG_S390
		/*
		 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
		 * can't handle this properly after s390_enable_sie, so we simply
		 * ignore the madvise to prevent qemu from causing a SIGSEGV.
		 */
		if (mm_has_pgste(vma->vm_mm))
			return 0;
#endif
		*vm_flags &= ~VM_NOHUGEPAGE;
		*vm_flags |= VM_HUGEPAGE;
		/*
		 * If the vma become good for khugepaged to scan,
		 * register it here without waiting a page fault that
		 * may not happen any time soon.
		 */
		if (!(*vm_flags & VM_NO_KHUGEPAGED) &&
				khugepaged_enter_vma_merge(vma, *vm_flags))
			return -ENOMEM;
		break;
	case MADV_NOHUGEPAGE:
		*vm_flags &= ~VM_HUGEPAGE;
		*vm_flags |= VM_NOHUGEPAGE;
		/*
		 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
		 * this vma even if we leave the mm registered in khugepaged if
		 * it got registered before VM_NOHUGEPAGE was set.
		 */
		break;
	}

	return 0;
}

int __init khugepaged_init(void)
{
	mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
					  sizeof(struct mm_slot),
					  __alignof__(struct mm_slot), 0, NULL);
	if (!mm_slot_cache)
		return -ENOMEM;

	khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
	khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
	khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;

	return 0;
}

void __init khugepaged_destroy(void)
{
	kmem_cache_destroy(mm_slot_cache);
}

static inline struct mm_slot *alloc_mm_slot(void)
{
	if (!mm_slot_cache)	/* initialization failed */
		return NULL;
	return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
}

static inline void free_mm_slot(struct mm_slot *mm_slot)
{
	kmem_cache_free(mm_slot_cache, mm_slot);
}

static struct mm_slot *get_mm_slot(struct mm_struct *mm)
{
	struct mm_slot *mm_slot;

	hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
		if (mm == mm_slot->mm)
			return mm_slot;

	return NULL;
}

static void insert_to_mm_slots_hash(struct mm_struct *mm,
				    struct mm_slot *mm_slot)
{
	mm_slot->mm = mm;
	hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
}

static inline int khugepaged_test_exit(struct mm_struct *mm)
{
	return atomic_read(&mm->mm_users) == 0;
}

407
408
static bool hugepage_vma_check(struct vm_area_struct *vma,
			       unsigned long vm_flags)
409
{
410
411
	if ((!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
	    (vm_flags & VM_NOHUGEPAGE) ||
412
413
	    test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
		return false;
414
415
416
417
418

	if (shmem_file(vma->vm_file) ||
	    (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
	     vma->vm_file &&
	     (vm_flags & VM_DENYWRITE))) {
419
420
421
422
423
424
425
426
427
		if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
			return false;
		return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
				HPAGE_PMD_NR);
	}
	if (!vma->anon_vma || vma->vm_ops)
		return false;
	if (is_vma_temporary_stack(vma))
		return false;
428
	return !(vm_flags & VM_NO_KHUGEPAGED);
429
430
}

431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
int __khugepaged_enter(struct mm_struct *mm)
{
	struct mm_slot *mm_slot;
	int wakeup;

	mm_slot = alloc_mm_slot();
	if (!mm_slot)
		return -ENOMEM;

	/* __khugepaged_exit() must not run from under us */
	VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
	if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
		free_mm_slot(mm_slot);
		return 0;
	}

	spin_lock(&khugepaged_mm_lock);
	insert_to_mm_slots_hash(mm, mm_slot);
	/*
	 * Insert just behind the scanning cursor, to let the area settle
	 * down a little.
	 */
	wakeup = list_empty(&khugepaged_scan.mm_head);
	list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
	spin_unlock(&khugepaged_mm_lock);

Vegard Nossum's avatar
Vegard Nossum committed
457
	mmgrab(mm);
458
459
460
461
462
463
464
465
466
467
	if (wakeup)
		wake_up_interruptible(&khugepaged_wait);

	return 0;
}

int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
			       unsigned long vm_flags)
{
	unsigned long hstart, hend;
468
469

	/*
470
471
472
	 * khugepaged only supports read-only files for non-shmem files.
	 * khugepaged does not yet work on special mappings. And
	 * file-private shmem THP is not supported.
473
	 */
474
	if (!hugepage_vma_check(vma, vm_flags))
475
		return 0;
476

477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
	hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
	hend = vma->vm_end & HPAGE_PMD_MASK;
	if (hstart < hend)
		return khugepaged_enter(vma, vm_flags);
	return 0;
}

void __khugepaged_exit(struct mm_struct *mm)
{
	struct mm_slot *mm_slot;
	int free = 0;

	spin_lock(&khugepaged_mm_lock);
	mm_slot = get_mm_slot(mm);
	if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
		hash_del(&mm_slot->hash);
		list_del(&mm_slot->mm_node);
		free = 1;
	}
	spin_unlock(&khugepaged_mm_lock);

	if (free) {
		clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
		free_mm_slot(mm_slot);
		mmdrop(mm);
	} else if (mm_slot) {
		/*
		 * This is required to serialize against
		 * khugepaged_test_exit() (which is guaranteed to run
		 * under mmap sem read mode). Stop here (after we
		 * return all pagetables will be destroyed) until
		 * khugepaged has finished working on the pagetables
		 * under the mmap_sem.
		 */
		down_write(&mm->mmap_sem);
		up_write(&mm->mmap_sem);
	}
}

static void release_pte_page(struct page *page)
{
518
	dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page));
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
	unlock_page(page);
	putback_lru_page(page);
}

static void release_pte_pages(pte_t *pte, pte_t *_pte)
{
	while (--_pte >= pte) {
		pte_t pteval = *_pte;
		if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
			release_pte_page(pte_page(pteval));
	}
}

static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
					unsigned long address,
					pte_t *pte)
{
	struct page *page = NULL;
	pte_t *_pte;
538
539
	int none_or_zero = 0, result = 0, referenced = 0;
	bool writable = false;
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563

	for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
	     _pte++, address += PAGE_SIZE) {
		pte_t pteval = *_pte;
		if (pte_none(pteval) || (pte_present(pteval) &&
				is_zero_pfn(pte_pfn(pteval)))) {
			if (!userfaultfd_armed(vma) &&
			    ++none_or_zero <= khugepaged_max_ptes_none) {
				continue;
			} else {
				result = SCAN_EXCEED_NONE_PTE;
				goto out;
			}
		}
		if (!pte_present(pteval)) {
			result = SCAN_PTE_NON_PRESENT;
			goto out;
		}
		page = vm_normal_page(vma, address, pteval);
		if (unlikely(!page)) {
			result = SCAN_PAGE_NULL;
			goto out;
		}

564
565
566
567
568
569
		/* TODO: teach khugepaged to collapse THP mapped with pte */
		if (PageCompound(page)) {
			result = SCAN_PAGE_COMPOUND;
			goto out;
		}

570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
		VM_BUG_ON_PAGE(!PageAnon(page), page);

		/*
		 * We can do it before isolate_lru_page because the
		 * page can't be freed from under us. NOTE: PG_lock
		 * is needed to serialize against split_huge_page
		 * when invoked from the VM.
		 */
		if (!trylock_page(page)) {
			result = SCAN_PAGE_LOCK;
			goto out;
		}

		/*
		 * cannot use mapcount: can't collapse if there's a gup pin.
		 * The page must only be referenced by the scanned process
		 * and page swap cache.
		 */
588
		if (page_count(page) != 1 + PageSwapCache(page)) {
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
			unlock_page(page);
			result = SCAN_PAGE_COUNT;
			goto out;
		}
		if (pte_write(pteval)) {
			writable = true;
		} else {
			if (PageSwapCache(page) &&
			    !reuse_swap_page(page, NULL)) {
				unlock_page(page);
				result = SCAN_SWAP_CACHE_PAGE;
				goto out;
			}
			/*
			 * Page is not in the swap cache. It can be collapsed
			 * into a THP.
			 */
		}

		/*
		 * Isolate the page to avoid collapsing an hugepage
		 * currently in use by the VM.
		 */
		if (isolate_lru_page(page)) {
			unlock_page(page);
			result = SCAN_DEL_PAGE_LRU;
			goto out;
		}
617
618
		inc_node_page_state(page,
				NR_ISOLATED_ANON + page_is_file_cache(page));
619
620
621
		VM_BUG_ON_PAGE(!PageLocked(page), page);
		VM_BUG_ON_PAGE(PageLRU(page), page);

622
		/* There should be enough young pte to collapse the page */
623
624
625
		if (pte_young(pteval) ||
		    page_is_young(page) || PageReferenced(page) ||
		    mmu_notifier_test_young(vma->vm_mm, address))
626
			referenced++;
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
	}
	if (likely(writable)) {
		if (likely(referenced)) {
			result = SCAN_SUCCEED;
			trace_mm_collapse_huge_page_isolate(page, none_or_zero,
							    referenced, writable, result);
			return 1;
		}
	} else {
		result = SCAN_PAGE_RO;
	}

out:
	release_pte_pages(pte, _pte);
	trace_mm_collapse_huge_page_isolate(page, none_or_zero,
					    referenced, writable, result);
	return 0;
}

static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
				      struct vm_area_struct *vma,
				      unsigned long address,
				      spinlock_t *ptl)
{
	pte_t *_pte;
652
653
	for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
				_pte++, page++, address += PAGE_SIZE) {
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
		pte_t pteval = *_pte;
		struct page *src_page;

		if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
			clear_user_highpage(page, address);
			add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
			if (is_zero_pfn(pte_pfn(pteval))) {
				/*
				 * ptl mostly unnecessary.
				 */
				spin_lock(ptl);
				/*
				 * paravirt calls inside pte_clear here are
				 * superfluous.
				 */
				pte_clear(vma->vm_mm, address, _pte);
				spin_unlock(ptl);
			}
		} else {
			src_page = pte_page(pteval);
			copy_user_highpage(page, src_page, address, vma);
			VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
			release_pte_page(src_page);
			/*
			 * ptl mostly unnecessary, but preempt has to
			 * be disabled to update the per-cpu stats
			 * inside page_remove_rmap().
			 */
			spin_lock(ptl);
			/*
			 * paravirt calls inside pte_clear here are
			 * superfluous.
			 */
			pte_clear(vma->vm_mm, address, _pte);
			page_remove_rmap(src_page, false);
			spin_unlock(ptl);
			free_page_and_swap_cache(src_page);
		}
	}
}

static void khugepaged_alloc_sleep(void)
{
	DEFINE_WAIT(wait);

	add_wait_queue(&khugepaged_wait, &wait);
	freezable_schedule_timeout_interruptible(
		msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
	remove_wait_queue(&khugepaged_wait, &wait);
}

static int khugepaged_node_load[MAX_NUMNODES];

static bool khugepaged_scan_abort(int nid)
{
	int i;

	/*
712
	 * If node_reclaim_mode is disabled, then no extra effort is made to
713
714
	 * allocate memory locally.
	 */
715
	if (!node_reclaim_mode)
716
717
718
719
720
721
722
723
724
		return false;

	/* If there is a count for this node already, it must be acceptable */
	if (khugepaged_node_load[nid])
		return false;

	for (i = 0; i < MAX_NUMNODES; i++) {
		if (!khugepaged_node_load[i])
			continue;
725
		if (node_distance(nid, i) > node_reclaim_distance)
726
727
728
729
730
731
732
733
			return true;
	}
	return false;
}

/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
{
734
	return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
}

#ifdef CONFIG_NUMA
static int khugepaged_find_target_node(void)
{
	static int last_khugepaged_target_node = NUMA_NO_NODE;
	int nid, target_node = 0, max_value = 0;

	/* find first node with max normal pages hit */
	for (nid = 0; nid < MAX_NUMNODES; nid++)
		if (khugepaged_node_load[nid] > max_value) {
			max_value = khugepaged_node_load[nid];
			target_node = nid;
		}

	/* do some balance if several nodes have the same hit record */
	if (target_node <= last_khugepaged_target_node)
		for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
				nid++)
			if (max_value == khugepaged_node_load[nid]) {
				target_node = nid;
				break;
			}

	last_khugepaged_target_node = target_node;
	return target_node;
}

static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
{
	if (IS_ERR(*hpage)) {
		if (!*wait)
			return false;

		*wait = false;
		*hpage = NULL;
		khugepaged_alloc_sleep();
	} else if (*hpage) {
		put_page(*hpage);
		*hpage = NULL;
	}

	return true;
}

static struct page *
781
khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
{
	VM_BUG_ON_PAGE(*hpage, *hpage);

	*hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
	if (unlikely(!*hpage)) {
		count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
		*hpage = ERR_PTR(-ENOMEM);
		return NULL;
	}

	prep_transhuge_page(*hpage);
	count_vm_event(THP_COLLAPSE_ALLOC);
	return *hpage;
}
#else
static int khugepaged_find_target_node(void)
{
	return 0;
}

static inline struct page *alloc_khugepaged_hugepage(void)
{
	struct page *page;

	page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
			   HPAGE_PMD_ORDER);
	if (page)
		prep_transhuge_page(page);
	return page;
}

static struct page *khugepaged_alloc_hugepage(bool *wait)
{
	struct page *hpage;

	do {
		hpage = alloc_khugepaged_hugepage();
		if (!hpage) {
			count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
			if (!*wait)
				return NULL;

			*wait = false;
			khugepaged_alloc_sleep();
		} else
			count_vm_event(THP_COLLAPSE_ALLOC);
	} while (unlikely(!hpage) && likely(khugepaged_enabled()));

	return hpage;
}

static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
{
	if (!*hpage)
		*hpage = khugepaged_alloc_hugepage(wait);

	if (unlikely(!*hpage))
		return false;

	return true;
}

static struct page *
845
khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
846
847
848
849
850
851
852
853
854
855
856
857
858
859
{
	VM_BUG_ON(!*hpage);

	return  *hpage;
}
#endif

/*
 * If mmap_sem temporarily dropped, revalidate vma
 * before taking mmap_sem.
 * Return 0 if succeeds, otherwise return none-zero
 * value (scan code).
 */

860
861
static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
		struct vm_area_struct **vmap)
862
863
864
865
866
867
868
{
	struct vm_area_struct *vma;
	unsigned long hstart, hend;

	if (unlikely(khugepaged_test_exit(mm)))
		return SCAN_ANY_PROCESS;

869
	*vmap = vma = find_vma(mm, address);
870
871
872
873
874
875
876
	if (!vma)
		return SCAN_VMA_NULL;

	hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
	hend = vma->vm_end & HPAGE_PMD_MASK;
	if (address < hstart || address + HPAGE_PMD_SIZE > hend)
		return SCAN_ADDRESS_RANGE;
877
	if (!hugepage_vma_check(vma, vma->vm_flags))
878
879
880
881
882
883
884
885
886
887
888
889
890
891
		return SCAN_VMA_CHECK;
	return 0;
}

/*
 * Bring missing pages in from swap, to complete THP collapse.
 * Only done if khugepaged_scan_pmd believes it is worthwhile.
 *
 * Called and returns without pte mapped or spinlocks held,
 * but with mmap_sem held to protect against vma changes.
 */

static bool __collapse_huge_page_swapin(struct mm_struct *mm,
					struct vm_area_struct *vma,
892
893
					unsigned long address, pmd_t *pmd,
					int referenced)
894
{
895
896
	int swapped_in = 0;
	vm_fault_t ret = 0;
897
	struct vm_fault vmf = {
898
899
900
901
		.vma = vma,
		.address = address,
		.flags = FAULT_FLAG_ALLOW_RETRY,
		.pmd = pmd,
902
		.pgoff = linear_page_index(vma, address),
903
904
	};

905
906
907
908
909
	/* we only decide to swapin, if there is enough young ptes */
	if (referenced < HPAGE_PMD_NR/2) {
		trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
		return false;
	}
910
911
912
	vmf.pte = pte_offset_map(pmd, address);
	for (; vmf.address < address + HPAGE_PMD_NR*PAGE_SIZE;
			vmf.pte++, vmf.address += PAGE_SIZE) {
913
914
		vmf.orig_pte = *vmf.pte;
		if (!is_swap_pte(vmf.orig_pte))
915
916
			continue;
		swapped_in++;
917
		ret = do_swap_page(&vmf);
918

919
920
921
		/* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
		if (ret & VM_FAULT_RETRY) {
			down_read(&mm->mmap_sem);
922
			if (hugepage_vma_revalidate(mm, address, &vmf.vma)) {
923
				/* vma is no longer available, don't continue to swapin */
924
				trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
925
				return false;
926
			}
927
			/* check if the pmd is still valid */
928
929
			if (mm_find_pmd(mm, address) != pmd) {
				trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
930
				return false;
931
			}
932
933
		}
		if (ret & VM_FAULT_ERROR) {
934
			trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
935
936
937
			return false;
		}
		/* pte is unmapped now, we need to map it */
938
		vmf.pte = pte_offset_map(pmd, vmf.address);
939
	}
940
941
	vmf.pte--;
	pte_unmap(vmf.pte);
942
	trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
943
944
945
946
947
948
	return true;
}

static void collapse_huge_page(struct mm_struct *mm,
				   unsigned long address,
				   struct page **hpage,
949
				   int node, int referenced)
950
951
952
953
954
955
956
957
{
	pmd_t *pmd, _pmd;
	pte_t *pte;
	pgtable_t pgtable;
	struct page *new_page;
	spinlock_t *pmd_ptl, *pte_ptl;
	int isolated = 0, result = 0;
	struct mem_cgroup *memcg;
958
	struct vm_area_struct *vma;
959
	struct mmu_notifier_range range;
960
961
962
963
964
	gfp_t gfp;

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	/* Only allocate from the target node */
Michal Hocko's avatar
Michal Hocko committed
965
	gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
966

967
968
969
970
971
972
973
974
	/*
	 * Before allocating the hugepage, release the mmap_sem read lock.
	 * The allocation can take potentially a long time if it involves
	 * sync compaction, and we do not need to hold the mmap_sem during
	 * that. We will recheck the vma after taking it again in write mode.
	 */
	up_read(&mm->mmap_sem);
	new_page = khugepaged_alloc_page(hpage, gfp, node);
975
976
977
978
979
	if (!new_page) {
		result = SCAN_ALLOC_HUGE_PAGE_FAIL;
		goto out_nolock;
	}

980
	if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
981
982
983
984
985
		result = SCAN_CGROUP_CHARGE_FAIL;
		goto out_nolock;
	}

	down_read(&mm->mmap_sem);
986
	result = hugepage_vma_revalidate(mm, address, &vma);
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
	if (result) {
		mem_cgroup_cancel_charge(new_page, memcg, true);
		up_read(&mm->mmap_sem);
		goto out_nolock;
	}

	pmd = mm_find_pmd(mm, address);
	if (!pmd) {
		result = SCAN_PMD_NULL;
		mem_cgroup_cancel_charge(new_page, memcg, true);
		up_read(&mm->mmap_sem);
		goto out_nolock;
	}

	/*
	 * __collapse_huge_page_swapin always returns with mmap_sem locked.
1003
	 * If it fails, we release mmap_sem and jump out_nolock.
1004
1005
	 * Continuing to collapse causes inconsistency.
	 */
1006
	if (!__collapse_huge_page_swapin(mm, vma, address, pmd, referenced)) {
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
		mem_cgroup_cancel_charge(new_page, memcg, true);
		up_read(&mm->mmap_sem);
		goto out_nolock;
	}

	up_read(&mm->mmap_sem);
	/*
	 * Prevent all access to pagetables with the exception of
	 * gup_fast later handled by the ptep_clear_flush and the VM
	 * handled by the anon_vma lock + PG_lock.
	 */
	down_write(&mm->mmap_sem);
1019
1020
1021
	result = SCAN_ANY_PROCESS;
	if (!mmget_still_valid(mm))
		goto out;
1022
	result = hugepage_vma_revalidate(mm, address, &vma);
1023
1024
1025
1026
1027
1028
1029
1030
	if (result)
		goto out;
	/* check if the pmd is still valid */
	if (mm_find_pmd(mm, address) != pmd)
		goto out;

	anon_vma_lock_write(vma->anon_vma);

1031
	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm,
1032
				address, address + HPAGE_PMD_SIZE);
1033
	mmu_notifier_invalidate_range_start(&range);
1034
1035
1036
1037

	pte = pte_offset_map(pmd, address);
	pte_ptl = pte_lockptr(mm, pmd);

1038
1039
1040
1041
1042
1043
1044
1045
1046
	pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
	/*
	 * After this gup_fast can't run anymore. This also removes
	 * any huge TLB entry from the CPU so we won't allow
	 * huge and small TLB entries for the same virtual address
	 * to avoid the risk of CPU bugs in that area.
	 */
	_pmd = pmdp_collapse_flush(vma, address, pmd);
	spin_unlock(pmd_ptl);
1047
	mmu_notifier_invalidate_range_end(&range);
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080

	spin_lock(pte_ptl);
	isolated = __collapse_huge_page_isolate(vma, address, pte);
	spin_unlock(pte_ptl);

	if (unlikely(!isolated)) {
		pte_unmap(pte);
		spin_lock(pmd_ptl);
		BUG_ON(!pmd_none(*pmd));
		/*
		 * We can only use set_pmd_at when establishing
		 * hugepmds and never for establishing regular pmds that
		 * points to regular pagetables. Use pmd_populate for that
		 */
		pmd_populate(mm, pmd, pmd_pgtable(_pmd));
		spin_unlock(pmd_ptl);
		anon_vma_unlock_write(vma->anon_vma);
		result = SCAN_FAIL;
		goto out;
	}

	/*
	 * All pages are isolated and locked so anon_vma rmap
	 * can't run anymore.
	 */
	anon_vma_unlock_write(vma->anon_vma);

	__collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
	pte_unmap(pte);
	__SetPageUptodate(new_page);
	pgtable = pmd_pgtable(_pmd);

	_pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
1081
	_pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093

	/*
	 * spin_lock() below is not the equivalent of smp_wmb(), so
	 * this is needed to avoid the copy_huge_page writes to become
	 * visible after the set_pmd_at() write.
	 */
	smp_wmb();

	spin_lock(pmd_ptl);
	BUG_ON(!pmd_none(*pmd));
	page_add_new_anon_rmap(new_page, vma, address, true);
	mem_cgroup_commit_charge(new_page, memcg, false, true);
1094
	count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
	lru_cache_add_active_or_unevictable(new_page, vma);
	pgtable_trans_huge_deposit(mm, pmd, pgtable);
	set_pmd_at(mm, address, pmd, _pmd);
	update_mmu_cache_pmd(vma, address, pmd);
	spin_unlock(pmd_ptl);

	*hpage = NULL;

	khugepaged_pages_collapsed++;
	result = SCAN_SUCCEED;
out_up_write:
	up_write(&mm->mmap_sem);
out_nolock:
	trace_mm_collapse_huge_page(mm, isolated, result);
	return;
out:
	mem_cgroup_cancel_charge(new_page, memcg, true);
	goto out_up_write;
}

static int khugepaged_scan_pmd(struct mm_struct *mm,
			       struct vm_area_struct *vma,
			       unsigned long address,
			       struct page **hpage)
{
	pmd_t *pmd;
	pte_t *pte, *_pte;
1122
	int ret = 0, none_or_zero = 0, result = 0, referenced = 0;
1123
1124
1125
1126
	struct page *page = NULL;
	unsigned long _address;
	spinlock_t *ptl;
	int node = NUMA_NO_NODE, unmapped = 0;
1127
	bool writable = false;
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	pmd = mm_find_pmd(mm, address);
	if (!pmd) {
		result = SCAN_PMD_NULL;
		goto out;
	}

	memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
	pte = pte_offset_map_lock(mm, pmd, address, &ptl);
	for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
	     _pte++, _address += PAGE_SIZE) {
		pte_t pteval = *_pte;
		if (is_swap_pte(pteval)) {
			if (++unmapped <= khugepaged_max_ptes_swap) {
				continue;
			} else {
				result = SCAN_EXCEED_SWAP_PTE;
				goto out_unmap;
			}
		}
		if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
			if (!userfaultfd_armed(vma) &&
			    ++none_or_zero <= khugepaged_max_ptes_none) {
				continue;
			} else {
				result = SCAN_EXCEED_NONE_PTE;
				goto out_unmap;
			}
		}
		if (!pte_present(pteval)) {
			result = SCAN_PTE_NON_PRESENT;
			goto out_unmap;
		}
		if (pte_write(pteval))
			writable = true;

		page = vm_normal_page(vma, _address, pteval);
		if (unlikely(!page)) {
			result = SCAN_PAGE_NULL;
			goto out_unmap;
		}

		/* TODO: teach khugepaged to collapse THP mapped with pte */
		if (PageCompound(page)) {
			result = SCAN_PAGE_COMPOUND;
			goto out_unmap;
		}

		/*
		 * Record which node the original page is from and save this
		 * information to khugepaged_node_load[].
		 * Khupaged will allocate hugepage from the node has the max
		 * hit record.
		 */
		node = page_to_nid(page);
		if (khugepaged_scan_abort(node)) {
			result = SCAN_SCAN_ABORT;
			goto out_unmap;
		}
		khugepaged_node_load[node]++;
		if (!PageLRU(page)) {
			result = SCAN_PAGE_LRU;
			goto out_unmap;
		}
		if (PageLocked(page)) {
			result = SCAN_PAGE_LOCK;
			goto out_unmap;
		}
		if (!PageAnon(page)) {
			result = SCAN_PAGE_ANON;
			goto out_unmap;
		}

		/*
		 * cannot use mapcount: can't collapse if there's a gup pin.
		 * The page must only be referenced by the scanned process
		 * and page swap cache.
		 */
1208
		if (page_count(page) != 1 + PageSwapCache(page)) {
1209
1210
1211
1212
1213
1214
			result = SCAN_PAGE_COUNT;
			goto out_unmap;
		}
		if (pte_young(pteval) ||
		    page_is_young(page) || PageReferenced(page) ||
		    mmu_notifier_test_young(vma->vm_mm, address))
1215
			referenced++;
1216
1217
1218
1219
1220
1221
	}
	if (writable) {
		if (referenced) {
			result = SCAN_SUCCEED;
			ret = 1;
		} else {
1222
			result = SCAN_LACK_REFERENCED_PAGE;
1223
1224
1225
1226
1227
1228
1229
1230
1231
		}
	} else {
		result = SCAN_PAGE_RO;
	}
out_unmap:
	pte_unmap_unlock(pte, ptl);
	if (ret) {
		node = khugepaged_find_target_node();
		/* collapse_huge_page will return with the mmap_sem released */
1232
		collapse_huge_page(mm, address, hpage, node, referenced);
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
	}
out:
	trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
				     none_or_zero, result, unmapped);
	return ret;
}

static void collect_mm_slot(struct mm_slot *mm_slot)
{
	struct mm_struct *mm = mm_slot->mm;

1244
	lockdep_assert_held(&khugepaged_mm_lock);
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262

	if (khugepaged_test_exit(mm)) {
		/* free mm_slot */
		hash_del(&mm_slot->hash);
		list_del(&mm_slot->mm_node);

		/*
		 * Not strictly needed because the mm exited already.
		 *
		 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
		 */

		/* khugepaged_mm_lock actually not necessary for the below */
		free_mm_slot(mm_slot);
		mmdrop(mm);
	}
}

1263
#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
/*
 * Notify khugepaged that given addr of the mm is pte-mapped THP. Then
 * khugepaged should try to collapse the page table.
 */
static int khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
					 unsigned long addr)
{
	struct mm_slot *mm_slot;

	VM_BUG_ON(addr & ~HPAGE_PMD_MASK);

	spin_lock(&khugepaged_mm_lock);
	mm_slot = get_mm_slot(mm);
	if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP))
		mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr;
	spin_unlock(&khugepaged_mm_lock);
	return 0;
}

/**
 * Try to collapse a pte-mapped THP for mm at address haddr.
 *
 * This function checks whether all the PTEs in the PMD are pointing to the
 * right THP. If so, retract the page table so the THP can refault in with
 * as pmd-mapped.
 */
void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
{
	unsigned long haddr = addr & HPAGE_PMD_MASK;
	struct vm_area_struct *vma = find_vma(mm, haddr);
	struct page *hpage = NULL;
	pte_t *start_pte, *pte;
	pmd_t *pmd, _pmd;
	spinlock_t *ptl;
	int count = 0;
	int i;

	if (!vma || !vma->vm_file ||
	    vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE)
		return;

	/*
	 * This vm_flags may not have VM_HUGEPAGE if the page was not
	 * collapsed by this mm. But we can still collapse if the page is
	 * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check()
	 * will not fail the vma for missing VM_HUGEPAGE
	 */
	if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE))
		return;

	pmd = mm_find_pmd(mm, haddr);
	if (!pmd)
		return;

	start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl);

	/* step 1: check all mapped PTEs are to the right huge page */
	for (i = 0, addr = haddr, pte = start_pte;
	     i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {
		struct page *page;

		/* empty pte, skip */
		if (pte_none(*pte))
			continue;

		/* page swapped out, abort */
		if (!pte_present(*pte))
			goto abort;

		page = vm_normal_page(vma, addr, *pte);

		if (!page || !PageCompound(page))
			goto abort;

		if (!hpage) {
			hpage = compound_head(page);
			/*
			 * The mapping of the THP should not change.
			 *
			 * Note that uprobe, debugger, or MAP_PRIVATE may
			 * change the page table, but the new page will
			 * not pass PageCompound() check.
			 */
			if (WARN_ON(hpage->mapping != vma->vm_file->f_mapping))
				goto abort;
		}

		/*
		 * Confirm the page maps to the correct subpage.
		 *
		 * Note that uprobe, debugger, or MAP_PRIVATE may change
		 * the page table, but the new page will not pass
		 * PageCompound() check.
		 */
		if (WARN_ON(hpage + i != page))
			goto abort;
		count++;
	}

	/* step 2: adjust rmap */
	for (i = 0, addr = haddr, pte = start_pte;
	     i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {
		struct page *page;

		if (pte_none(*pte))
			continue;
		page = vm_normal_page(vma, addr, *pte);
		page_remove_rmap(page, false);
	}

	pte_unmap_unlock(start_pte, ptl);

	/* step 3: set proper refcount and mm_counters. */
	if (hpage) {
		page_ref_sub(hpage, count);
		add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count);
	}

	/* step 4: collapse pmd */
	ptl = pmd_lock(vma->vm_mm, pmd);
	_pmd = pmdp_collapse_flush(vma, addr, pmd);
	spin_unlock(ptl);
	mm_dec_nr_ptes(mm);
	pte_free(mm, pmd_pgtable(_pmd));
	return;

abort:
	pte_unmap_unlock(start_pte, ptl);
}

static int khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
{
	struct mm_struct *mm = mm_slot->mm;
	int i;

	if (likely(mm_slot->nr_pte_mapped_thp == 0))
		return 0;

	if (!down_write_trylock(&mm->mmap_sem))
		return -EBUSY;

	if (unlikely(khugepaged_test_exit(mm)))
		goto out;

	for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++)
		collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]);

out:
	mm_slot->nr_pte_mapped_thp = 0;
	up_write(&mm->mmap_sem);
	return 0;
}

1417
1418
1419
1420
1421
1422
1423
1424
static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
{
	struct vm_area_struct *vma;
	unsigned long addr;
	pmd_t *pmd, _pmd;

	i_mmap_lock_write(mapping);
	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
		/*
		 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that
		 * got written to. These VMAs are likely not worth investing
		 * down_write(mmap_sem) as PMD-mapping is likely to be split
		 * later.
		 *
		 * Not that vma->anon_vma check is racy: it can be set up after
		 * the check but before we took mmap_sem by the fault path.
		 * But page lock would prevent establishing any new ptes of the
		 * page, so we are safe.
		 *
		 * An alternative would be drop the check, but check that page
		 * table is clear before calling pmdp_collapse_flush() under
		 * ptl. It has higher chance to recover THP for the VMA, but
		 * has higher cost too.
		 */
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
		if (vma->anon_vma)
			continue;
		addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
		if (addr & ~HPAGE_PMD_MASK)
			continue;
		if (vma->vm_end < addr + HPAGE_PMD_SIZE)
			continue;
		pmd = mm_find_pmd(vma->vm_mm, addr);
		if (!pmd)
			continue;
		/*
		 * We need exclusive mmap_sem to retract page table.
1453
1454
1455
1456
		 *
		 * We use trylock due to lock inversion: we need to acquire
		 * mmap_sem while holding page lock. Fault path does it in
		 * reverse order. Trylock is a way to avoid deadlock.
1457
1458
1459
1460
1461
1462
1463
		 */
		if (down_write_trylock(&vma->vm_mm->mmap_sem)) {
			spinlock_t *ptl = pmd_lock(vma->vm_mm, pmd);
			/* assume page table is clear */
			_pmd = pmdp_collapse_flush(vma, addr, pmd);
			spin_unlock(ptl);
			up_write(&vma->vm_mm->mmap_sem);
1464
			mm_dec_nr_ptes(vma->vm_mm);
1465
			pte_free(vma->vm_mm, pmd_pgtable(_pmd));
1466
1467
1468
		} else {
			/* Try again later */
			khugepaged_add_pte_mapped_thp(vma->vm_mm, addr);
1469
1470
1471
1472
1473
1474
		}
	}
	i_mmap_unlock_write(mapping);
}

/**
1475
 * collapse_file - collapse filemap/tmpfs/shmem pages into huge one.
1476
1477
 *
 * Basic scheme is simple, details are more complex:
1478
 *  - allocate and lock a new huge page;
1479
 *  - scan page cache replacing old pages with the new one
1480
 *    + swap/gup in pages if necessary;
1481
 *    + fill in gaps;
1482
1483
 *    + keep old pages around in case rollback is required;
 *  - if replacing succeeds:
1484
1485
 *    + copy data over;
 *    + free old pages;
1486
 *    + unlock huge page;
1487
1488
 *  - if replacing failed;
 *    + put all pages back and unfreeze them;
1489
 *    + restore gaps in the page cache;
1490
 *    + unlock and free huge page;
1491
 */
1492
1493
static void collapse_file(struct mm_struct *mm,
		struct file *file, pgoff_t start,
1494
1495
		struct page **hpage, int node)
{
1496
	struct address_space *mapping = file->f_mapping;
1497
	gfp_t gfp;
1498
	struct page *new_page;
1499
1500
1501
	struct mem_cgroup *memcg;
	pgoff_t index, end = start + HPAGE_PMD_NR;
	LIST_HEAD(pagelist);
1502
	XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
1503
	int nr_none = 0, result = SCAN_SUCCEED;
1504
	bool is_shmem = shmem_file(file);
1505

1506
	VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
1507
1508
1509
	VM_BUG_ON(start & (HPAGE_PMD_NR - 1));

	/* Only allocate from the target node */
Michal Hocko's avatar
Michal Hocko committed
1510
	gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
1511
1512
1513
1514
1515
1516
1517

	new_page = khugepaged_alloc_page(hpage, gfp, node);
	if (!new_page) {
		result = SCAN_ALLOC_HUGE_PAGE_FAIL;
		goto out;
	}

1518
	if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
1519
1520
1521
1522
		result = SCAN_CGROUP_CHARGE_FAIL;
		goto out;
	}

1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
	/* This will be less messy when we use multi-index entries */
	do {
		xas_lock_irq(&xas);
		xas_create_range(&xas);
		if (!xas_error(&xas))
			break;
		xas_unlock_irq(&xas);
		if (!xas_nomem(&xas, GFP_KERNEL)) {
			mem_cgroup_cancel_charge(new_page, memcg, true);
			result = SCAN_FAIL;
			goto out;
		}
	} while (1);

1537
	__SetPageLocked(new_page);
1538
1539
	if (is_shmem)
		__SetPageSwapBacked(new_page);
1540
1541
1542
1543
	new_page->index = start;
	new_page->mapping = mapping;

	/*
1544
1545
1546
	 * At this point the new_page is locked and not up-to-date.
	 * It's safe to insert it into the page cache, because nobody would
	 * be able to map it or use it in another way until we unlock it.
1547
1548
	 */

1549
1550
1551
1552
1553
	xas_set(&xas, start);
	for (index = start; index < end; index++) {
		struct page *page = xas_next(&xas);

		VM_BUG_ON(index != xas.xa_index);
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
		if (is_shmem) {
			if (!page) {
				/*
				 * Stop if extent has been truncated or
				 * hole-punched, and is now completely
				 * empty.
				 */
				if (index == start) {
					if (!xas_next_entry(&xas, end - 1)) {
						result = SCAN_TRUNCATED;
						goto xa_locked;
					}
					xas_set(&xas, index);
				}
				if (!shmem_charge(mapping->host, 1)) {
					result = SCAN_FAIL;
1570
					goto xa_locked;
1571
				}
1572
1573
1574
				xas_store(&xas, new_page);
				nr_none++;
				continue;
1575
			}
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589

			if (xa_is_value(page) || !PageUptodate(page)) {
				xas_unlock_irq(&xas);
				/* swap in or instantiate fallocated page */
				if (shmem_getpage(mapping->host, index, &page,
						  SGP_NOHUGE)) {
					result = SCAN_FAIL;
					goto xa_unlocked;
				}
			} else if (trylock_page(page)) {
				get_page(page);
				xas_unlock_irq(&xas);
			} else {
				result = SCAN_PAGE_LOCK;
1590
				goto xa_locked;
1591
			}
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
		} else {	/* !is_shmem */
			if (!page || xa_is_value(page)) {
				xas_unlock_irq(&xas);
				page_cache_sync_readahead(mapping, &file->f_ra,
							  file, index,
							  PAGE_SIZE);
				/* drain pagevecs to help isolate_lru_page() */
				lru_add_drain();
				page = find_lock_page(mapping, index);
				if (unlikely(page == NULL)) {
					result = SCAN_FAIL;
					goto xa_unlocked;
				}
			} else if (trylock_page(page)) {
				get_page(page);
				xas_unlock_irq(&xas);
			} else {
				result = SCAN_PAGE_LOCK;
				goto xa_locked;
1611
1612
1613
1614
			}
		}

		/*
Matthew Wilcox's avatar
Matthew Wilcox committed
1615
		 * The page must be locked, so we can drop the i_pages lock
1616
1617
1618
		 * without racing with truncate.
		 */
		VM_BUG_ON_PAGE(!PageLocked(page), page);
1619
1620
1621
1622
1623
1624

		/* make sure the page is up to date */
		if (unlikely(!PageUptodate(page))) {
			result = SCAN_FAIL;
			goto out_unlock;
		}
1625
1626
1627
1628
1629
1630
1631
1632
1633

		/*
		 * If file was truncated then extended, or hole-punched, before
		 * we locked the first page, then a THP might be there already.
		 */
		if (PageTransCompound(page)) {
			result = SCAN_PAGE_COMPOUND;
			goto out_unlock;
		}
1634
1635
1636
1637
1638
1639

		if (page_mapping(page) != mapping) {
			result = SCAN_TRUNCATED;
			goto out_unlock;
		}

1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
		if (!is_shmem && PageDirty(page)) {
			/*
			 * khugepaged only works on read-only fd, so this
			 * page is dirty because it hasn't been flushed
			 * since first write.
			 */
			result = SCAN_FAIL;
			goto out_unlock;
		}

1650
1651
		if (isolate_lru_page(page)) {
			result = SCAN_DEL_PAGE_LRU;
1652
			goto out_unlock;
1653
1654
		}

1655
1656
1657
1658
1659
1660
		if (page_has_private(page) &&
		    !try_to_release_page(page, GFP_KERNEL)) {
			result = SCAN_PAGE_HAS_PRIVATE;
			goto out_unlock;
		}

1661
		if (page_mapped(page))
Matthew Wilcox's avatar
Matthew Wilcox committed
1662
			unmap_mapping_pages(mapping, index, 1, false);
1663

1664
1665
		xas_lock_irq(&xas);
		xas_set(&xas, index);
1666

1667
		VM_BUG_ON_PAGE(page != xas_load(&xas), page);
1668
1669
1670
1671
1672
		VM_BUG_ON_PAGE(page_mapped(page), page);

		/*
		 * The page is expected to have page_count() == 3:
		 *  - we hold a pin on it;
1673
		 *  - one reference from page cache;
1674
1675
1676
1677
		 *  - one from isolate_lru_page;
		 */
		if (!page_ref_freeze(page, 3)) {
			result = SCAN_PAGE_COUNT;
1678
1679
1680
			xas_unlock_irq(&xas);
			putback_lru_page(page);
			goto out_unlock;
1681
1682
1683
1684
1685
1686
1687
1688
1689
		}

		/*
		 * Add the page to the list to be able to undo the collapse if
		 * something go wrong.
		 */
		list_add_tail(&page->lru, &pagelist);

		/* Finally, replace with the new page. */
1690
		xas_store(&xas, new_page);
1691
1692
1693
1694
		continue;
out_unlock:
		unlock_page(page);
		put_page(page);
1695
		goto xa_unlocked;
1696
1697
	}

1698
1699
	if (is_shmem)
		__inc_node_page_state(new_page, NR_SHMEM_THPS);
1700
	else {
1701
		__inc_node_page_state(new_page, NR_FILE_THPS);
1702
1703
		filemap_nr_thps_inc(mapping);
	}
1704

1705
1706
1707
1708
	if (nr_none) {
		struct zone *zone = page_zone(new_page);

		__mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
1709
1710
1711
		if (is_shmem)
			__mod_node_page_state(zone->zone_pgdat,
					      NR_SHMEM, nr_none);
1712
1713
1714
1715
	}

xa_locked:
	xas_unlock_irq(&xas);
1716
xa_unlocked:
1717

1718
	if (result == SCAN_SUCCEED) {
1719
		struct page *page, *tmp;
1720
1721

		/*
1722
1723
		 * Replacing old pages with new one has succeeded, now we
		 * need to copy the content and free the old pages.
1724
		 */
1725
		index = start;
1726
		list_for_each_entry_safe(page, tmp, &pagelist, lru) {
1727
1728
1729
1730
			while (index < page->index) {
				clear_highpage(new_page + (index % HPAGE_PMD_NR));
				index++;
			}
1731
1732
1733
1734
			copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
					page);
			list_del(&page->lru);
			page->mapping = NULL;
1735
			page_ref_unfreeze(page, 1);
1736
1737
			ClearPageActive(page);
			ClearPageUnevictable(page);
1738
			unlock_page(page);
1739
			put_page(page);
1740
1741
1742
1743
1744
			index++;
		}
		while (index < end) {
			clear_highpage(new_page + (index % HPAGE_PMD_NR));
			index++;
1745
1746
1747
		}

		SetPageUptodate(new_page);
1748
		page_ref_add(new_page, HPAGE_PMD_NR - 1);
1749
		mem_cgroup_commit_charge(new_page, memcg, false, true);
1750
1751
1752
1753
1754
1755
1756

		if (is_shmem) {
			set_page_dirty(new_page);
			lru_cache_add_anon(new_page);
		} else {
			lru_cache_add_file(new_page);
		}
1757
		count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
1758

1759
1760
1761
1762
		/*
		 * Remove pte page tables, so we can re-fault the page as huge.
		 */
		retract_page_tables(mapping, start);
1763
		*hpage = NULL;
1764
1765

		khugepaged_pages_collapsed++;
1766
	} else {
1767
		struct page *page;
1768

1769
1770
		/* Something went wrong: roll back page cache changes */
		xas_lock_irq(&xas);
1771
		mapping->nrpages -= nr_none;
1772
1773
1774

		if (is_shmem)
			shmem_uncharge(mapping->host, nr_none);
1775

1776
1777
		xas_set(&xas, start);
		xas_for_each(&xas, page, end - 1) {
1778
1779
			page = list_first_entry_or_null(&pagelist,
					struct page, lru);
1780
			if (!page || xas.xa_index < page->index) {
1781
1782
1783
				if (!nr_none)
					break;
				nr_none--