page_io.c 10.8 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 *  linux/mm/page_io.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  Swap reorganised 29.12.95, 
 *  Asynchronous swapping added 30.12.95. Stephen Tweedie
 *  Removed race in async swapping. 14.4.1996. Bruno Haible
 *  Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
 *  Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
 */

#include <linux/mm.h>
#include <linux/kernel_stat.h>
16
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
18
19
20
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/swapops.h>
21
#include <linux/buffer_head.h>
Linus Torvalds's avatar
Linus Torvalds committed
22
#include <linux/writeback.h>
23
#include <linux/frontswap.h>
24
#include <linux/blkdev.h>
25
#include <linux/psi.h>
26
#include <linux/uio.h>
27
#include <linux/sched/task.h>
Linus Torvalds's avatar
Linus Torvalds committed
28
29
#include <asm/pgtable.h>

30
static struct bio *get_swap_bio(gfp_t gfp_flags,
Linus Torvalds's avatar
Linus Torvalds committed
31
32
33
34
				struct page *page, bio_end_io_t end_io)
{
	struct bio *bio;

Huang Ying's avatar
Huang Ying committed
35
	bio = bio_alloc(gfp_flags, 1);
Linus Torvalds's avatar
Linus Torvalds committed
36
	if (bio) {
37
38
39
40
		struct block_device *bdev;

		bio->bi_iter.bi_sector = map_swap_page(page, &bdev);
		bio_set_dev(bio, bdev);
41
		bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
Linus Torvalds's avatar
Linus Torvalds committed
42
		bio->bi_end_io = end_io;
43

Huang Ying's avatar
Huang Ying committed
44
		bio_add_page(bio, page, PAGE_SIZE * hpage_nr_pages(page), 0);
Linus Torvalds's avatar
Linus Torvalds committed
45
46
47
48
	}
	return bio;
}

49
void end_swap_bio_write(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
50
{
51
	struct page *page = bio_first_page_all(bio);
Linus Torvalds's avatar
Linus Torvalds committed
52

53
	if (bio->bi_status) {
Linus Torvalds's avatar
Linus Torvalds committed
54
		SetPageError(page);
55
56
57
58
59
60
61
62
63
		/*
		 * We failed to write the page out to swap-space.
		 * Re-dirty the page in order to avoid it being reclaimed.
		 * Also print a dire warning that things will go BAD (tm)
		 * very quickly.
		 *
		 * Also clear PG_reclaim to avoid rotate_reclaimable_page()
		 */
		set_page_dirty(page);
64
		pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
65
			 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
66
			 (unsigned long long)bio->bi_iter.bi_sector);
67
68
		ClearPageReclaim(page);
	}
Linus Torvalds's avatar
Linus Torvalds committed
69
70
71
72
	end_page_writeback(page);
	bio_put(bio);
}

73
74
75
76
static void swap_slot_free_notify(struct page *page)
{
	struct swap_info_struct *sis;
	struct gendisk *disk;
77
	swp_entry_t entry;
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

	/*
	 * There is no guarantee that the page is in swap cache - the software
	 * suspend code (at least) uses end_swap_bio_read() against a non-
	 * swapcache page.  So we must check PG_swapcache before proceeding with
	 * this optimization.
	 */
	if (unlikely(!PageSwapCache(page)))
		return;

	sis = page_swap_info(page);
	if (!(sis->flags & SWP_BLKDEV))
		return;

	/*
	 * The swap subsystem performs lazy swap slot freeing,
	 * expecting that the page will be swapped out again.
	 * So we can avoid an unnecessary write if the page
	 * isn't redirtied.
	 * This is good for real swap storage because we can
	 * reduce unnecessary I/O and enhance wear-leveling
	 * if an SSD is used as the as swap device.
	 * But if in-memory swap device (eg zram) is used,
	 * this causes a duplicated copy between uncompressed
	 * data in VM-owned memory and compressed data in
	 * zram-owned memory.  So let's free zram-owned memory
	 * and make the VM-owned decompressed page *dirty*,
	 * so the page should be swapped out somewhere again if
	 * we again wish to reclaim it.
	 */
	disk = sis->bdev->bd_disk;
109
110
	entry.val = page_private(page);
	if (disk->fops->swap_slot_free_notify && __swap_count(entry) == 1) {
111
112
113
114
115
116
117
118
119
120
		unsigned long offset;

		offset = swp_offset(entry);

		SetPageDirty(page);
		disk->fops->swap_slot_free_notify(sis->bdev,
				offset);
	}
}

121
static void end_swap_bio_read(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
122
{
123
	struct page *page = bio_first_page_all(bio);
124
	struct task_struct *waiter = bio->bi_private;
Linus Torvalds's avatar
Linus Torvalds committed
125

126
	if (bio->bi_status) {
Linus Torvalds's avatar
Linus Torvalds committed
127
128
		SetPageError(page);
		ClearPageUptodate(page);
129
		pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
130
			 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
131
			 (unsigned long long)bio->bi_iter.bi_sector);
132
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
133
	}
134
135

	SetPageUptodate(page);
136
	swap_slot_free_notify(page);
137
out:
Linus Torvalds's avatar
Linus Torvalds committed
138
	unlock_page(page);
139
	WRITE_ONCE(bio->bi_private, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
140
	bio_put(bio);
141
142
143
144
	if (waiter) {
		blk_wake_io_task(waiter);
		put_task_struct(waiter);
	}
Linus Torvalds's avatar
Linus Torvalds committed
145
146
}

147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
int generic_swapfile_activate(struct swap_info_struct *sis,
				struct file *swap_file,
				sector_t *span)
{
	struct address_space *mapping = swap_file->f_mapping;
	struct inode *inode = mapping->host;
	unsigned blocks_per_page;
	unsigned long page_no;
	unsigned blkbits;
	sector_t probe_block;
	sector_t last_block;
	sector_t lowest_block = -1;
	sector_t highest_block = 0;
	int nr_extents = 0;
	int ret;

	blkbits = inode->i_blkbits;
	blocks_per_page = PAGE_SIZE >> blkbits;

	/*
167
	 * Map all the blocks into the extent tree.  This code doesn't try
168
169
170
171
172
173
174
175
176
177
	 * to be very smart.
	 */
	probe_block = 0;
	page_no = 0;
	last_block = i_size_read(inode) >> blkbits;
	while ((probe_block + blocks_per_page) <= last_block &&
			page_no < sis->max) {
		unsigned block_in_page;
		sector_t first_block;

178
179
		cond_resched();

180
181
182
		first_block = probe_block;
		ret = bmap(inode, &first_block);
		if (ret || !first_block)
183
184
185
186
187
188
189
190
191
192
193
194
195
196
			goto bad_bmap;

		/*
		 * It must be PAGE_SIZE aligned on-disk
		 */
		if (first_block & (blocks_per_page - 1)) {
			probe_block++;
			goto reprobe;
		}

		for (block_in_page = 1; block_in_page < blocks_per_page;
					block_in_page++) {
			sector_t block;

197
198
199
			block = probe_block + block_in_page;
			ret = bmap(inode, &block);
			if (ret || !block)
200
				goto bad_bmap;
201

202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
			if (block != first_block + block_in_page) {
				/* Discontiguity */
				probe_block++;
				goto reprobe;
			}
		}

		first_block >>= (PAGE_SHIFT - blkbits);
		if (page_no) {	/* exclude the header page */
			if (first_block < lowest_block)
				lowest_block = first_block;
			if (first_block > highest_block)
				highest_block = first_block;
		}

		/*
		 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
		 */
		ret = add_swap_extent(sis, page_no, 1, first_block);
		if (ret < 0)
			goto out;
		nr_extents += ret;
		page_no++;
		probe_block += blocks_per_page;
reprobe:
		continue;
	}
	ret = nr_extents;
	*span = 1 + highest_block - lowest_block;
	if (page_no == 0)
		page_no = 1;	/* force Empty message */
	sis->max = page_no;
	sis->pages = page_no - 1;
	sis->highest_bit = page_no - 1;
out:
	return ret;
bad_bmap:
239
	pr_err("swapon: swapfile has holes\n");
240
241
242
243
	ret = -EINVAL;
	goto out;
}

Linus Torvalds's avatar
Linus Torvalds committed
244
245
246
247
248
249
/*
 * We may have stale swap cache pages in memory: notice
 * them here and get rid of the unnecessary final write.
 */
int swap_writepage(struct page *page, struct writeback_control *wbc)
{
250
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
251

252
	if (try_to_free_swap(page)) {
Linus Torvalds's avatar
Linus Torvalds committed
253
254
255
		unlock_page(page);
		goto out;
	}
256
	if (frontswap_store(page) == 0) {
257
258
259
260
261
		set_page_writeback(page);
		unlock_page(page);
		end_page_writeback(page);
		goto out;
	}
262
	ret = __swap_writepage(page, wbc, end_swap_bio_write);
263
264
265
266
out:
	return ret;
}

267
268
static sector_t swap_page_sector(struct page *page)
{
269
	return (sector_t)__page_file_index(page) << (PAGE_SHIFT - 9);
270
271
}

272
273
274
275
276
277
278
279
280
static inline void count_swpout_vm_event(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	if (unlikely(PageTransHuge(page)))
		count_vm_event(THP_SWPOUT);
#endif
	count_vm_events(PSWPOUT, hpage_nr_pages(page));
}

281
int __swap_writepage(struct page *page, struct writeback_control *wbc,
282
		bio_end_io_t end_write_func)
283
284
{
	struct bio *bio;
285
	int ret;
286
	struct swap_info_struct *sis = page_swap_info(page);
287

288
	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
289
	if (sis->flags & SWP_FS) {
290
291
292
		struct kiocb kiocb;
		struct file *swap_file = sis->swap_file;
		struct address_space *mapping = swap_file->f_mapping;
Al Viro's avatar
Al Viro committed
293
294
295
296
297
		struct bio_vec bv = {
			.bv_page = page,
			.bv_len  = PAGE_SIZE,
			.bv_offset = 0
		};
Al Viro's avatar
Al Viro committed
298
		struct iov_iter from;
299

300
		iov_iter_bvec(&from, WRITE, &bv, 1, PAGE_SIZE);
301
302
303
		init_sync_kiocb(&kiocb, swap_file);
		kiocb.ki_pos = page_file_offset(page);

304
		set_page_writeback(page);
305
		unlock_page(page);
306
		ret = mapping->a_ops->direct_IO(&kiocb, &from);
307
308
309
		if (ret == PAGE_SIZE) {
			count_vm_event(PSWPOUT);
			ret = 0;
310
		} else {
311
312
313
314
315
316
317
318
319
320
			/*
			 * In the case of swap-over-nfs, this can be a
			 * temporary failure if the system has limited
			 * memory for allocating transmit buffers.
			 * Mark the page dirty and avoid
			 * rotate_reclaimable_page but rate-limit the
			 * messages but do not flag PageError like
			 * the normal direct-to-bio case as it could
			 * be temporary.
			 */
321
			set_page_dirty(page);
322
			ClearPageReclaim(page);
323
324
			pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
					   page_file_offset(page));
325
		}
326
		end_page_writeback(page);
327
328
329
		return ret;
	}

330
331
	ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
	if (!ret) {
332
		count_swpout_vm_event(page);
333
334
335
336
		return 0;
	}

	ret = 0;
337
	bio = get_swap_bio(GFP_NOIO, page, end_write_func);
Linus Torvalds's avatar
Linus Torvalds committed
338
339
340
341
342
343
	if (bio == NULL) {
		set_page_dirty(page);
		unlock_page(page);
		ret = -ENOMEM;
		goto out;
	}
Josef Bacik's avatar
Josef Bacik committed
344
	bio->bi_opf = REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc);
345
	bio_associate_blkg_from_page(bio, page);
346
	count_swpout_vm_event(page);
Linus Torvalds's avatar
Linus Torvalds committed
347
348
	set_page_writeback(page);
	unlock_page(page);
349
	submit_bio(bio);
Linus Torvalds's avatar
Linus Torvalds committed
350
351
352
353
out:
	return ret;
}

354
int swap_readpage(struct page *page, bool synchronous)
Linus Torvalds's avatar
Linus Torvalds committed
355
356
357
{
	struct bio *bio;
	int ret = 0;
358
	struct swap_info_struct *sis = page_swap_info(page);
359
	blk_qc_t qc;
360
	struct gendisk *disk;
361
	unsigned long pflags;
Linus Torvalds's avatar
Linus Torvalds committed
362

363
	VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
364
365
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(PageUptodate(page), page);
366
367
368
369
370
371
372
373

	/*
	 * Count submission time as memory stall. When the device is congested,
	 * or the submitting cgroup IO-throttled, submission can be a
	 * significant part of overall IO time.
	 */
	psi_memstall_enter(&pflags);

374
	if (frontswap_load(page) == 0) {
375
376
377
378
		SetPageUptodate(page);
		unlock_page(page);
		goto out;
	}
379

380
	if (sis->flags & SWP_FS) {
381
382
383
384
385
386
		struct file *swap_file = sis->swap_file;
		struct address_space *mapping = swap_file->f_mapping;

		ret = mapping->a_ops->readpage(swap_file, page);
		if (!ret)
			count_vm_event(PSWPIN);
387
		goto out;
388
389
	}

390
391
	ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
	if (!ret) {
392
393
394
395
396
		if (trylock_page(page)) {
			swap_slot_free_notify(page);
			unlock_page(page);
		}

397
		count_vm_event(PSWPIN);
398
		goto out;
399
400
401
	}

	ret = 0;
402
	bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);
Linus Torvalds's avatar
Linus Torvalds committed
403
404
405
406
407
	if (bio == NULL) {
		unlock_page(page);
		ret = -ENOMEM;
		goto out;
	}
408
	disk = bio->bi_disk;
409
410
411
412
	/*
	 * Keep this task valid during swap readpage because the oom killer may
	 * attempt to access it in the page fault retry time check.
	 */
413
	bio_set_op_attrs(bio, REQ_OP_READ, 0);
414
	if (synchronous) {
415
		bio->bi_opf |= REQ_HIPRI;
416
417
418
		get_task_struct(current);
		bio->bi_private = current;
	}
419
	count_vm_event(PSWPIN);
420
421
	bio_get(bio);
	qc = submit_bio(bio);
422
	while (synchronous) {
423
		set_current_state(TASK_UNINTERRUPTIBLE);
424
425
426
		if (!READ_ONCE(bio->bi_private))
			break;

427
		if (!blk_poll(disk->queue, qc, true))
428
			io_schedule();
429
430
431
432
	}
	__set_current_state(TASK_RUNNING);
	bio_put(bio);

Linus Torvalds's avatar
Linus Torvalds committed
433
out:
434
	psi_memstall_leave(&pflags);
Linus Torvalds's avatar
Linus Torvalds committed
435
436
	return ret;
}
437
438
439
440
441

int swap_set_page_dirty(struct page *page)
{
	struct swap_info_struct *sis = page_swap_info(page);

442
	if (sis->flags & SWP_FS) {
443
		struct address_space *mapping = sis->swap_file->f_mapping;
444
445

		VM_BUG_ON_PAGE(!PageSwapCache(page), page);
446
447
448
449
450
		return mapping->a_ops->set_page_dirty(page);
	} else {
		return __set_page_dirty_no_writeback(page);
	}
}