zswap.c 34.5 KB
Newer Older
Seth Jennings's avatar
Seth Jennings committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/*
 * zswap.c - zswap driver file
 *
 * zswap is a backend for frontswap that takes pages that are in the process
 * of being swapped out and attempts to compress and store them in a
 * RAM-based memory pool.  This can result in a significant I/O reduction on
 * the swap device and, in the case where decompressing from RAM is faster
 * than reading from the swap device, can also improve workload performance.
 *
 * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
*/

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/frontswap.h>
#include <linux/rbtree.h>
#include <linux/swap.h>
#include <linux/crypto.h>
#include <linux/mempool.h>
37
#include <linux/zpool.h>
Seth Jennings's avatar
Seth Jennings committed
38
39
40
41
42
43
44
45
46
47

#include <linux/mm_types.h>
#include <linux/page-flags.h>
#include <linux/swapops.h>
#include <linux/writeback.h>
#include <linux/pagemap.h>

/*********************************
* statistics
**********************************/
48
49
/* Total bytes used by the compressed storage */
static u64 zswap_pool_total_size;
Seth Jennings's avatar
Seth Jennings committed
50
51
/* The number of compressed pages currently stored in zswap */
static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
52
53
/* The number of same-value filled pages currently stored in zswap */
static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0);
Seth Jennings's avatar
Seth Jennings committed
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

/*
 * The statistics below are not protected from concurrent access for
 * performance reasons so they may not be a 100% accurate.  However,
 * they do provide useful information on roughly how many times a
 * certain event is occurring.
*/

/* Pool limit was hit (see zswap_max_pool_percent) */
static u64 zswap_pool_limit_hit;
/* Pages written back when pool limit was reached */
static u64 zswap_written_back_pages;
/* Store failed due to a reclaim failure after pool limit was reached */
static u64 zswap_reject_reclaim_fail;
/* Compressed page was too big for the allocator to (optimally) store */
static u64 zswap_reject_compress_poor;
/* Store failed because underlying allocator could not get memory */
static u64 zswap_reject_alloc_fail;
/* Store failed because the entry metadata could not be allocated (rare) */
static u64 zswap_reject_kmemcache_fail;
/* Duplicate store was encountered (rare) */
static u64 zswap_duplicate_entry;

/*********************************
* tunables
**********************************/
Dan Streetman's avatar
Dan Streetman committed
80

81
82
#define ZSWAP_PARAM_UNSET ""

Dan Streetman's avatar
Dan Streetman committed
83
84
/* Enable/disable zswap (disabled by default) */
static bool zswap_enabled;
85
86
87
88
89
90
91
static int zswap_enabled_param_set(const char *,
				   const struct kernel_param *);
static struct kernel_param_ops zswap_enabled_param_ops = {
	.set =		zswap_enabled_param_set,
	.get =		param_get_bool,
};
module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
Seth Jennings's avatar
Seth Jennings committed
92

93
/* Crypto compressor to use */
Seth Jennings's avatar
Seth Jennings committed
94
#define ZSWAP_COMPRESSOR_DEFAULT "lzo"
95
static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
96
97
98
99
static int zswap_compressor_param_set(const char *,
				      const struct kernel_param *);
static struct kernel_param_ops zswap_compressor_param_ops = {
	.set =		zswap_compressor_param_set,
100
101
	.get =		param_get_charp,
	.free =		param_free_charp,
102
103
};
module_param_cb(compressor, &zswap_compressor_param_ops,
104
		&zswap_compressor, 0644);
Seth Jennings's avatar
Seth Jennings committed
105

106
/* Compressed storage zpool to use */
107
#define ZSWAP_ZPOOL_DEFAULT "zbud"
108
static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
109
110
static int zswap_zpool_param_set(const char *, const struct kernel_param *);
static struct kernel_param_ops zswap_zpool_param_ops = {
111
112
113
	.set =		zswap_zpool_param_set,
	.get =		param_get_charp,
	.free =		param_free_charp,
114
};
115
module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
116

117
118
119
/* The maximum percentage of memory that the compressed pool can occupy */
static unsigned int zswap_max_pool_percent = 20;
module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
120

121
122
123
124
125
/* Enable/disable handling same-value filled pages (enabled by default) */
static bool zswap_same_filled_pages_enabled = true;
module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled,
		   bool, 0644);

Seth Jennings's avatar
Seth Jennings committed
126
/*********************************
Dan Streetman's avatar
Dan Streetman committed
127
* data structures
Seth Jennings's avatar
Seth Jennings committed
128
129
**********************************/

Dan Streetman's avatar
Dan Streetman committed
130
131
132
133
134
struct zswap_pool {
	struct zpool *zpool;
	struct crypto_comp * __percpu *tfm;
	struct kref kref;
	struct list_head list;
135
	struct work_struct work;
136
	struct hlist_node node;
Dan Streetman's avatar
Dan Streetman committed
137
	char tfm_name[CRYPTO_MAX_ALG_NAME];
Seth Jennings's avatar
Seth Jennings committed
138
139
140
141
142
143
144
145
146
};

/*
 * struct zswap_entry
 *
 * This structure contains the metadata for tracking a single compressed
 * page within zswap.
 *
 * rbnode - links the entry into red-black tree for the appropriate swap type
Dan Streetman's avatar
Dan Streetman committed
147
 * offset - the swap offset for the entry.  Index into the red-black tree.
Seth Jennings's avatar
Seth Jennings committed
148
149
 * refcount - the number of outstanding reference to the entry. This is needed
 *            to protect against premature freeing of the entry by code
150
 *            concurrent calls to load, invalidate, and writeback.  The lock
Seth Jennings's avatar
Seth Jennings committed
151
152
153
154
 *            for the zswap_tree structure that contains the entry must
 *            be held while changing the refcount.  Since the lock must
 *            be held, there is no reason to also make refcount atomic.
 * length - the length in bytes of the compressed page data.  Needed during
155
 *          decompression. For a same value filled page length is 0.
Dan Streetman's avatar
Dan Streetman committed
156
157
 * pool - the zswap_pool the entry's data is in
 * handle - zpool allocation handle that stores the compressed page data
158
 * value - value of the same-value filled pages which have same content
Seth Jennings's avatar
Seth Jennings committed
159
160
161
162
163
164
 */
struct zswap_entry {
	struct rb_node rbnode;
	pgoff_t offset;
	int refcount;
	unsigned int length;
Dan Streetman's avatar
Dan Streetman committed
165
	struct zswap_pool *pool;
166
167
168
169
	union {
		unsigned long handle;
		unsigned long value;
	};
Seth Jennings's avatar
Seth Jennings committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
};

struct zswap_header {
	swp_entry_t swpentry;
};

/*
 * The tree lock in the zswap_tree struct protects a few things:
 * - the rbtree
 * - the refcount field of each entry in the tree
 */
struct zswap_tree {
	struct rb_root rbroot;
	spinlock_t lock;
};

static struct zswap_tree *zswap_trees[MAX_SWAPFILES];

Dan Streetman's avatar
Dan Streetman committed
188
189
190
191
/* RCU-protected iteration */
static LIST_HEAD(zswap_pools);
/* protects zswap_pools list modification */
static DEFINE_SPINLOCK(zswap_pools_lock);
192
193
/* pool counter to provide unique names to zpool */
static atomic_t zswap_pools_count = ATOMIC_INIT(0);
Dan Streetman's avatar
Dan Streetman committed
194

195
196
197
/* used by param callback function */
static bool zswap_init_started;

198
199
200
/* fatal error during init */
static bool zswap_init_failed;

201
202
203
/* init completed, but couldn't create the initial pool */
static bool zswap_has_pool;

Dan Streetman's avatar
Dan Streetman committed
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
/*********************************
* helpers and fwd declarations
**********************************/

#define zswap_pool_debug(msg, p)				\
	pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name,		\
		 zpool_get_type((p)->zpool))

static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
static int zswap_pool_get(struct zswap_pool *pool);
static void zswap_pool_put(struct zswap_pool *pool);

static const struct zpool_ops zswap_zpool_ops = {
	.evict = zswap_writeback_entry
};

static bool zswap_is_full(void)
{
	return totalram_pages * zswap_max_pool_percent / 100 <
		DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
}

static void zswap_update_total_size(void)
{
	struct zswap_pool *pool;
	u64 total = 0;

	rcu_read_lock();

	list_for_each_entry_rcu(pool, &zswap_pools, list)
		total += zpool_get_total_size(pool->zpool);

	rcu_read_unlock();

	zswap_pool_total_size = total;
}

Seth Jennings's avatar
Seth Jennings committed
241
242
243
244
245
/*********************************
* zswap entry functions
**********************************/
static struct kmem_cache *zswap_entry_cache;

246
static int __init zswap_entry_cache_create(void)
Seth Jennings's avatar
Seth Jennings committed
247
248
{
	zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
249
	return zswap_entry_cache == NULL;
Seth Jennings's avatar
Seth Jennings committed
250
251
}

252
static void __init zswap_entry_cache_destroy(void)
Seth Jennings's avatar
Seth Jennings committed
253
254
255
256
257
258
259
260
261
262
263
{
	kmem_cache_destroy(zswap_entry_cache);
}

static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
{
	struct zswap_entry *entry;
	entry = kmem_cache_alloc(zswap_entry_cache, gfp);
	if (!entry)
		return NULL;
	entry->refcount = 1;
264
	RB_CLEAR_NODE(&entry->rbnode);
Seth Jennings's avatar
Seth Jennings committed
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
	return entry;
}

static void zswap_entry_cache_free(struct zswap_entry *entry)
{
	kmem_cache_free(zswap_entry_cache, entry);
}

/*********************************
* rbtree functions
**********************************/
static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
{
	struct rb_node *node = root->rb_node;
	struct zswap_entry *entry;

	while (node) {
		entry = rb_entry(node, struct zswap_entry, rbnode);
		if (entry->offset > offset)
			node = node->rb_left;
		else if (entry->offset < offset)
			node = node->rb_right;
		else
			return entry;
	}
	return NULL;
}

/*
 * In the case that a entry with the same offset is found, a pointer to
 * the existing entry is stored in dupentry and the function returns -EEXIST
 */
static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
			struct zswap_entry **dupentry)
{
	struct rb_node **link = &root->rb_node, *parent = NULL;
	struct zswap_entry *myentry;

	while (*link) {
		parent = *link;
		myentry = rb_entry(parent, struct zswap_entry, rbnode);
		if (myentry->offset > entry->offset)
			link = &(*link)->rb_left;
		else if (myentry->offset < entry->offset)
			link = &(*link)->rb_right;
		else {
			*dupentry = myentry;
			return -EEXIST;
		}
	}
	rb_link_node(&entry->rbnode, parent, link);
	rb_insert_color(&entry->rbnode, root);
	return 0;
}

320
321
322
323
324
325
326
327
328
static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
{
	if (!RB_EMPTY_NODE(&entry->rbnode)) {
		rb_erase(&entry->rbnode, root);
		RB_CLEAR_NODE(&entry->rbnode);
	}
}

/*
329
 * Carries out the common pattern of freeing and entry's zpool allocation,
330
331
 * freeing the entry itself, and decrementing the number of stored pages.
 */
332
static void zswap_free_entry(struct zswap_entry *entry)
333
{
334
335
336
337
338
339
	if (!entry->length)
		atomic_dec(&zswap_same_filled_pages);
	else {
		zpool_free(entry->pool->zpool, entry->handle);
		zswap_pool_put(entry->pool);
	}
340
341
	zswap_entry_cache_free(entry);
	atomic_dec(&zswap_stored_pages);
Dan Streetman's avatar
Dan Streetman committed
342
	zswap_update_total_size();
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
}

/* caller must hold the tree lock */
static void zswap_entry_get(struct zswap_entry *entry)
{
	entry->refcount++;
}

/* caller must hold the tree lock
* remove from the tree and free it, if nobody reference the entry
*/
static void zswap_entry_put(struct zswap_tree *tree,
			struct zswap_entry *entry)
{
	int refcount = --entry->refcount;

	BUG_ON(refcount < 0);
	if (refcount == 0) {
		zswap_rb_erase(&tree->rbroot, entry);
362
		zswap_free_entry(entry);
363
364
365
366
367
368
369
	}
}

/* caller must hold the tree lock */
static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
				pgoff_t offset)
{
370
	struct zswap_entry *entry;
371
372
373
374
375
376
377
378

	entry = zswap_rb_search(root, offset);
	if (entry)
		zswap_entry_get(entry);

	return entry;
}

Seth Jennings's avatar
Seth Jennings committed
379
380
381
382
383
/*********************************
* per-cpu code
**********************************/
static DEFINE_PER_CPU(u8 *, zswap_dstmem);

384
static int zswap_dstmem_prepare(unsigned int cpu)
Seth Jennings's avatar
Seth Jennings committed
385
386
387
{
	u8 *dst;

388
	dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
389
	if (!dst)
390
		return -ENOMEM;
391

392
393
	per_cpu(zswap_dstmem, cpu) = dst;
	return 0;
Seth Jennings's avatar
Seth Jennings committed
394
395
}

396
static int zswap_dstmem_dead(unsigned int cpu)
Seth Jennings's avatar
Seth Jennings committed
397
{
398
	u8 *dst;
Seth Jennings's avatar
Seth Jennings committed
399

400
401
402
	dst = per_cpu(zswap_dstmem, cpu);
	kfree(dst);
	per_cpu(zswap_dstmem, cpu) = NULL;
Dan Streetman's avatar
Dan Streetman committed
403
404
405
406

	return 0;
}

407
static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
Dan Streetman's avatar
Dan Streetman committed
408
{
409
	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
Dan Streetman's avatar
Dan Streetman committed
410
411
	struct crypto_comp *tfm;

412
413
	if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
		return 0;
Dan Streetman's avatar
Dan Streetman committed
414

415
416
417
418
419
420
421
	tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
	if (IS_ERR_OR_NULL(tfm)) {
		pr_err("could not alloc crypto comp %s : %ld\n",
		       pool->tfm_name, PTR_ERR(tfm));
		return -ENOMEM;
	}
	*per_cpu_ptr(pool->tfm, cpu) = tfm;
Seth Jennings's avatar
Seth Jennings committed
422
423
424
	return 0;
}

425
static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
Dan Streetman's avatar
Dan Streetman committed
426
{
427
428
	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
	struct crypto_comp *tfm;
Dan Streetman's avatar
Dan Streetman committed
429

430
431
432
433
434
	tfm = *per_cpu_ptr(pool->tfm, cpu);
	if (!IS_ERR_OR_NULL(tfm))
		crypto_free_comp(tfm);
	*per_cpu_ptr(pool->tfm, cpu) = NULL;
	return 0;
Dan Streetman's avatar
Dan Streetman committed
435
436
}

Seth Jennings's avatar
Seth Jennings committed
437
/*********************************
Dan Streetman's avatar
Dan Streetman committed
438
* pool functions
Seth Jennings's avatar
Seth Jennings committed
439
**********************************/
Dan Streetman's avatar
Dan Streetman committed
440
441

static struct zswap_pool *__zswap_pool_current(void)
Seth Jennings's avatar
Seth Jennings committed
442
{
Dan Streetman's avatar
Dan Streetman committed
443
444
445
	struct zswap_pool *pool;

	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
446
447
	WARN_ONCE(!pool && zswap_has_pool,
		  "%s: no page storage pool!\n", __func__);
Dan Streetman's avatar
Dan Streetman committed
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465

	return pool;
}

static struct zswap_pool *zswap_pool_current(void)
{
	assert_spin_locked(&zswap_pools_lock);

	return __zswap_pool_current();
}

static struct zswap_pool *zswap_pool_current_get(void)
{
	struct zswap_pool *pool;

	rcu_read_lock();

	pool = __zswap_pool_current();
466
	if (!zswap_pool_get(pool))
Dan Streetman's avatar
Dan Streetman committed
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
		pool = NULL;

	rcu_read_unlock();

	return pool;
}

static struct zswap_pool *zswap_pool_last_get(void)
{
	struct zswap_pool *pool, *last = NULL;

	rcu_read_lock();

	list_for_each_entry_rcu(pool, &zswap_pools, list)
		last = pool;
482
483
484
	WARN_ONCE(!last && zswap_has_pool,
		  "%s: no page storage pool!\n", __func__);
	if (!zswap_pool_get(last))
Dan Streetman's avatar
Dan Streetman committed
485
486
487
488
489
490
491
		last = NULL;

	rcu_read_unlock();

	return last;
}

492
/* type and compressor must be null-terminated */
Dan Streetman's avatar
Dan Streetman committed
493
494
495
496
497
498
499
static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
{
	struct zswap_pool *pool;

	assert_spin_locked(&zswap_pools_lock);

	list_for_each_entry_rcu(pool, &zswap_pools, list) {
500
		if (strcmp(pool->tfm_name, compressor))
Dan Streetman's avatar
Dan Streetman committed
501
			continue;
502
		if (strcmp(zpool_get_type(pool->zpool), type))
Dan Streetman's avatar
Dan Streetman committed
503
504
505
506
507
508
509
510
511
512
513
514
515
			continue;
		/* if we can't get it, it's about to be destroyed */
		if (!zswap_pool_get(pool))
			continue;
		return pool;
	}

	return NULL;
}

static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
{
	struct zswap_pool *pool;
516
	char name[38]; /* 'zswap' + 32 char (max) num + \0 */
517
	gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
518
	int ret;
Dan Streetman's avatar
Dan Streetman committed
519

520
521
522
523
524
525
526
527
528
529
530
	if (!zswap_has_pool) {
		/* if either are unset, pool initialization failed, and we
		 * need both params to be set correctly before trying to
		 * create a pool.
		 */
		if (!strcmp(type, ZSWAP_PARAM_UNSET))
			return NULL;
		if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
			return NULL;
	}

Dan Streetman's avatar
Dan Streetman committed
531
	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
532
	if (!pool)
Dan Streetman's avatar
Dan Streetman committed
533
534
		return NULL;

535
536
537
538
	/* unique name for each pool specifically required by zsmalloc */
	snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));

	pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
Dan Streetman's avatar
Dan Streetman committed
539
540
541
542
543
544
545
546
547
548
549
550
551
	if (!pool->zpool) {
		pr_err("%s zpool not available\n", type);
		goto error;
	}
	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));

	strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
	pool->tfm = alloc_percpu(struct crypto_comp *);
	if (!pool->tfm) {
		pr_err("percpu alloc failed\n");
		goto error;
	}

552
553
554
	ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
				       &pool->node);
	if (ret)
Dan Streetman's avatar
Dan Streetman committed
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
		goto error;
	pr_debug("using %s compressor\n", pool->tfm_name);

	/* being the current pool takes 1 ref; this func expects the
	 * caller to always add the new pool as the current pool
	 */
	kref_init(&pool->kref);
	INIT_LIST_HEAD(&pool->list);

	zswap_pool_debug("created", pool);

	return pool;

error:
	free_percpu(pool->tfm);
	if (pool->zpool)
		zpool_destroy_pool(pool->zpool);
	kfree(pool);
	return NULL;
}

576
static __init struct zswap_pool *__zswap_pool_create_fallback(void)
Dan Streetman's avatar
Dan Streetman committed
577
{
578
579
580
581
	bool has_comp, has_zpool;

	has_comp = crypto_has_comp(zswap_compressor, 0, 0);
	if (!has_comp && strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) {
Dan Streetman's avatar
Dan Streetman committed
582
583
		pr_err("compressor %s not available, using default %s\n",
		       zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
584
585
		param_free_charp(&zswap_compressor);
		zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
586
		has_comp = crypto_has_comp(zswap_compressor, 0, 0);
Dan Streetman's avatar
Dan Streetman committed
587
	}
588
589
590
591
592
593
594
595
596
	if (!has_comp) {
		pr_err("default compressor %s not available\n",
		       zswap_compressor);
		param_free_charp(&zswap_compressor);
		zswap_compressor = ZSWAP_PARAM_UNSET;
	}

	has_zpool = zpool_has_pool(zswap_zpool_type);
	if (!has_zpool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
Dan Streetman's avatar
Dan Streetman committed
597
598
		pr_err("zpool %s not available, using default %s\n",
		       zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
599
600
		param_free_charp(&zswap_zpool_type);
		zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
601
		has_zpool = zpool_has_pool(zswap_zpool_type);
Dan Streetman's avatar
Dan Streetman committed
602
	}
603
604
605
606
607
608
609
610
611
	if (!has_zpool) {
		pr_err("default zpool %s not available\n",
		       zswap_zpool_type);
		param_free_charp(&zswap_zpool_type);
		zswap_zpool_type = ZSWAP_PARAM_UNSET;
	}

	if (!has_comp || !has_zpool)
		return NULL;
Dan Streetman's avatar
Dan Streetman committed
612
613
614
615
616
617
618
619

	return zswap_pool_create(zswap_zpool_type, zswap_compressor);
}

static void zswap_pool_destroy(struct zswap_pool *pool)
{
	zswap_pool_debug("destroying", pool);

620
	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
Dan Streetman's avatar
Dan Streetman committed
621
622
623
624
625
626
627
	free_percpu(pool->tfm);
	zpool_destroy_pool(pool->zpool);
	kfree(pool);
}

static int __must_check zswap_pool_get(struct zswap_pool *pool)
{
628
629
630
	if (!pool)
		return 0;

Dan Streetman's avatar
Dan Streetman committed
631
632
633
	return kref_get_unless_zero(&pool->kref);
}

634
static void __zswap_pool_release(struct work_struct *work)
Dan Streetman's avatar
Dan Streetman committed
635
{
636
637
638
	struct zswap_pool *pool = container_of(work, typeof(*pool), work);

	synchronize_rcu();
Dan Streetman's avatar
Dan Streetman committed
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657

	/* nobody should have been able to get a kref... */
	WARN_ON(kref_get_unless_zero(&pool->kref));

	/* pool is now off zswap_pools list and has no references. */
	zswap_pool_destroy(pool);
}

static void __zswap_pool_empty(struct kref *kref)
{
	struct zswap_pool *pool;

	pool = container_of(kref, typeof(*pool), kref);

	spin_lock(&zswap_pools_lock);

	WARN_ON(pool == zswap_pool_current());

	list_del_rcu(&pool->list);
658
659
660

	INIT_WORK(&pool->work, __zswap_pool_release);
	schedule_work(&pool->work);
Dan Streetman's avatar
Dan Streetman committed
661
662
663
664
665
666
667

	spin_unlock(&zswap_pools_lock);
}

static void zswap_pool_put(struct zswap_pool *pool)
{
	kref_put(&pool->kref, __zswap_pool_empty);
Seth Jennings's avatar
Seth Jennings committed
668
669
}

670
671
672
673
/*********************************
* param callbacks
**********************************/

674
/* val must be a null-terminated string */
675
676
677
678
static int __zswap_param_set(const char *val, const struct kernel_param *kp,
			     char *type, char *compressor)
{
	struct zswap_pool *pool, *put_pool = NULL;
679
	char *s = strstrip((char *)val);
680
681
	int ret;

682
683
684
685
686
	if (zswap_init_failed) {
		pr_err("can't set param, initialization failed\n");
		return -ENODEV;
	}

687
	/* no change required */
688
	if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
689
		return 0;
690
691
692
693
694

	/* if this is load-time (pre-init) param setting,
	 * don't create a pool; that's done during init.
	 */
	if (!zswap_init_started)
695
		return param_set_charp(s, kp);
696
697

	if (!type) {
698
699
		if (!zpool_has_pool(s)) {
			pr_err("zpool %s not available\n", s);
700
701
			return -ENOENT;
		}
702
		type = s;
703
	} else if (!compressor) {
704
705
		if (!crypto_has_comp(s, 0, 0)) {
			pr_err("compressor %s not available\n", s);
706
707
			return -ENOENT;
		}
708
709
710
711
		compressor = s;
	} else {
		WARN_ON(1);
		return -EINVAL;
712
713
714
715
716
717
718
	}

	spin_lock(&zswap_pools_lock);

	pool = zswap_pool_find_get(type, compressor);
	if (pool) {
		zswap_pool_debug("using existing", pool);
719
		WARN_ON(pool == zswap_pool_current());
720
721
722
		list_del_rcu(&pool->list);
	}

723
724
725
726
727
	spin_unlock(&zswap_pools_lock);

	if (!pool)
		pool = zswap_pool_create(type, compressor);

728
	if (pool)
729
		ret = param_set_charp(s, kp);
730
731
732
	else
		ret = -EINVAL;

733
734
	spin_lock(&zswap_pools_lock);

735
736
737
	if (!ret) {
		put_pool = zswap_pool_current();
		list_add_rcu(&pool->list, &zswap_pools);
738
		zswap_has_pool = true;
739
740
741
742
743
744
745
	} else if (pool) {
		/* add the possibly pre-existing pool to the end of the pools
		 * list; if it's new (and empty) then it'll be removed and
		 * destroyed by the put after we drop the lock
		 */
		list_add_tail_rcu(&pool->list, &zswap_pools);
		put_pool = pool;
746
747
748
749
750
	}

	spin_unlock(&zswap_pools_lock);

	if (!zswap_has_pool && !pool) {
751
752
753
754
755
756
757
758
		/* if initial pool creation failed, and this pool creation also
		 * failed, maybe both compressor and zpool params were bad.
		 * Allow changing this param, so pool creation will succeed
		 * when the other param is changed. We already verified this
		 * param is ok in the zpool_has_pool() or crypto_has_comp()
		 * checks above.
		 */
		ret = param_set_charp(s, kp);
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
	}

	/* drop the ref from either the old current pool,
	 * or the new pool we failed to add
	 */
	if (put_pool)
		zswap_pool_put(put_pool);

	return ret;
}

static int zswap_compressor_param_set(const char *val,
				      const struct kernel_param *kp)
{
	return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
}

static int zswap_zpool_param_set(const char *val,
				 const struct kernel_param *kp)
{
	return __zswap_param_set(val, kp, NULL, zswap_compressor);
}

782
783
784
785
786
787
788
static int zswap_enabled_param_set(const char *val,
				   const struct kernel_param *kp)
{
	if (zswap_init_failed) {
		pr_err("can't enable, initialization failed\n");
		return -ENODEV;
	}
789
790
791
792
	if (!zswap_has_pool && zswap_init_started) {
		pr_err("can't enable, no pool configured\n");
		return -ENODEV;
	}
793
794
795
796

	return param_set_bool(val, kp);
}

Seth Jennings's avatar
Seth Jennings committed
797
798
799
800
801
802
803
/*********************************
* writeback code
**********************************/
/* return enum for zswap_get_swap_cache_page */
enum zswap_get_swap_ret {
	ZSWAP_SWAPCACHE_NEW,
	ZSWAP_SWAPCACHE_EXIST,
804
	ZSWAP_SWAPCACHE_FAIL,
Seth Jennings's avatar
Seth Jennings committed
805
806
807
808
809
810
811
812
813
814
815
816
817
};

/*
 * zswap_get_swap_cache_page
 *
 * This is an adaption of read_swap_cache_async()
 *
 * This function tries to find a page with the given swap entry
 * in the swapper_space address space (the swap cache).  If the page
 * is found, it is returned in retpage.  Otherwise, a page is allocated,
 * added to the swap cache, and returned in retpage.
 *
 * If success, the swap cache page is returned in retpage
818
819
820
821
 * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
 * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
 *     the new page is added to swapcache and locked
 * Returns ZSWAP_SWAPCACHE_FAIL on error
Seth Jennings's avatar
Seth Jennings committed
822
823
824
825
 */
static int zswap_get_swap_cache_page(swp_entry_t entry,
				struct page **retpage)
{
826
	bool page_was_allocated;
Seth Jennings's avatar
Seth Jennings committed
827

828
829
830
831
832
	*retpage = __read_swap_cache_async(entry, GFP_KERNEL,
			NULL, 0, &page_was_allocated);
	if (page_was_allocated)
		return ZSWAP_SWAPCACHE_NEW;
	if (!*retpage)
833
		return ZSWAP_SWAPCACHE_FAIL;
Seth Jennings's avatar
Seth Jennings committed
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
	return ZSWAP_SWAPCACHE_EXIST;
}

/*
 * Attempts to free an entry by adding a page to the swap cache,
 * decompressing the entry data into the page, and issuing a
 * bio write to write the page back to the swap device.
 *
 * This can be thought of as a "resumed writeback" of the page
 * to the swap device.  We are basically resuming the same swap
 * writeback path that was intercepted with the frontswap_store()
 * in the first place.  After the page has been decompressed into
 * the swap cache, the compressed version stored by zswap can be
 * freed.
 */
849
static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
Seth Jennings's avatar
Seth Jennings committed
850
851
852
853
854
855
856
{
	struct zswap_header *zhdr;
	swp_entry_t swpentry;
	struct zswap_tree *tree;
	pgoff_t offset;
	struct zswap_entry *entry;
	struct page *page;
Dan Streetman's avatar
Dan Streetman committed
857
	struct crypto_comp *tfm;
Seth Jennings's avatar
Seth Jennings committed
858
859
	u8 *src, *dst;
	unsigned int dlen;
860
	int ret;
Seth Jennings's avatar
Seth Jennings committed
861
862
863
864
865
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_NONE,
	};

	/* extract swpentry from data */
866
	zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
Seth Jennings's avatar
Seth Jennings committed
867
	swpentry = zhdr->swpentry; /* here */
868
	zpool_unmap_handle(pool, handle);
Seth Jennings's avatar
Seth Jennings committed
869
870
871
872
873
	tree = zswap_trees[swp_type(swpentry)];
	offset = swp_offset(swpentry);

	/* find and ref zswap entry */
	spin_lock(&tree->lock);
874
	entry = zswap_entry_find_get(&tree->rbroot, offset);
Seth Jennings's avatar
Seth Jennings committed
875
876
877
878
879
880
881
882
883
884
	if (!entry) {
		/* entry was invalidated */
		spin_unlock(&tree->lock);
		return 0;
	}
	spin_unlock(&tree->lock);
	BUG_ON(offset != entry->offset);

	/* try to allocate swap cache page */
	switch (zswap_get_swap_cache_page(swpentry, &page)) {
885
	case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
Seth Jennings's avatar
Seth Jennings committed
886
887
888
		ret = -ENOMEM;
		goto fail;

889
	case ZSWAP_SWAPCACHE_EXIST:
Seth Jennings's avatar
Seth Jennings committed
890
		/* page is already in the swap cache, ignore for now */
891
		put_page(page);
Seth Jennings's avatar
Seth Jennings committed
892
893
894
895
896
897
		ret = -EEXIST;
		goto fail;

	case ZSWAP_SWAPCACHE_NEW: /* page is locked */
		/* decompress */
		dlen = PAGE_SIZE;
Dan Streetman's avatar
Dan Streetman committed
898
		src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
899
				ZPOOL_MM_RO) + sizeof(struct zswap_header);
Seth Jennings's avatar
Seth Jennings committed
900
		dst = kmap_atomic(page);
Dan Streetman's avatar
Dan Streetman committed
901
902
903
904
		tfm = *get_cpu_ptr(entry->pool->tfm);
		ret = crypto_comp_decompress(tfm, src, entry->length,
					     dst, &dlen);
		put_cpu_ptr(entry->pool->tfm);
Seth Jennings's avatar
Seth Jennings committed
905
		kunmap_atomic(dst);
Dan Streetman's avatar
Dan Streetman committed
906
		zpool_unmap_handle(entry->pool->zpool, entry->handle);
Seth Jennings's avatar
Seth Jennings committed
907
908
909
910
911
912
913
		BUG_ON(ret);
		BUG_ON(dlen != PAGE_SIZE);

		/* page is up to date */
		SetPageUptodate(page);
	}

914
915
916
	/* move it to the tail of the inactive list after end_writeback */
	SetPageReclaim(page);

Seth Jennings's avatar
Seth Jennings committed
917
918
	/* start writeback */
	__swap_writepage(page, &wbc, end_swap_bio_write);
919
	put_page(page);
Seth Jennings's avatar
Seth Jennings committed
920
921
922
923
	zswap_written_back_pages++;

	spin_lock(&tree->lock);
	/* drop local reference */
924
	zswap_entry_put(tree, entry);
Seth Jennings's avatar
Seth Jennings committed
925
926

	/*
927
928
929
930
931
932
933
934
	* There are two possible situations for entry here:
	* (1) refcount is 1(normal case),  entry is valid and on the tree
	* (2) refcount is 0, entry is freed and not on the tree
	*     because invalidate happened during writeback
	*  search the tree and free the entry if find entry
	*/
	if (entry == zswap_rb_search(&tree->rbroot, offset))
		zswap_entry_put(tree, entry);
Seth Jennings's avatar
Seth Jennings committed
935
936
	spin_unlock(&tree->lock);

937
938
939
940
941
942
943
944
945
	goto end;

	/*
	* if we get here due to ZSWAP_SWAPCACHE_EXIST
	* a load may happening concurrently
	* it is safe and okay to not free the entry
	* if we free the entry in the following put
	* it it either okay to return !0
	*/
Seth Jennings's avatar
Seth Jennings committed
946
947
fail:
	spin_lock(&tree->lock);
948
	zswap_entry_put(tree, entry);
Seth Jennings's avatar
Seth Jennings committed
949
	spin_unlock(&tree->lock);
950
951

end:
Seth Jennings's avatar
Seth Jennings committed
952
953
954
	return ret;
}

Dan Streetman's avatar
Dan Streetman committed
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
static int zswap_shrink(void)
{
	struct zswap_pool *pool;
	int ret;

	pool = zswap_pool_last_get();
	if (!pool)
		return -ENOENT;

	ret = zpool_shrink(pool->zpool, 1, NULL);

	zswap_pool_put(pool);

	return ret;
}

971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
static int zswap_is_page_same_filled(void *ptr, unsigned long *value)
{
	unsigned int pos;
	unsigned long *page;

	page = (unsigned long *)ptr;
	for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) {
		if (page[pos] != page[0])
			return 0;
	}
	*value = page[0];
	return 1;
}

static void zswap_fill_page(void *ptr, unsigned long value)
{
	unsigned long *page;

	page = (unsigned long *)ptr;
	memset_l(page, value, PAGE_SIZE / sizeof(unsigned long));
}

Seth Jennings's avatar
Seth Jennings committed
993
994
995
996
997
998
999
1000
1001
/*********************************
* frontswap hooks
**********************************/
/* attempts to compress and store an single page */
static int zswap_frontswap_store(unsigned type, pgoff_t offset,
				struct page *page)
{
	struct zswap_tree *tree = zswap_trees[type];
	struct zswap_entry *entry, *dupentry;
Dan Streetman's avatar
Dan Streetman committed
1002
	struct crypto_comp *tfm;
Seth Jennings's avatar
Seth Jennings committed
1003
	int ret;
1004
	unsigned int hlen, dlen = PAGE_SIZE;
1005
	unsigned long handle, value;
Seth Jennings's avatar
Seth Jennings committed
1006
1007
	char *buf;
	u8 *src, *dst;
1008
	struct zswap_header zhdr = { .swpentry = swp_entry(type, offset) };
Seth Jennings's avatar
Seth Jennings committed
1009

1010
1011
1012
1013
1014
1015
	/* THP isn't supported */
	if (PageTransHuge(page)) {
		ret = -EINVAL;
		goto reject;
	}

Dan Streetman's avatar
Dan Streetman committed
1016
	if (!zswap_enabled || !tree) {
Seth Jennings's avatar
Seth Jennings committed
1017
1018
1019
1020
1021
1022
1023
		ret = -ENODEV;
		goto reject;
	}

	/* reclaim space if needed */
	if (zswap_is_full()) {
		zswap_pool_limit_hit++;
Dan Streetman's avatar
Dan Streetman committed
1024
		if (zswap_shrink()) {
Seth Jennings's avatar
Seth Jennings committed
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
			zswap_reject_reclaim_fail++;
			ret = -ENOMEM;
			goto reject;
		}
	}

	/* allocate entry */
	entry = zswap_entry_cache_alloc(GFP_KERNEL);
	if (!entry) {
		zswap_reject_kmemcache_fail++;
		ret = -ENOMEM;
		goto reject;
	}

1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
	if (zswap_same_filled_pages_enabled) {
		src = kmap_atomic(page);
		if (zswap_is_page_same_filled(src, &value)) {
			kunmap_atomic(src);
			entry->offset = offset;
			entry->length = 0;
			entry->value = value;
			atomic_inc(&zswap_same_filled_pages);
			goto insert_entry;
		}
		kunmap_atomic(src);
	}

Dan Streetman's avatar
Dan Streetman committed
1052
1053
1054
1055
1056
1057
1058
	/* if entry is successfully added, it keeps the reference */
	entry->pool = zswap_pool_current_get();
	if (!entry->pool) {
		ret = -EINVAL;
		goto freepage;
	}

Seth Jennings's avatar
Seth Jennings committed
1059
1060
	/* compress */
	dst = get_cpu_var(zswap_dstmem);
Dan Streetman's avatar
Dan Streetman committed
1061
	tfm = *get_cpu_ptr(entry->pool->tfm);
Seth Jennings's avatar
Seth Jennings committed
1062
	src = kmap_atomic(page);
Dan Streetman's avatar
Dan Streetman committed
1063
	ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
Seth Jennings's avatar
Seth Jennings committed
1064
	kunmap_atomic(src);
Dan Streetman's avatar
Dan Streetman committed
1065
	put_cpu_ptr(entry->pool->tfm);
Seth Jennings's avatar
Seth Jennings committed
1066
1067
	if (ret) {
		ret = -EINVAL;
Dan Streetman's avatar
Dan Streetman committed
1068
		goto put_dstmem;
Seth Jennings's avatar
Seth Jennings committed
1069
1070
1071
	}

	/* store */
1072
1073
	hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0;
	ret = zpool_malloc(entry->pool->zpool, hlen + dlen,
1074
1075
			   __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM,
			   &handle);
Seth Jennings's avatar
Seth Jennings committed
1076
1077
	if (ret == -ENOSPC) {
		zswap_reject_compress_poor++;
Dan Streetman's avatar
Dan Streetman committed
1078
		goto put_dstmem;
Seth Jennings's avatar
Seth Jennings committed
1079
1080
1081
	}
	if (ret) {
		zswap_reject_alloc_fail++;
Dan Streetman's avatar
Dan Streetman committed
1082
		goto put_dstmem;
Seth Jennings's avatar
Seth Jennings committed
1083
	}
1084
1085
1086
	buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
	memcpy(buf, &zhdr, hlen);
	memcpy(buf + hlen, dst, dlen);
Dan Streetman's avatar
Dan Streetman committed
1087
	zpool_unmap_handle(entry->pool->zpool, handle);
Seth Jennings's avatar
Seth Jennings committed
1088
1089
1090
1091
1092
1093
1094
	put_cpu_var(zswap_dstmem);

	/* populate entry */
	entry->offset = offset;
	entry->handle = handle;
	entry->length = dlen;

1095
insert_entry:
Seth Jennings's avatar
Seth Jennings committed
1096
1097
1098
1099
1100
1101
1102
	/* map */
	spin_lock(&tree->lock);
	do {
		ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
		if (ret == -EEXIST) {
			zswap_duplicate_entry++;
			/* remove from rbtree */
1103
1104
			zswap_rb_erase(&tree->rbroot, dupentry);
			zswap_entry_put(tree, dupentry);
Seth Jennings's avatar
Seth Jennings committed
1105
1106
1107
1108
1109
1110
		}
	} while (ret == -EEXIST);
	spin_unlock(&tree->lock);

	/* update stats */
	atomic_inc(&zswap_stored_pages);
Dan Streetman's avatar
Dan Streetman committed
1111
	zswap_update_total_size();
Seth Jennings's avatar
Seth Jennings committed
1112
1113
1114

	return 0;

Dan Streetman's avatar
Dan Streetman committed
1115
put_dstmem:
Seth Jennings's avatar
Seth Jennings committed
1116
	put_cpu_var(zswap_dstmem);
Dan Streetman's avatar
Dan Streetman committed
1117
1118
	zswap_pool_put(entry->pool);
freepage:
Seth Jennings's avatar
Seth Jennings committed
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
	zswap_entry_cache_free(entry);
reject:
	return ret;
}

/*
 * returns 0 if the page was successfully decompressed
 * return -1 on entry not found or error
*/
static int zswap_frontswap_load(unsigned type, pgoff_t offset,
				struct page *page)
{
	struct zswap_tree *tree = zswap_trees[type];
	struct zswap_entry *entry;
Dan Streetman's avatar
Dan Streetman committed
1133
	struct crypto_comp *tfm;
Seth Jennings's avatar
Seth Jennings committed
1134
1135
	u8 *src, *dst;
	unsigned int dlen;
1136
	int ret;
Seth Jennings's avatar
Seth Jennings committed
1137
1138
1139

	/* find */
	spin_lock(&tree->lock);
1140
	entry = zswap_entry_find_get(&tree->rbroot, offset);
Seth Jennings's avatar
Seth Jennings committed
1141
1142
1143
1144
1145
1146
1147
	if (!entry) {
		/* entry was written back */
		spin_unlock(&tree->lock);
		return -1;
	}
	spin_unlock(&tree->lock);

1148
1149
1150
1151
1152
1153
1154
	if (!entry->length) {
		dst = kmap_atomic(page);
		zswap_fill_page(dst, entry->value);
		kunmap_atomic(dst);
		goto freeentry;
	}

Seth Jennings's avatar
Seth Jennings committed
1155
1156
	/* decompress */
	dlen = PAGE_SIZE;
1157
1158
1159
	src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO);
	if (zpool_evictable(entry->pool->zpool))
		src += sizeof(struct zswap_header);
Seth Jennings's avatar
Seth Jennings committed
1160
	dst = kmap_atomic(page);
Dan Streetman's avatar
Dan Streetman committed
1161
1162
1163
	tfm = *get_cpu_ptr(entry->pool->tfm);
	ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
	put_cpu_ptr(entry->pool->tfm);
Seth Jennings's avatar
Seth Jennings committed
1164
	kunmap_atomic(dst);
Dan Streetman's avatar
Dan Streetman committed
1165
	zpool_unmap_handle(entry->pool->zpool, entry->handle);
Seth Jennings's avatar
Seth Jennings committed
1166
1167
	BUG_ON(ret);

1168
freeentry:
Seth Jennings's avatar
Seth Jennings committed
1169
	spin_lock(&tree->lock);
1170
	zswap_entry_put(tree, entry);
Seth Jennings's avatar
Seth Jennings committed
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
	spin_unlock(&tree->lock);

	return 0;
}

/* frees an entry in zswap */
static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
{
	struct zswap_tree *tree = zswap_trees[type];
	struct zswap_entry *entry;

	/* find */
	spin_lock(&tree->lock);
	entry = zswap_rb_search(&tree->rbroot, offset);
	if (!entry) {
		/* entry was written back */
		spin_unlock(&tree->lock);
		return;
	}

	/* remove from rbtree */
1192
	zswap_rb_erase(&tree->rbroot, entry);
Seth Jennings's avatar
Seth Jennings committed
1193
1194

	/* drop the initial reference from entry creation */
1195
	zswap_entry_put(tree, entry);
Seth Jennings's avatar
Seth Jennings committed
1196
1197
1198
1199
1200
1201
1202
1203

	spin_unlock(&tree->lock);
}

/* frees all zswap entries for the given swap type */
static void zswap_frontswap_invalidate_area(unsigned type)
{
	struct zswap_tree *tree = zswap_trees[type];
1204
	struct zswap_entry *entry, *n;
Seth Jennings's avatar
Seth Jennings committed
1205
1206
1207
1208
1209
1210

	if (!tree)
		return;

	/* walk the tree and free everything */
	spin_lock(&tree->lock);
1211
	rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
1212
		zswap_free_entry(entry);
Seth Jennings's avatar
Seth Jennings committed
1213
1214
	tree->rbroot = RB_ROOT;
	spin_unlock(&tree->lock);
1215
1216
	kfree(tree);
	zswap_trees[type] = NULL;
Seth Jennings's avatar
Seth Jennings committed
1217
1218
1219
1220
1221
1222
}

static void zswap_frontswap_init(unsigned type)
{
	struct zswap_tree *tree;

1223
	tree = kzalloc(sizeof(*tree), GFP_KERNEL);
1224
1225
1226
1227
1228
	if (!tree) {
		pr_err("alloc failed, zswap disabled for swap type %d\n", type);
		return;
	}

Seth Jennings's avatar
Seth Jennings committed
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
	tree->rbroot = RB_ROOT;
	spin_lock_init(&tree->lock);
	zswap_trees[type] = tree;
}

static struct frontswap_ops zswap_frontswap_ops = {
	.store = zswap_frontswap_store,
	.load = zswap_frontswap_load,
	.invalidate_page = zswap_frontswap_invalidate_page,
	.invalidate_area = zswap_frontswap_invalidate_area,
	.init = zswap_frontswap_init
};

/*********************************
* debugfs functions
**********************************/
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>

static struct dentry *zswap_debugfs_root;

static int __init zswap_debugfs_init(void)
{
	if (!debugfs_initialized())
		return -ENODEV;

	zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
	if (!zswap_debugfs_root)
		return -ENOMEM;

1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
	debugfs_create_u64("pool_limit_hit", 0444,
			   zswap_debugfs_root, &zswap_pool_limit_hit);
	debugfs_create_u64("reject_reclaim_fail", 0444,
			   zswap_debugfs_root, &zswap_reject_reclaim_fail);
	debugfs_create_u64("reject_alloc_fail", 0444,
			   zswap_debugfs_root, &zswap_reject_alloc_fail);
	debugfs_create_u64("reject_kmemcache_fail", 0444,
			   zswap_debugfs_root, &zswap_reject_kmemcache_fail);
	debugfs_create_u64("reject_compress_poor", 0444,
			   zswap_debugfs_root, &zswap_reject_compress_poor);
	debugfs_create_u64("written_back_pages", 0444,
			   zswap_debugfs_root, &zswap_written_back_pages);
	debugfs_create_u64("duplicate_entry", 0444,
			   zswap_debugfs_root, &zswap_duplicate_entry);
	debugfs_create_u64("pool_total_size", 0444,
			   zswap_debugfs_root, &zswap_pool_total_size);
	debugfs_create_atomic_t("stored_pages", 0444,
				zswap_debugfs_root, &zswap_stored_pages);
1277
	debugfs_create_atomic_t("same_filled_pages", 0444,
1278
				zswap_debugfs_root, &zswap_same_filled_pages);
Seth Jennings's avatar
Seth Jennings committed
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300

	return 0;
}

static void __exit zswap_debugfs_exit(void)
{
	debugfs_remove_recursive(zswap_debugfs_root);
}
#else
static int __init zswap_debugfs_init(void)
{
	return 0;
}

static void __exit zswap_debugfs_exit(void) { }
#endif

/*********************************
* module init and exit
**********************************/
static int __init init_zswap(void)
{
Dan Streetman's avatar
Dan Streetman committed
1301
	struct zswap_pool *pool;
1302
	int ret;
1303

1304
1305
	zswap_init_started = true;

Seth Jennings's avatar
Seth Jennings committed
1306
1307
	if (zswap_entry_cache_create()) {
		pr_err("entry cache creation failed\n");
Dan Streetman's avatar
Dan Streetman committed
1308
		goto cache_fail;
Seth Jennings's avatar
Seth Jennings committed
1309
	}
Dan Streetman's avatar
Dan Streetman committed
1310

1311
1312
1313
	ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",
				zswap_dstmem_prepare, zswap_dstmem_dead);
	if (ret) {
Dan Streetman's avatar
Dan Streetman committed
1314
1315
		pr_err("dstmem alloc failed\n");
		goto dstmem_fail;
Seth Jennings's avatar
Seth Jennings committed
1316
	}
Dan Streetman's avatar
Dan Streetman committed
1317

1318
1319
1320
1321
1322
1323
1324
	ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
				      "mm/zswap_pool:prepare",
				      zswap_cpu_comp_prepare,
				      zswap_cpu_comp_dead);
	if (ret)
		goto hp_fail;

Dan Streetman's avatar
Dan Streetman committed
1325
	pool = __zswap_pool_create_fallback();
1326
1327
1328
1329
1330
1331
	if (pool) {
		pr_info("loaded using pool %s/%s\n", pool->tfm_name,
			zpool_get_type(pool->zpool));
		list_add(&pool->list, &zswap_pools);
		zswap_has_pool = true;
	} else {
Dan Streetman's avatar
Dan Streetman committed
1332
		pr_err("pool creation failed\n");
1333
		zswap_enabled = false;
Seth Jennings's avatar
Seth Jennings committed
1334
	}
1335

Seth Jennings's avatar
Seth Jennings committed
1336
1337
1338
1339
	frontswap_register_ops(&zswap_frontswap_ops);
	if (zswap_debugfs_init())
		pr_warn("debugfs initialization failed\n");
	return 0;
Dan Streetman's avatar
Dan Streetman committed
1340

1341
hp_fail:
1342
	cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);
Dan Streetman's avatar
Dan Streetman committed
1343
dstmem_fail:
1344
	zswap_entry_cache_destroy();
Dan Streetman's avatar
Dan Streetman committed
1345
cache_fail:
1346
1347
1348
	/* if built-in, we aren't unloaded on failure; don't allow use */
	zswap_init_failed = true;
	zswap_enabled = false;
Seth Jennings's avatar
Seth Jennings committed
1349
1350
1351
1352
1353
1354
	return -ENOMEM;
}
/* must be late so crypto has time to come up */
late_initcall(init_zswap);

MODULE_LICENSE("GPL");
1355
MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
Seth Jennings's avatar
Seth Jennings committed
1356
MODULE_DESCRIPTION("Compressed cache for swap pages");