dev.c 51.1 KB
Newer Older
1
2
/*
  FUSE: Filesystem in Userspace
Miklos Szeredi's avatar
Miklos Szeredi committed
3
  Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

  This program can be distributed under the terms of the GNU GPL.
  See the file COPYING.
*/

#include "fuse_i.h"

#include <linux/init.h>
#include <linux/module.h>
#include <linux/poll.h>
#include <linux/uio.h>
#include <linux/miscdevice.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/slab.h>
19
#include <linux/pipe_fs_i.h>
20
21
#include <linux/swap.h>
#include <linux/splice.h>
22
23

MODULE_ALIAS_MISCDEV(FUSE_MINOR);
24
MODULE_ALIAS("devname:fuse");
25

26
static struct kmem_cache *fuse_req_cachep;
27

28
static struct fuse_dev *fuse_get_dev(struct file *file)
29
{
30
31
32
33
	/*
	 * Lockless access is OK, because file->private data is set
	 * once during mount and is valid until the file is released.
	 */
34
	return ACCESS_ONCE(file->private_data);
35
36
}

37
static void fuse_request_init(struct fuse_req *req, struct page **pages,
38
			      struct fuse_page_desc *page_descs,
39
			      unsigned npages)
40
41
{
	memset(req, 0, sizeof(*req));
42
	memset(pages, 0, sizeof(*pages) * npages);
43
	memset(page_descs, 0, sizeof(*page_descs) * npages);
44
	INIT_LIST_HEAD(&req->list);
45
	INIT_LIST_HEAD(&req->intr_entry);
46
47
	init_waitqueue_head(&req->waitq);
	atomic_set(&req->count, 1);
48
	req->pages = pages;
49
	req->page_descs = page_descs;
50
	req->max_pages = npages;
Miklos Szeredi's avatar
Miklos Szeredi committed
51
	__set_bit(FR_PENDING, &req->flags);
52
53
}

54
static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
55
{
56
57
58
	struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
	if (req) {
		struct page **pages;
59
		struct fuse_page_desc *page_descs;
60

61
		if (npages <= FUSE_REQ_INLINE_PAGES) {
62
			pages = req->inline_pages;
63
64
			page_descs = req->inline_page_descs;
		} else {
65
			pages = kmalloc(sizeof(struct page *) * npages, flags);
66
67
68
			page_descs = kmalloc(sizeof(struct fuse_page_desc) *
					     npages, flags);
		}
69

70
71
72
		if (!pages || !page_descs) {
			kfree(pages);
			kfree(page_descs);
73
74
75
76
			kmem_cache_free(fuse_req_cachep, req);
			return NULL;
		}

77
		fuse_request_init(req, pages, page_descs, npages);
78
	}
79
80
	return req;
}
81
82
83
84
85

struct fuse_req *fuse_request_alloc(unsigned npages)
{
	return __fuse_request_alloc(npages, GFP_KERNEL);
}
86
EXPORT_SYMBOL_GPL(fuse_request_alloc);
87

88
struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
Miklos Szeredi's avatar
Miklos Szeredi committed
89
{
90
	return __fuse_request_alloc(npages, GFP_NOFS);
Miklos Szeredi's avatar
Miklos Szeredi committed
91
92
}

93
94
void fuse_request_free(struct fuse_req *req)
{
95
	if (req->pages != req->inline_pages) {
96
		kfree(req->pages);
97
98
		kfree(req->page_descs);
	}
99
100
101
	kmem_cache_free(fuse_req_cachep, req);
}

102
void __fuse_get_request(struct fuse_req *req)
103
104
105
106
107
108
109
110
111
112
113
{
	atomic_inc(&req->count);
}

/* Must be called with > 1 refcount */
static void __fuse_put_request(struct fuse_req *req)
{
	BUG_ON(atomic_read(&req->count) < 2);
	atomic_dec(&req->count);
}

114
115
static void fuse_req_init_context(struct fuse_req *req)
{
116
117
	req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
	req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
118
119
120
	req->in.h.pid = current->pid;
}

121
122
123
124
125
126
127
void fuse_set_initialized(struct fuse_conn *fc)
{
	/* Make sure stores before this are seen on another CPU */
	smp_wmb();
	fc->initialized = 1;
}

128
129
130
131
132
static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
{
	return !fc->initialized || (for_background && fc->blocked);
}

133
134
static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
				       bool for_background)
135
{
136
137
	struct fuse_req *req;
	int err;
138
	atomic_inc(&fc->num_waiting);
139
140
141

	if (fuse_block_alloc(fc, for_background)) {
		err = -EINTR;
142
143
		if (wait_event_killable_exclusive(fc->blocked_waitq,
				!fuse_block_alloc(fc, for_background)))
144
145
			goto out;
	}
146
147
	/* Matches smp_wmb() in fuse_set_initialized() */
	smp_rmb();
148

149
150
151
152
	err = -ENOTCONN;
	if (!fc->connected)
		goto out;

153
154
155
156
	err = -ECONNREFUSED;
	if (fc->conn_error)
		goto out;

157
	req = fuse_request_alloc(npages);
158
	err = -ENOMEM;
159
160
161
	if (!req) {
		if (for_background)
			wake_up(&fc->blocked_waitq);
162
		goto out;
163
	}
164

165
	fuse_req_init_context(req);
Miklos Szeredi's avatar
Miklos Szeredi committed
166
167
168
169
	__set_bit(FR_WAITING, &req->flags);
	if (for_background)
		__set_bit(FR_BACKGROUND, &req->flags);

170
	return req;
171
172
173
174

 out:
	atomic_dec(&fc->num_waiting);
	return ERR_PTR(err);
175
}
176
177
178
179
180

struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
{
	return __fuse_get_req(fc, npages, false);
}
181
EXPORT_SYMBOL_GPL(fuse_get_req);
182

183
184
185
186
187
188
189
struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
					     unsigned npages)
{
	return __fuse_get_req(fc, npages, true);
}
EXPORT_SYMBOL_GPL(fuse_get_req_for_background);

190
191
192
193
194
195
196
197
198
199
200
201
/*
 * Return request in fuse_file->reserved_req.  However that may
 * currently be in use.  If that is the case, wait for it to become
 * available.
 */
static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
					 struct file *file)
{
	struct fuse_req *req = NULL;
	struct fuse_file *ff = file->private_data;

	do {
202
		wait_event(fc->reserved_req_waitq, ff->reserved_req);
203
204
205
206
		spin_lock(&fc->lock);
		if (ff->reserved_req) {
			req = ff->reserved_req;
			ff->reserved_req = NULL;
Al Viro's avatar
Al Viro committed
207
			req->stolen_file = get_file(file);
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
		}
		spin_unlock(&fc->lock);
	} while (!req);

	return req;
}

/*
 * Put stolen request back into fuse_file->reserved_req
 */
static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
{
	struct file *file = req->stolen_file;
	struct fuse_file *ff = file->private_data;

	spin_lock(&fc->lock);
224
	fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
225
226
	BUG_ON(ff->reserved_req);
	ff->reserved_req = req;
227
	wake_up_all(&fc->reserved_req_waitq);
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
	spin_unlock(&fc->lock);
	fput(file);
}

/*
 * Gets a requests for a file operation, always succeeds
 *
 * This is used for sending the FLUSH request, which must get to
 * userspace, due to POSIX locks which may need to be unlocked.
 *
 * If allocation fails due to OOM, use the reserved request in
 * fuse_file.
 *
 * This is very unlikely to deadlock accidentally, since the
 * filesystem should not have it's own file open.  If deadlock is
 * intentional, it can still be broken by "aborting" the filesystem.
 */
245
246
struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
					     struct file *file)
247
248
249
250
{
	struct fuse_req *req;

	atomic_inc(&fc->num_waiting);
251
	wait_event(fc->blocked_waitq, fc->initialized);
252
253
	/* Matches smp_wmb() in fuse_set_initialized() */
	smp_rmb();
254
	req = fuse_request_alloc(0);
255
256
257
258
	if (!req)
		req = get_reserved_req(fc, file);

	fuse_req_init_context(req);
Miklos Szeredi's avatar
Miklos Szeredi committed
259
260
	__set_bit(FR_WAITING, &req->flags);
	__clear_bit(FR_BACKGROUND, &req->flags);
261
262
263
	return req;
}

264
void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
265
266
{
	if (atomic_dec_and_test(&req->count)) {
Miklos Szeredi's avatar
Miklos Szeredi committed
267
		if (test_bit(FR_BACKGROUND, &req->flags)) {
268
269
270
271
272
273
274
275
276
277
			/*
			 * We get here in the unlikely case that a background
			 * request was allocated but not sent
			 */
			spin_lock(&fc->lock);
			if (!fc->blocked)
				wake_up(&fc->blocked_waitq);
			spin_unlock(&fc->lock);
		}

Miklos Szeredi's avatar
Miklos Szeredi committed
278
279
		if (test_bit(FR_WAITING, &req->flags)) {
			__clear_bit(FR_WAITING, &req->flags);
280
			atomic_dec(&fc->num_waiting);
Miklos Szeredi's avatar
Miklos Szeredi committed
281
		}
282
283
284
285
286

		if (req->stolen_file)
			put_reserved_req(fc, req);
		else
			fuse_request_free(req);
287
288
	}
}
289
EXPORT_SYMBOL_GPL(fuse_put_request);
290

291
292
293
294
295
296
297
298
299
300
301
static unsigned len_args(unsigned numargs, struct fuse_arg *args)
{
	unsigned nbytes = 0;
	unsigned i;

	for (i = 0; i < numargs; i++)
		nbytes += args[i].size;

	return nbytes;
}

302
static u64 fuse_get_unique(struct fuse_iqueue *fiq)
303
{
304
	return ++fiq->reqctr;
305
306
}

307
static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
308
309
310
{
	req->in.h.len = sizeof(struct fuse_in_header) +
		len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
311
	list_add_tail(&req->list, &fiq->pending);
Miklos Szeredi's avatar
Miklos Szeredi committed
312
	wake_up_locked(&fiq->waitq);
313
	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
314
315
}

316
317
318
void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
		       u64 nodeid, u64 nlookup)
{
319
320
	struct fuse_iqueue *fiq = &fc->iq;

321
322
	forget->forget_one.nodeid = nodeid;
	forget->forget_one.nlookup = nlookup;
323

Miklos Szeredi's avatar
Miklos Szeredi committed
324
	spin_lock(&fiq->waitq.lock);
325
	if (fiq->connected) {
326
327
		fiq->forget_list_tail->next = forget;
		fiq->forget_list_tail = forget;
Miklos Szeredi's avatar
Miklos Szeredi committed
328
		wake_up_locked(&fiq->waitq);
329
		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredi's avatar
Miklos Szeredi committed
330
331
332
	} else {
		kfree(forget);
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
333
	spin_unlock(&fiq->waitq.lock);
334
335
}

336
337
static void flush_bg_queue(struct fuse_conn *fc)
{
338
	while (fc->active_background < fc->max_background &&
339
340
	       !list_empty(&fc->bg_queue)) {
		struct fuse_req *req;
341
		struct fuse_iqueue *fiq = &fc->iq;
342
343
344
345

		req = list_entry(fc->bg_queue.next, struct fuse_req, list);
		list_del(&req->list);
		fc->active_background++;
Miklos Szeredi's avatar
Miklos Szeredi committed
346
		spin_lock(&fiq->waitq.lock);
347
348
		req->in.h.unique = fuse_get_unique(fiq);
		queue_request(fiq, req);
Miklos Szeredi's avatar
Miklos Szeredi committed
349
		spin_unlock(&fiq->waitq.lock);
350
351
352
	}
}

353
354
/*
 * This function is called when a request is finished.  Either a reply
355
 * has arrived or it was aborted (and not yet sent) or some error
356
 * occurred during communication with userspace, or the device file
357
358
359
 * was closed.  The requester thread is woken up (if still waiting),
 * the 'end' callback is called if given, else the reference to the
 * request is released
360
361
362
 */
static void request_end(struct fuse_conn *fc, struct fuse_req *req)
{
Miklos Szeredi's avatar
Miklos Szeredi committed
363
	struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi's avatar
Miklos Szeredi committed
364

365
	if (test_and_set_bit(FR_FINISHED, &req->flags))
Miklos Szeredi's avatar
Miklos Szeredi committed
366
367
		return;

Miklos Szeredi's avatar
Miklos Szeredi committed
368
	spin_lock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
369
	list_del_init(&req->intr_entry);
Miklos Szeredi's avatar
Miklos Szeredi committed
370
	spin_unlock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
371
372
	WARN_ON(test_bit(FR_PENDING, &req->flags));
	WARN_ON(test_bit(FR_SENT, &req->flags));
Miklos Szeredi's avatar
Miklos Szeredi committed
373
	if (test_bit(FR_BACKGROUND, &req->flags)) {
374
		spin_lock(&fc->lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
375
		clear_bit(FR_BACKGROUND, &req->flags);
376
		if (fc->num_background == fc->max_background)
377
			fc->blocked = 0;
378
379

		/* Wake up next waiter, if any */
Miklos Szeredi's avatar
Miklos Szeredi committed
380
		if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
381
382
			wake_up(&fc->blocked_waitq);

383
		if (fc->num_background == fc->congestion_threshold &&
384
		    fc->connected && fc->bdi_initialized) {
385
386
			clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
			clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
387
		}
388
		fc->num_background--;
389
390
		fc->active_background--;
		flush_bg_queue(fc);
391
		spin_unlock(&fc->lock);
392
	}
393
	wake_up(&req->waitq);
Miklos Szeredi's avatar
Miklos Szeredi committed
394
395
	if (req->end)
		req->end(fc, req);
396
	fuse_put_request(fc, req);
397
398
}

399
static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
400
{
Miklos Szeredi's avatar
Miklos Szeredi committed
401
	spin_lock(&fiq->waitq.lock);
402
403
404
405
	if (test_bit(FR_FINISHED, &req->flags)) {
		spin_unlock(&fiq->waitq.lock);
		return;
	}
406
407
408
409
	if (list_empty(&req->intr_entry)) {
		list_add_tail(&req->intr_entry, &fiq->interrupts);
		wake_up_locked(&fiq->waitq);
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
410
	spin_unlock(&fiq->waitq.lock);
411
	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
412
413
}

414
static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
415
{
Miklos Szeredi's avatar
Miklos Szeredi committed
416
	struct fuse_iqueue *fiq = &fc->iq;
417
418
	int err;

419
420
	if (!fc->no_interrupt) {
		/* Any signal may interrupt this */
421
		err = wait_event_interruptible(req->waitq,
Miklos Szeredi's avatar
Miklos Szeredi committed
422
					test_bit(FR_FINISHED, &req->flags));
423
		if (!err)
424
425
			return;

Miklos Szeredi's avatar
Miklos Szeredi committed
426
		set_bit(FR_INTERRUPTED, &req->flags);
427
428
		/* matches barrier in fuse_dev_do_read() */
		smp_mb__after_atomic();
Miklos Szeredi's avatar
Miklos Szeredi committed
429
		if (test_bit(FR_SENT, &req->flags))
Miklos Szeredi's avatar
Miklos Szeredi committed
430
			queue_interrupt(fiq, req);
431
432
	}

Miklos Szeredi's avatar
Miklos Szeredi committed
433
	if (!test_bit(FR_FORCE, &req->flags)) {
434
		/* Only fatal signals may interrupt this */
435
		err = wait_event_killable(req->waitq,
Miklos Szeredi's avatar
Miklos Szeredi committed
436
					test_bit(FR_FINISHED, &req->flags));
437
		if (!err)
Miklos Szeredi's avatar
Miklos Szeredi committed
438
439
			return;

Miklos Szeredi's avatar
Miklos Szeredi committed
440
		spin_lock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
441
		/* Request is not yet in userspace, bail out */
Miklos Szeredi's avatar
Miklos Szeredi committed
442
		if (test_bit(FR_PENDING, &req->flags)) {
Miklos Szeredi's avatar
Miklos Szeredi committed
443
			list_del(&req->list);
Miklos Szeredi's avatar
Miklos Szeredi committed
444
			spin_unlock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
445
446
447
448
			__fuse_put_request(req);
			req->out.h.error = -EINTR;
			return;
		}
Miklos Szeredi's avatar
Miklos Szeredi committed
449
		spin_unlock(&fiq->waitq.lock);
450
	}
451

Miklos Szeredi's avatar
Miklos Szeredi committed
452
453
454
455
	/*
	 * Either request is already in userspace, or it was forced.
	 * Wait it out.
	 */
Miklos Szeredi's avatar
Miklos Szeredi committed
456
	wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
457
458
}

459
static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
460
{
461
462
	struct fuse_iqueue *fiq = &fc->iq;

Miklos Szeredi's avatar
Miklos Szeredi committed
463
	BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
Miklos Szeredi's avatar
Miklos Szeredi committed
464
	spin_lock(&fiq->waitq.lock);
465
	if (!fiq->connected) {
Miklos Szeredi's avatar
Miklos Szeredi committed
466
		spin_unlock(&fiq->waitq.lock);
467
		req->out.h.error = -ENOTCONN;
468
	} else {
469
470
		req->in.h.unique = fuse_get_unique(fiq);
		queue_request(fiq, req);
471
472
473
		/* acquire extra reference, since request is still needed
		   after request_end() */
		__fuse_get_request(req);
Miklos Szeredi's avatar
Miklos Szeredi committed
474
		spin_unlock(&fiq->waitq.lock);
475

476
		request_wait_answer(fc, req);
477
478
		/* Pairs with smp_wmb() in request_end() */
		smp_rmb();
479
480
	}
}
481
482
483

void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
{
Miklos Szeredi's avatar
Miklos Szeredi committed
484
485
486
	__set_bit(FR_ISREPLY, &req->flags);
	if (!test_bit(FR_WAITING, &req->flags)) {
		__set_bit(FR_WAITING, &req->flags);
487
488
		atomic_inc(&fc->num_waiting);
	}
489
490
	__fuse_request_send(fc, req);
}
491
EXPORT_SYMBOL_GPL(fuse_request_send);
492

493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
{
	if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
		args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;

	if (fc->minor < 9) {
		switch (args->in.h.opcode) {
		case FUSE_LOOKUP:
		case FUSE_CREATE:
		case FUSE_MKNOD:
		case FUSE_MKDIR:
		case FUSE_SYMLINK:
		case FUSE_LINK:
			args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
			break;
		case FUSE_GETATTR:
		case FUSE_SETATTR:
			args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
			break;
		}
	}
	if (fc->minor < 12) {
		switch (args->in.h.opcode) {
		case FUSE_CREATE:
			args->in.args[0].size = sizeof(struct fuse_open_in);
			break;
		case FUSE_MKNOD:
			args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
			break;
		}
	}
}

526
527
528
529
530
531
532
533
534
ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
{
	struct fuse_req *req;
	ssize_t ret;

	req = fuse_get_req(fc, 0);
	if (IS_ERR(req))
		return PTR_ERR(req);

535
536
537
	/* Needs to be done after fuse_get_req() so that fc->minor is valid */
	fuse_adjust_compat(fc, args);

538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
	req->in.h.opcode = args->in.h.opcode;
	req->in.h.nodeid = args->in.h.nodeid;
	req->in.numargs = args->in.numargs;
	memcpy(req->in.args, args->in.args,
	       args->in.numargs * sizeof(struct fuse_in_arg));
	req->out.argvar = args->out.argvar;
	req->out.numargs = args->out.numargs;
	memcpy(req->out.args, args->out.args,
	       args->out.numargs * sizeof(struct fuse_arg));
	fuse_request_send(fc, req);
	ret = req->out.h.error;
	if (!ret && args->out.argvar) {
		BUG_ON(args->out.numargs != 1);
		ret = req->out.args[0].size;
	}
	fuse_put_request(fc, req);

	return ret;
}

558
559
560
561
562
563
564
/*
 * Called under fc->lock
 *
 * fc->connected must have been checked previously
 */
void fuse_request_send_background_locked(struct fuse_conn *fc,
					 struct fuse_req *req)
565
{
Miklos Szeredi's avatar
Miklos Szeredi committed
566
567
568
	BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
	if (!test_bit(FR_WAITING, &req->flags)) {
		__set_bit(FR_WAITING, &req->flags);
569
570
		atomic_inc(&fc->num_waiting);
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
571
	__set_bit(FR_ISREPLY, &req->flags);
572
	fc->num_background++;
573
	if (fc->num_background == fc->max_background)
574
		fc->blocked = 1;
575
	if (fc->num_background == fc->congestion_threshold &&
576
	    fc->bdi_initialized) {
577
578
		set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
		set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
579
580
581
582
583
	}
	list_add_tail(&req->list, &fc->bg_queue);
	flush_bg_queue(fc);
}

584
void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
585
{
586
	BUG_ON(!req->end);
587
	spin_lock(&fc->lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
588
	if (fc->connected) {
589
		fuse_request_send_background_locked(fc, req);
590
		spin_unlock(&fc->lock);
591
	} else {
592
		spin_unlock(&fc->lock);
593
		req->out.h.error = -ENOTCONN;
594
595
		req->end(fc, req);
		fuse_put_request(fc, req);
596
597
	}
}
598
EXPORT_SYMBOL_GPL(fuse_request_send_background);
599

Miklos Szeredi's avatar
Miklos Szeredi committed
600
601
602
603
static int fuse_request_send_notify_reply(struct fuse_conn *fc,
					  struct fuse_req *req, u64 unique)
{
	int err = -ENODEV;
604
	struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi's avatar
Miklos Szeredi committed
605

Miklos Szeredi's avatar
Miklos Szeredi committed
606
	__clear_bit(FR_ISREPLY, &req->flags);
Miklos Szeredi's avatar
Miklos Szeredi committed
607
	req->in.h.unique = unique;
Miklos Szeredi's avatar
Miklos Szeredi committed
608
	spin_lock(&fiq->waitq.lock);
609
	if (fiq->connected) {
610
		queue_request(fiq, req);
Miklos Szeredi's avatar
Miklos Szeredi committed
611
612
		err = 0;
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
613
	spin_unlock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
614
615
616
617

	return err;
}

618
619
void fuse_force_forget(struct file *file, u64 nodeid)
{
620
	struct inode *inode = file_inode(file);
621
622
623
624
625
626
	struct fuse_conn *fc = get_fuse_conn(inode);
	struct fuse_req *req;
	struct fuse_forget_in inarg;

	memset(&inarg, 0, sizeof(inarg));
	inarg.nlookup = 1;
627
	req = fuse_get_req_nofail_nopages(fc, file);
628
629
630
631
632
	req->in.h.opcode = FUSE_FORGET;
	req->in.h.nodeid = nodeid;
	req->in.numargs = 1;
	req->in.args[0].size = sizeof(inarg);
	req->in.args[0].value = &inarg;
Miklos Szeredi's avatar
Miklos Szeredi committed
633
	__clear_bit(FR_ISREPLY, &req->flags);
634
635
636
	__fuse_request_send(fc, req);
	/* ignore errors */
	fuse_put_request(fc, req);
637
638
}

639
640
641
/*
 * Lock the request.  Up to the next unlock_request() there mustn't be
 * anything that could cause a page-fault.  If the request was already
642
 * aborted bail out.
643
 */
644
static int lock_request(struct fuse_req *req)
645
646
647
{
	int err = 0;
	if (req) {
648
		spin_lock(&req->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
649
		if (test_bit(FR_ABORTED, &req->flags))
650
651
			err = -ENOENT;
		else
Miklos Szeredi's avatar
Miklos Szeredi committed
652
			set_bit(FR_LOCKED, &req->flags);
653
		spin_unlock(&req->waitq.lock);
654
655
656
657
658
	}
	return err;
}

/*
Miklos Szeredi's avatar
Miklos Szeredi committed
659
660
 * Unlock request.  If it was aborted while locked, caller is responsible
 * for unlocking and ending the request.
661
 */
662
static int unlock_request(struct fuse_req *req)
663
{
Miklos Szeredi's avatar
Miklos Szeredi committed
664
	int err = 0;
665
	if (req) {
666
		spin_lock(&req->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
667
		if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi's avatar
Miklos Szeredi committed
668
669
			err = -ENOENT;
		else
Miklos Szeredi's avatar
Miklos Szeredi committed
670
			clear_bit(FR_LOCKED, &req->flags);
671
		spin_unlock(&req->waitq.lock);
672
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
673
	return err;
674
675
676
677
678
}

struct fuse_copy_state {
	int write;
	struct fuse_req *req;
679
	struct iov_iter *iter;
680
681
682
	struct pipe_buffer *pipebufs;
	struct pipe_buffer *currbuf;
	struct pipe_inode_info *pipe;
683
684
685
	unsigned long nr_segs;
	struct page *pg;
	unsigned len;
686
	unsigned offset;
687
	unsigned move_pages:1;
688
689
};

690
static void fuse_copy_init(struct fuse_copy_state *cs, int write,
691
			   struct iov_iter *iter)
692
693
694
{
	memset(cs, 0, sizeof(*cs));
	cs->write = write;
695
	cs->iter = iter;
696
697
698
}

/* Unmap and put previous page of userspace buffer */
699
static void fuse_copy_finish(struct fuse_copy_state *cs)
700
{
701
702
703
	if (cs->currbuf) {
		struct pipe_buffer *buf = cs->currbuf;

704
		if (cs->write)
705
			buf->len = PAGE_SIZE - cs->len;
706
		cs->currbuf = NULL;
707
	} else if (cs->pg) {
708
709
710
711
712
713
		if (cs->write) {
			flush_dcache_page(cs->pg);
			set_page_dirty_lock(cs->pg);
		}
		put_page(cs->pg);
	}
714
	cs->pg = NULL;
715
716
717
718
719
720
721
722
}

/*
 * Get another pagefull of userspace buffer, and map it to kernel
 * address space, and lock request
 */
static int fuse_copy_fill(struct fuse_copy_state *cs)
{
723
	struct page *page;
724
725
	int err;

726
	err = unlock_request(cs->req);
Miklos Szeredi's avatar
Miklos Szeredi committed
727
728
729
	if (err)
		return err;

730
	fuse_copy_finish(cs);
731
732
733
	if (cs->pipebufs) {
		struct pipe_buffer *buf = cs->pipebufs;

734
		if (!cs->write) {
735
			err = pipe_buf_confirm(cs->pipe, buf);
736
737
738
739
740
			if (err)
				return err;

			BUG_ON(!cs->nr_segs);
			cs->currbuf = buf;
741
742
			cs->pg = buf->page;
			cs->offset = buf->offset;
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
			cs->len = buf->len;
			cs->pipebufs++;
			cs->nr_segs--;
		} else {
			if (cs->nr_segs == cs->pipe->buffers)
				return -EIO;

			page = alloc_page(GFP_HIGHUSER);
			if (!page)
				return -ENOMEM;

			buf->page = page;
			buf->offset = 0;
			buf->len = 0;

			cs->currbuf = buf;
759
760
			cs->pg = page;
			cs->offset = 0;
761
762
763
764
			cs->len = PAGE_SIZE;
			cs->pipebufs++;
			cs->nr_segs++;
		}
765
	} else {
766
767
		size_t off;
		err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
768
769
		if (err < 0)
			return err;
770
771
772
		BUG_ON(!err);
		cs->len = err;
		cs->offset = off;
773
		cs->pg = page;
774
		iov_iter_advance(cs->iter, err);
775
776
	}

777
	return lock_request(cs->req);
778
779
780
}

/* Do as much copy to/from userspace buffer as we can */
781
static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
782
783
784
{
	unsigned ncpy = min(*size, cs->len);
	if (val) {
785
786
787
		void *pgaddr = kmap_atomic(cs->pg);
		void *buf = pgaddr + cs->offset;

788
		if (cs->write)
789
			memcpy(buf, *val, ncpy);
790
		else
791
792
793
			memcpy(*val, buf, ncpy);

		kunmap_atomic(pgaddr);
794
795
796
797
		*val += ncpy;
	}
	*size -= ncpy;
	cs->len -= ncpy;
798
	cs->offset += ncpy;
799
800
801
	return ncpy;
}

802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
static int fuse_check_page(struct page *page)
{
	if (page_mapcount(page) ||
	    page->mapping != NULL ||
	    page_count(page) != 1 ||
	    (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
	     ~(1 << PG_locked |
	       1 << PG_referenced |
	       1 << PG_uptodate |
	       1 << PG_lru |
	       1 << PG_active |
	       1 << PG_reclaim))) {
		printk(KERN_WARNING "fuse: trying to steal weird page\n");
		printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
		return 1;
	}
	return 0;
}

static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
{
	int err;
	struct page *oldpage = *pagep;
	struct page *newpage;
	struct pipe_buffer *buf = cs->pipebufs;

828
	err = unlock_request(cs->req);
Miklos Szeredi's avatar
Miklos Szeredi committed
829
830
831
	if (err)
		return err;

832
833
	fuse_copy_finish(cs);

834
	err = pipe_buf_confirm(cs->pipe, buf);
835
836
837
838
839
840
841
842
843
844
845
846
	if (err)
		return err;

	BUG_ON(!cs->nr_segs);
	cs->currbuf = buf;
	cs->len = buf->len;
	cs->pipebufs++;
	cs->nr_segs--;

	if (cs->len != PAGE_SIZE)
		goto out_fallback;

847
	if (pipe_buf_steal(cs->pipe, buf) != 0)
848
849
850
851
		goto out_fallback;

	newpage = buf->page;

852
853
	if (!PageUptodate(newpage))
		SetPageUptodate(newpage);
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872

	ClearPageMappedToDisk(newpage);

	if (fuse_check_page(newpage) != 0)
		goto out_fallback_unlock;

	/*
	 * This is a new and locked page, it shouldn't be mapped or
	 * have any special flags on it
	 */
	if (WARN_ON(page_mapped(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(page_has_private(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(PageMlocked(oldpage)))
		goto out_fallback_unlock;

873
	err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
874
	if (err) {
875
876
		unlock_page(newpage);
		return err;
877
	}
878

879
	get_page(newpage);
880
881
882
883
884

	if (!(buf->flags & PIPE_BUF_FLAG_LRU))
		lru_cache_add_file(newpage);

	err = 0;
885
	spin_lock(&cs->req->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
886
	if (test_bit(FR_ABORTED, &cs->req->flags))
887
888
889
		err = -ENOENT;
	else
		*pagep = newpage;
890
	spin_unlock(&cs->req->waitq.lock);
891
892
893

	if (err) {
		unlock_page(newpage);
894
		put_page(newpage);
895
896
897
898
		return err;
	}

	unlock_page(oldpage);
899
	put_page(oldpage);
900
901
902
903
904
905
906
	cs->len = 0;

	return 0;

out_fallback_unlock:
	unlock_page(newpage);
out_fallback:
907
908
	cs->pg = buf->page;
	cs->offset = buf->offset;
909

910
	err = lock_request(cs->req);
911
912
913
914
915
916
	if (err)
		return err;

	return 1;
}

917
918
919
920
static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
			 unsigned offset, unsigned count)
{
	struct pipe_buffer *buf;
Miklos Szeredi's avatar
Miklos Szeredi committed
921
	int err;
922
923
924
925

	if (cs->nr_segs == cs->pipe->buffers)
		return -EIO;

926
	err = unlock_request(cs->req);
Miklos Szeredi's avatar
Miklos Szeredi committed
927
928
929
	if (err)
		return err;

930
931
932
	fuse_copy_finish(cs);

	buf = cs->pipebufs;
933
	get_page(page);
934
935
936
937
938
939
940
941
942
943
944
	buf->page = page;
	buf->offset = offset;
	buf->len = count;

	cs->pipebufs++;
	cs->nr_segs++;
	cs->len = 0;

	return 0;
}

945
946
947
948
/*
 * Copy a page in the request to/from the userspace buffer.  Must be
 * done atomically
 */
949
static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
950
			  unsigned offset, unsigned count, int zeroing)
951
{
952
953
954
	int err;
	struct page *page = *pagep;

955
956
957
	if (page && zeroing && count < PAGE_SIZE)
		clear_highpage(page);

958
	while (count) {
959
960
961
		if (cs->write && cs->pipebufs && page) {
			return fuse_ref_page(cs, page, offset, count);
		} else if (!cs->len) {
962
963
964
965
966
967
968
969
970
971
			if (cs->move_pages && page &&
			    offset == 0 && count == PAGE_SIZE) {
				err = fuse_try_move_page(cs, pagep);
				if (err <= 0)
					return err;
			} else {
				err = fuse_copy_fill(cs);
				if (err)
					return err;
			}
Miklos Szeredi's avatar
Miklos Szeredi committed
972
		}
973
		if (page) {
974
			void *mapaddr = kmap_atomic(page);
975
976
			void *buf = mapaddr + offset;
			offset += fuse_copy_do(cs, &buf, &count);
977
			kunmap_atomic(mapaddr);
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
		} else
			offset += fuse_copy_do(cs, NULL, &count);
	}
	if (page && !cs->write)
		flush_dcache_page(page);
	return 0;
}

/* Copy pages in the request to/from userspace buffer */
static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
			   int zeroing)
{
	unsigned i;
	struct fuse_req *req = cs->req;

	for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
994
		int err;
995
996
		unsigned offset = req->page_descs[i].offset;
		unsigned count = min(nbytes, req->page_descs[i].length);
997
998
999

		err = fuse_copy_page(cs, &req->pages[i], offset, count,
				     zeroing);
1000
		if (err)
For faster browsing, not all history is shown. View entire blame