netback.c 47.7 KB
Newer Older
Ian Campbell's avatar
Ian Campbell committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/*
 * Back-end of the driver for virtual network devices. This portion of the
 * driver exports a 'unified' network-device interface that can be accessed
 * by any operating system that implements a compatible front end. A
 * reference front-end implementation can be found in:
 *  drivers/net/xen-netfront.c
 *
 * Copyright (c) 2002-2005, K A Fraser
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation; or, when distributed
 * separately from the Linux kernel or incorporated into other
 * software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "common.h"

#include <linux/kthread.h>
#include <linux/if_vlan.h>
#include <linux/udp.h>
40
#include <linux/highmem.h>
Ian Campbell's avatar
Ian Campbell committed
41
42
43

#include <net/tcp.h>

Stefano Stabellini's avatar
Stefano Stabellini committed
44
#include <xen/xen.h>
Ian Campbell's avatar
Ian Campbell committed
45
46
47
48
49
50
#include <xen/events.h>
#include <xen/interface/memory.h>

#include <asm/xen/hypercall.h>
#include <asm/xen/page.h>

51
52
53
54
55
56
57
/* Provide an option to disable split event channels at load time as
 * event channels are limited resource. Split event channels are
 * enabled by default.
 */
bool separate_tx_rx_irq = 1;
module_param(separate_tx_rx_irq, bool, 0644);

58
/* When guest ring is filled up, qdisc queues the packets for us, but we have
59
 * to timeout them, otherwise other guests' packets can get stuck there
60
61
62
63
64
 */
unsigned int rx_drain_timeout_msecs = 10000;
module_param(rx_drain_timeout_msecs, uint, 0444);
unsigned int rx_drain_timeout_jiffies;

65
66
67
68
/*
 * This is the maximum slots a skb can have. If a guest sends a skb
 * which exceeds this limit it is considered malicious.
 */
69
70
71
72
#define FATAL_SKB_SLOTS_DEFAULT 20
static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT;
module_param(fatal_skb_slots, uint, 0444);

Wei Liu's avatar
Wei Liu committed
73
74
75
static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx,
			       u8 status);

Ian Campbell's avatar
Ian Campbell committed
76
77
78
static void make_tx_response(struct xenvif *vif,
			     struct xen_netif_tx_request *txp,
			     s8       st);
79
80
81
82

static inline int tx_work_todo(struct xenvif *vif);
static inline int rx_work_todo(struct xenvif *vif);

Ian Campbell's avatar
Ian Campbell committed
83
84
85
86
87
88
89
static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
					     u16      id,
					     s8       st,
					     u16      offset,
					     u16      size,
					     u16      flags);

90
static inline unsigned long idx_to_pfn(struct xenvif *vif,
91
				       u16 idx)
Ian Campbell's avatar
Ian Campbell committed
92
{
93
	return page_to_pfn(vif->mmap_pages[idx]);
Ian Campbell's avatar
Ian Campbell committed
94
95
}

96
static inline unsigned long idx_to_kaddr(struct xenvif *vif,
97
					 u16 idx)
Ian Campbell's avatar
Ian Campbell committed
98
{
99
	return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif, idx));
Ian Campbell's avatar
Ian Campbell committed
100
101
}

102
103
/* Find the containing VIF's structure from a pointer in pending_tx_info array
 */
104
105
static inline struct xenvif* ubuf_to_vif(struct ubuf_info *ubuf)
{
106
107
108
109
110
111
	u16 pending_idx = ubuf->desc;
	struct pending_tx_info *temp =
		container_of(ubuf, struct pending_tx_info, callback_struct);
	return container_of(temp - pending_idx,
			    struct xenvif,
			    pending_tx_info[0]);
112
}
113

114
115
116
117
/* This is a miniumum size for the linear area to avoid lots of
 * calls to __pskb_pull_tail() as we set up checksum offsets. The
 * value 128 was chosen as it covers all IPv4 and most likely
 * IPv6 headers.
Ian Campbell's avatar
Ian Campbell committed
118
 */
119
#define PKT_PROT_LEN 128
Ian Campbell's avatar
Ian Campbell committed
120

121
122
123
124
125
126
127
128
129
130
static u16 frag_get_pending_idx(skb_frag_t *frag)
{
	return (u16)frag->page_offset;
}

static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx)
{
	frag->page_offset = pending_idx;
}

Ian Campbell's avatar
Ian Campbell committed
131
132
133
134
135
static inline pending_ring_idx_t pending_index(unsigned i)
{
	return i & (MAX_PENDING_REQS-1);
}

136
bool xenvif_rx_ring_slots_available(struct xenvif *vif, int needed)
Ian Campbell's avatar
Ian Campbell committed
137
{
138
	RING_IDX prod, cons;
Ian Campbell's avatar
Ian Campbell committed
139

140
141
142
	do {
		prod = vif->rx.sring->req_prod;
		cons = vif->rx.req_cons;
Ian Campbell's avatar
Ian Campbell committed
143

144
145
		if (prod - cons >= needed)
			return true;
Ian Campbell's avatar
Ian Campbell committed
146

147
		vif->rx.sring->req_event = prod + 1;
Ian Campbell's avatar
Ian Campbell committed
148

149
150
151
152
153
		/* Make sure event is visible before we check prod
		 * again.
		 */
		mb();
	} while (vif->rx.sring->req_prod != prod);
Ian Campbell's avatar
Ian Campbell committed
154

155
	return false;
Ian Campbell's avatar
Ian Campbell committed
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
}

/*
 * Returns true if we should start a new receive buffer instead of
 * adding 'size' bytes to a buffer which currently contains 'offset'
 * bytes.
 */
static bool start_new_rx_buffer(int offset, unsigned long size, int head)
{
	/* simple case: we have completely filled the current buffer. */
	if (offset == MAX_BUFFER_OFFSET)
		return true;

	/*
	 * complex case: start a fresh buffer if the current frag
	 * would overflow the current buffer but only if:
	 *     (i)   this frag would fit completely in the next buffer
	 * and (ii)  there is already some data in the current buffer
	 * and (iii) this is not the head buffer.
	 *
	 * Where:
	 * - (i) stops us splitting a frag into two copies
	 *   unless the frag is too large for a single buffer.
	 * - (ii) stops us from leaving a buffer pointlessly empty.
	 * - (iii) stops us leaving the first buffer
	 *   empty. Strictly speaking this is already covered
	 *   by (ii) but is explicitly checked because
	 *   netfront relies on the first buffer being
	 *   non-empty and can crash otherwise.
	 *
	 * This means we will effectively linearise small
	 * frags but do not needlessly split large buffers
	 * into multiple copies tend to give large frags their
	 * own buffers as before.
	 */
	if ((offset + size > MAX_BUFFER_OFFSET) &&
	    (size <= MAX_BUFFER_OFFSET) && offset && !head)
		return true;

	return false;
}

struct netrx_pending_operations {
	unsigned copy_prod, copy_cons;
	unsigned meta_prod, meta_cons;
	struct gnttab_copy *copy;
202
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
203
204
205
206
	int copy_off;
	grant_ref_t copy_gref;
};

207
208
static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif *vif,
						 struct netrx_pending_operations *npo)
Ian Campbell's avatar
Ian Campbell committed
209
{
210
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
211
212
213
214
215
	struct xen_netif_rx_request *req;

	req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);

	meta = npo->meta + npo->meta_prod++;
216
	meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
Ian Campbell's avatar
Ian Campbell committed
217
218
219
220
221
222
223
224
225
226
	meta->gso_size = 0;
	meta->size = 0;
	meta->id = req->id;

	npo->copy_off = 0;
	npo->copy_gref = req->gref;

	return meta;
}

227
228
229
230
/*
 * Set up the grant operations for this fragment. If it's a flipping
 * interface, we also set up the unmap request from here.
 */
Wei Liu's avatar
Wei Liu committed
231
232
233
static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
				 struct netrx_pending_operations *npo,
				 struct page *page, unsigned long size,
234
235
236
				 unsigned long offset, int *head,
				 struct xenvif *foreign_vif,
				 grant_ref_t foreign_gref)
Ian Campbell's avatar
Ian Campbell committed
237
238
{
	struct gnttab_copy *copy_gop;
239
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
240
	unsigned long bytes;
241
	int gso_type = XEN_NETIF_GSO_TYPE_NONE;
Ian Campbell's avatar
Ian Campbell committed
242
243

	/* Data must not cross a page boundary. */
244
	BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
Ian Campbell's avatar
Ian Campbell committed
245
246
247

	meta = npo->meta + npo->meta_prod - 1;

248
249
250
251
	/* Skip unused frames from start of page */
	page += offset >> PAGE_SHIFT;
	offset &= ~PAGE_MASK;

Ian Campbell's avatar
Ian Campbell committed
252
	while (size > 0) {
253
		BUG_ON(offset >= PAGE_SIZE);
Ian Campbell's avatar
Ian Campbell committed
254
255
		BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);

256
257
258
259
260
		bytes = PAGE_SIZE - offset;

		if (bytes > size)
			bytes = size;

261
		if (start_new_rx_buffer(npo->copy_off, bytes, *head)) {
Ian Campbell's avatar
Ian Campbell committed
262
263
264
265
			/*
			 * Netfront requires there to be some data in the head
			 * buffer.
			 */
266
			BUG_ON(*head);
Ian Campbell's avatar
Ian Campbell committed
267
268
269
270
271
272
273
274
275

			meta = get_next_rx_buffer(vif, npo);
		}

		if (npo->copy_off + bytes > MAX_BUFFER_OFFSET)
			bytes = MAX_BUFFER_OFFSET - npo->copy_off;

		copy_gop = npo->copy + npo->copy_prod++;
		copy_gop->flags = GNTCOPY_dest_gref;
276
277
		copy_gop->len = bytes;

278
279
280
281
282
283
284
285
286
		if (foreign_vif) {
			copy_gop->source.domid = foreign_vif->domid;
			copy_gop->source.u.ref = foreign_gref;
			copy_gop->flags |= GNTCOPY_source_gref;
		} else {
			copy_gop->source.domid = DOMID_SELF;
			copy_gop->source.u.gmfn =
				virt_to_mfn(page_address(page));
		}
Ian Campbell's avatar
Ian Campbell committed
287
288
		copy_gop->source.offset = offset;

289
		copy_gop->dest.domid = vif->domid;
Ian Campbell's avatar
Ian Campbell committed
290
291
292
293
294
295
296
297
298
		copy_gop->dest.offset = npo->copy_off;
		copy_gop->dest.u.ref = npo->copy_gref;

		npo->copy_off += bytes;
		meta->size += bytes;

		offset += bytes;
		size -= bytes;

299
300
301
302
303
304
305
		/* Next frame */
		if (offset == PAGE_SIZE && size) {
			BUG_ON(!PageCompound(page));
			page++;
			offset = 0;
		}

Ian Campbell's avatar
Ian Campbell committed
306
		/* Leave a gap for the GSO descriptor. */
307
308
309
310
311
312
		if (skb_is_gso(skb)) {
			if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
				gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
			else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
				gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
		}
313
314

		if (*head && ((1 << gso_type) & vif->gso_mask))
Ian Campbell's avatar
Ian Campbell committed
315
316
			vif->rx.req_cons++;

317
		*head = 0; /* There must be something in this buffer now. */
Ian Campbell's avatar
Ian Campbell committed
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333

	}
}

/*
 * Prepare an SKB to be transmitted to the frontend.
 *
 * This function is responsible for allocating grant operations, meta
 * structures, etc.
 *
 * It returns the number of meta structures consumed. The number of
 * ring slots used is always equal to the number of meta slots used
 * plus the number of GSO descriptors used. Currently, we use either
 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
 * frontend-side LRO).
 */
Wei Liu's avatar
Wei Liu committed
334
335
static int xenvif_gop_skb(struct sk_buff *skb,
			  struct netrx_pending_operations *npo)
Ian Campbell's avatar
Ian Campbell committed
336
337
338
339
340
{
	struct xenvif *vif = netdev_priv(skb->dev);
	int nr_frags = skb_shinfo(skb)->nr_frags;
	int i;
	struct xen_netif_rx_request *req;
341
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
342
	unsigned char *data;
343
	int head = 1;
Ian Campbell's avatar
Ian Campbell committed
344
	int old_meta_prod;
345
	int gso_type;
346
347
348
	struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg;
	grant_ref_t foreign_grefs[MAX_SKB_FRAGS];
	struct xenvif *foreign_vif = NULL;
Ian Campbell's avatar
Ian Campbell committed
349
350
351

	old_meta_prod = npo->meta_prod;

352
353
354
355
356
357
	gso_type = XEN_NETIF_GSO_TYPE_NONE;
	if (skb_is_gso(skb)) {
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
			gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
			gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
358
359
	}

Ian Campbell's avatar
Ian Campbell committed
360
	/* Set up a GSO prefix descriptor, if necessary */
361
	if ((1 << gso_type) & vif->gso_prefix_mask) {
Ian Campbell's avatar
Ian Campbell committed
362
363
		req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
		meta = npo->meta + npo->meta_prod++;
364
		meta->gso_type = gso_type;
365
		meta->gso_size = skb_shinfo(skb)->gso_size;
Ian Campbell's avatar
Ian Campbell committed
366
367
368
369
370
371
372
		meta->size = 0;
		meta->id = req->id;
	}

	req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
	meta = npo->meta + npo->meta_prod++;

373
374
	if ((1 << gso_type) & vif->gso_mask) {
		meta->gso_type = gso_type;
375
		meta->gso_size = skb_shinfo(skb)->gso_size;
376
377
	} else {
		meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
Ian Campbell's avatar
Ian Campbell committed
378
		meta->gso_size = 0;
379
	}
Ian Campbell's avatar
Ian Campbell committed
380
381
382
383
384
385

	meta->size = 0;
	meta->id = req->id;
	npo->copy_off = 0;
	npo->copy_gref = req->gref;

386
387
388
389
390
391
392
393
394
395
396
397
398
	if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
		 (ubuf->callback == &xenvif_zerocopy_callback)) {
		int i = 0;
		foreign_vif = ubuf_to_vif(ubuf);

		do {
			u16 pending_idx = ubuf->desc;
			foreign_grefs[i++] =
				foreign_vif->pending_tx_info[pending_idx].req.gref;
			ubuf = (struct ubuf_info *) ubuf->ctx;
		} while (ubuf);
	}

Ian Campbell's avatar
Ian Campbell committed
399
400
401
402
403
404
405
406
	data = skb->data;
	while (data < skb_tail_pointer(skb)) {
		unsigned int offset = offset_in_page(data);
		unsigned int len = PAGE_SIZE - offset;

		if (data + len > skb_tail_pointer(skb))
			len = skb_tail_pointer(skb) - data;

Wei Liu's avatar
Wei Liu committed
407
		xenvif_gop_frag_copy(vif, skb, npo,
408
409
410
				     virt_to_page(data), len, offset, &head,
				     NULL,
				     0);
Ian Campbell's avatar
Ian Campbell committed
411
412
413
414
		data += len;
	}

	for (i = 0; i < nr_frags; i++) {
Wei Liu's avatar
Wei Liu committed
415
416
417
418
		xenvif_gop_frag_copy(vif, skb, npo,
				     skb_frag_page(&skb_shinfo(skb)->frags[i]),
				     skb_frag_size(&skb_shinfo(skb)->frags[i]),
				     skb_shinfo(skb)->frags[i].page_offset,
419
420
421
				     &head,
				     foreign_vif,
				     foreign_grefs[i]);
Ian Campbell's avatar
Ian Campbell committed
422
423
424
425
426
427
	}

	return npo->meta_prod - old_meta_prod;
}

/*
Wei Liu's avatar
Wei Liu committed
428
 * This is a twin to xenvif_gop_skb.  Assume that xenvif_gop_skb was
Ian Campbell's avatar
Ian Campbell committed
429
430
431
432
 * used to set up the operations on the top of
 * netrx_pending_operations, which have since been done.  Check that
 * they didn't give any errors and advance over them.
 */
Wei Liu's avatar
Wei Liu committed
433
434
static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
			    struct netrx_pending_operations *npo)
Ian Campbell's avatar
Ian Campbell committed
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
{
	struct gnttab_copy     *copy_op;
	int status = XEN_NETIF_RSP_OKAY;
	int i;

	for (i = 0; i < nr_meta_slots; i++) {
		copy_op = npo->copy + npo->copy_cons++;
		if (copy_op->status != GNTST_okay) {
			netdev_dbg(vif->dev,
				   "Bad status %d from copy to DOM%d.\n",
				   copy_op->status, vif->domid);
			status = XEN_NETIF_RSP_ERROR;
		}
	}

	return status;
}

Wei Liu's avatar
Wei Liu committed
453
454
455
static void xenvif_add_frag_responses(struct xenvif *vif, int status,
				      struct xenvif_rx_meta *meta,
				      int nr_meta_slots)
Ian Campbell's avatar
Ian Campbell committed
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
{
	int i;
	unsigned long offset;

	/* No fragments used */
	if (nr_meta_slots <= 1)
		return;

	nr_meta_slots--;

	for (i = 0; i < nr_meta_slots; i++) {
		int flags;
		if (i == nr_meta_slots - 1)
			flags = 0;
		else
			flags = XEN_NETRXF_more_data;

		offset = 0;
		make_rx_response(vif, meta[i].id, status, offset,
				 meta[i].size, flags);
	}
}

479
struct xenvif_rx_cb {
480
481
482
	int meta_slots_used;
};

483
484
#define XENVIF_RX_CB(skb) ((struct xenvif_rx_cb *)(skb)->cb)

485
void xenvif_kick_thread(struct xenvif *vif)
486
487
488
489
{
	wake_up(&vif->wq);
}

490
static void xenvif_rx_action(struct xenvif *vif)
Ian Campbell's avatar
Ian Campbell committed
491
492
{
	s8 status;
493
	u16 flags;
Ian Campbell's avatar
Ian Campbell committed
494
495
496
497
498
499
	struct xen_netif_rx_response *resp;
	struct sk_buff_head rxq;
	struct sk_buff *skb;
	LIST_HEAD(notify);
	int ret;
	unsigned long offset;
500
	bool need_to_notify = false;
Ian Campbell's avatar
Ian Campbell committed
501
502

	struct netrx_pending_operations npo = {
503
504
		.copy  = vif->grant_copy_op,
		.meta  = vif->meta,
Ian Campbell's avatar
Ian Campbell committed
505
506
507
508
	};

	skb_queue_head_init(&rxq);

509
	while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) {
510
		RING_IDX max_slots_needed;
511
512
513
514
515
516
517
518
519
520
521
522
523
524
		int i;

		/* We need a cheap worse case estimate for the number of
		 * slots we'll use.
		 */

		max_slots_needed = DIV_ROUND_UP(offset_in_page(skb->data) +
						skb_headlen(skb),
						PAGE_SIZE);
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
			unsigned int size;
			size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
			max_slots_needed += DIV_ROUND_UP(size, PAGE_SIZE);
		}
525
526
527
		if (skb_is_gso(skb) &&
		   (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 ||
		    skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
528
529
530
531
532
			max_slots_needed++;

		/* If the skb may not fit then bail out now */
		if (!xenvif_rx_ring_slots_available(vif, max_slots_needed)) {
			skb_queue_head(&vif->rx_queue, skb);
533
			need_to_notify = true;
534
			vif->rx_last_skb_slots = max_slots_needed;
535
			break;
536
537
		} else
			vif->rx_last_skb_slots = 0;
Ian Campbell's avatar
Ian Campbell committed
538

539
540
		XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo);
		BUG_ON(XENVIF_RX_CB(skb)->meta_slots_used > max_slots_needed);
Ian Campbell's avatar
Ian Campbell committed
541
542
543
544

		__skb_queue_tail(&rxq, skb);
	}

545
	BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta));
Ian Campbell's avatar
Ian Campbell committed
546
547

	if (!npo.copy_prod)
548
		goto done;
Ian Campbell's avatar
Ian Campbell committed
549

550
	BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS);
551
	gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod);
Ian Campbell's avatar
Ian Campbell committed
552
553
554

	while ((skb = __skb_dequeue(&rxq)) != NULL) {

555
556
		if ((1 << vif->meta[npo.meta_cons].gso_type) &
		    vif->gso_prefix_mask) {
Ian Campbell's avatar
Ian Campbell committed
557
			resp = RING_GET_RESPONSE(&vif->rx,
558
						 vif->rx.rsp_prod_pvt++);
Ian Campbell's avatar
Ian Campbell committed
559
560
561

			resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;

562
563
			resp->offset = vif->meta[npo.meta_cons].gso_size;
			resp->id = vif->meta[npo.meta_cons].id;
564
			resp->status = XENVIF_RX_CB(skb)->meta_slots_used;
Ian Campbell's avatar
Ian Campbell committed
565
566

			npo.meta_cons++;
567
			XENVIF_RX_CB(skb)->meta_slots_used--;
Ian Campbell's avatar
Ian Campbell committed
568
569
570
571
572
573
		}


		vif->dev->stats.tx_bytes += skb->len;
		vif->dev->stats.tx_packets++;

574
575
576
		status = xenvif_check_gop(vif,
					  XENVIF_RX_CB(skb)->meta_slots_used,
					  &npo);
Ian Campbell's avatar
Ian Campbell committed
577

578
		if (XENVIF_RX_CB(skb)->meta_slots_used == 1)
Ian Campbell's avatar
Ian Campbell committed
579
580
581
582
583
584
585
586
587
588
589
			flags = 0;
		else
			flags = XEN_NETRXF_more_data;

		if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */
			flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated;
		else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
			/* remote but checksummed. */
			flags |= XEN_NETRXF_data_validated;

		offset = 0;
590
		resp = make_rx_response(vif, vif->meta[npo.meta_cons].id,
Ian Campbell's avatar
Ian Campbell committed
591
					status, offset,
592
					vif->meta[npo.meta_cons].size,
Ian Campbell's avatar
Ian Campbell committed
593
594
					flags);

595
596
		if ((1 << vif->meta[npo.meta_cons].gso_type) &
		    vif->gso_mask) {
Ian Campbell's avatar
Ian Campbell committed
597
598
599
600
601
602
603
			struct xen_netif_extra_info *gso =
				(struct xen_netif_extra_info *)
				RING_GET_RESPONSE(&vif->rx,
						  vif->rx.rsp_prod_pvt++);

			resp->flags |= XEN_NETRXF_extra_info;

604
			gso->u.gso.type = vif->meta[npo.meta_cons].gso_type;
605
			gso->u.gso.size = vif->meta[npo.meta_cons].gso_size;
Ian Campbell's avatar
Ian Campbell committed
606
607
608
609
610
611
612
			gso->u.gso.pad = 0;
			gso->u.gso.features = 0;

			gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
			gso->flags = 0;
		}

Wei Liu's avatar
Wei Liu committed
613
614
		xenvif_add_frag_responses(vif, status,
					  vif->meta + npo.meta_cons + 1,
615
					  XENVIF_RX_CB(skb)->meta_slots_used);
Ian Campbell's avatar
Ian Campbell committed
616
617
618

		RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret);

619
		need_to_notify |= !!ret;
620

621
		npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used;
Ian Campbell's avatar
Ian Campbell committed
622
623
624
		dev_kfree_skb(skb);
	}

625
done:
626
	if (need_to_notify)
627
		notify_remote_via_irq(vif->rx_irq);
Ian Campbell's avatar
Ian Campbell committed
628
629
}

Wei Liu's avatar
Wei Liu committed
630
void xenvif_check_rx_xenvif(struct xenvif *vif)
Ian Campbell's avatar
Ian Campbell committed
631
632
633
634
635
636
{
	int more_to_do;

	RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do);

	if (more_to_do)
637
		napi_schedule(&vif->napi);
Ian Campbell's avatar
Ian Campbell committed
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
}

static void tx_add_credit(struct xenvif *vif)
{
	unsigned long max_burst, max_credit;

	/*
	 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
	 * Otherwise the interface can seize up due to insufficient credit.
	 */
	max_burst = RING_GET_REQUEST(&vif->tx, vif->tx.req_cons)->size;
	max_burst = min(max_burst, 131072UL);
	max_burst = max(max_burst, vif->credit_bytes);

	/* Take care that adding a new chunk of credit doesn't wrap to zero. */
	max_credit = vif->remaining_credit + vif->credit_bytes;
	if (max_credit < vif->remaining_credit)
		max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */

	vif->remaining_credit = min(max_credit, max_burst);
}

static void tx_credit_callback(unsigned long data)
{
	struct xenvif *vif = (struct xenvif *)data;
	tx_add_credit(vif);
Wei Liu's avatar
Wei Liu committed
664
	xenvif_check_rx_xenvif(vif);
Ian Campbell's avatar
Ian Campbell committed
665
666
}

Wei Liu's avatar
Wei Liu committed
667
668
static void xenvif_tx_err(struct xenvif *vif,
			  struct xen_netif_tx_request *txp, RING_IDX end)
Ian Campbell's avatar
Ian Campbell committed
669
670
{
	RING_IDX cons = vif->tx.req_cons;
671
	unsigned long flags;
Ian Campbell's avatar
Ian Campbell committed
672
673

	do {
674
		spin_lock_irqsave(&vif->response_lock, flags);
Ian Campbell's avatar
Ian Campbell committed
675
		make_tx_response(vif, txp, XEN_NETIF_RSP_ERROR);
676
		spin_unlock_irqrestore(&vif->response_lock, flags);
677
		if (cons == end)
Ian Campbell's avatar
Ian Campbell committed
678
679
680
681
682
683
			break;
		txp = RING_GET_REQUEST(&vif->tx, cons++);
	} while (1);
	vif->tx.req_cons = cons;
}

Wei Liu's avatar
Wei Liu committed
684
static void xenvif_fatal_tx_err(struct xenvif *vif)
685
686
687
688
689
{
	netdev_err(vif->dev, "fatal error; disabling device\n");
	xenvif_carrier_off(vif);
}

Wei Liu's avatar
Wei Liu committed
690
691
692
693
static int xenvif_count_requests(struct xenvif *vif,
				 struct xen_netif_tx_request *first,
				 struct xen_netif_tx_request *txp,
				 int work_to_do)
Ian Campbell's avatar
Ian Campbell committed
694
695
{
	RING_IDX cons = vif->tx.req_cons;
696
697
	int slots = 0;
	int drop_err = 0;
698
	int more_data;
Ian Campbell's avatar
Ian Campbell committed
699
700
701
702
703

	if (!(first->flags & XEN_NETTXF_more_data))
		return 0;

	do {
704
705
		struct xen_netif_tx_request dropped_tx = { 0 };

706
707
708
709
		if (slots >= work_to_do) {
			netdev_err(vif->dev,
				   "Asked for %d slots but exceeds this limit\n",
				   work_to_do);
Wei Liu's avatar
Wei Liu committed
710
			xenvif_fatal_tx_err(vif);
711
			return -ENODATA;
Ian Campbell's avatar
Ian Campbell committed
712
713
		}

714
715
716
		/* This guest is really using too many slots and
		 * considered malicious.
		 */
717
		if (unlikely(slots >= fatal_skb_slots)) {
718
719
			netdev_err(vif->dev,
				   "Malicious frontend using %d slots, threshold %u\n",
720
				   slots, fatal_skb_slots);
Wei Liu's avatar
Wei Liu committed
721
			xenvif_fatal_tx_err(vif);
722
			return -E2BIG;
Ian Campbell's avatar
Ian Campbell committed
723
724
		}

725
		/* Xen network protocol had implicit dependency on
726
727
728
729
730
		 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
		 * the historical MAX_SKB_FRAGS value 18 to honor the
		 * same behavior as before. Any packet using more than
		 * 18 slots but less than fatal_skb_slots slots is
		 * dropped
731
		 */
732
		if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) {
733
734
735
			if (net_ratelimit())
				netdev_dbg(vif->dev,
					   "Too many slots (%d) exceeding limit (%d), dropping packet\n",
736
					   slots, XEN_NETBK_LEGACY_SLOTS_MAX);
737
738
739
			drop_err = -E2BIG;
		}

740
741
742
		if (drop_err)
			txp = &dropped_tx;

743
		memcpy(txp, RING_GET_REQUEST(&vif->tx, cons + slots),
Ian Campbell's avatar
Ian Campbell committed
744
		       sizeof(*txp));
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760

		/* If the guest submitted a frame >= 64 KiB then
		 * first->size overflowed and following slots will
		 * appear to be larger than the frame.
		 *
		 * This cannot be fatal error as there are buggy
		 * frontends that do this.
		 *
		 * Consume all slots and drop the packet.
		 */
		if (!drop_err && txp->size > first->size) {
			if (net_ratelimit())
				netdev_dbg(vif->dev,
					   "Invalid tx request, slot size %u > remaining size %u\n",
					   txp->size, first->size);
			drop_err = -EIO;
Ian Campbell's avatar
Ian Campbell committed
761
762
763
		}

		first->size -= txp->size;
764
		slots++;
Ian Campbell's avatar
Ian Campbell committed
765
766

		if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
767
			netdev_err(vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n",
Ian Campbell's avatar
Ian Campbell committed
768
				 txp->offset, txp->size);
Wei Liu's avatar
Wei Liu committed
769
			xenvif_fatal_tx_err(vif);
770
			return -EINVAL;
Ian Campbell's avatar
Ian Campbell committed
771
		}
772
773
774
775
776
777
778

		more_data = txp->flags & XEN_NETTXF_more_data;

		if (!drop_err)
			txp++;

	} while (more_data);
779
780

	if (drop_err) {
Wei Liu's avatar
Wei Liu committed
781
		xenvif_tx_err(vif, first, cons + slots);
782
783
784
785
		return drop_err;
	}

	return slots;
Ian Campbell's avatar
Ian Campbell committed
786
787
}

788
789
790
791
792
793
794

struct xenvif_tx_cb {
	u16 pending_idx;
};

#define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)

795
796
797
798
799
800
801
802
803
804
805
806
807
808
static inline void xenvif_tx_create_gop(struct xenvif *vif,
					u16 pending_idx,
					struct xen_netif_tx_request *txp,
					struct gnttab_map_grant_ref *gop)
{
	vif->pages_to_map[gop-vif->tx_map_ops] = vif->mmap_pages[pending_idx];
	gnttab_set_map_op(gop, idx_to_kaddr(vif, pending_idx),
			  GNTMAP_host_map | GNTMAP_readonly,
			  txp->gref, vif->domid);

	memcpy(&vif->pending_tx_info[pending_idx].req, txp,
	       sizeof(*txp));
}

809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
static inline struct sk_buff *xenvif_alloc_skb(unsigned int size)
{
	struct sk_buff *skb =
		alloc_skb(size + NET_SKB_PAD + NET_IP_ALIGN,
			  GFP_ATOMIC | __GFP_NOWARN);
	if (unlikely(skb == NULL))
		return NULL;

	/* Packets passed to netif_rx() must have some headroom. */
	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);

	/* Initialize it here to avoid later surprises */
	skb_shinfo(skb)->destructor_arg = NULL;

	return skb;
}

826
827
828
829
static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
							struct sk_buff *skb,
							struct xen_netif_tx_request *txp,
							struct gnttab_map_grant_ref *gop)
Ian Campbell's avatar
Ian Campbell committed
830
831
832
{
	struct skb_shared_info *shinfo = skb_shinfo(skb);
	skb_frag_t *frags = shinfo->frags;
833
	u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
834
835
	int start;
	pending_ring_idx_t index;
836
	unsigned int nr_slots, frag_overflow = 0;
837
838

	/* At this point shinfo->nr_frags is in fact the number of
839
	 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
840
	 */
841
842
843
844
845
	if (shinfo->nr_frags > MAX_SKB_FRAGS) {
		frag_overflow = shinfo->nr_frags - MAX_SKB_FRAGS;
		BUG_ON(frag_overflow > MAX_SKB_FRAGS);
		shinfo->nr_frags = MAX_SKB_FRAGS;
	}
846
	nr_slots = shinfo->nr_frags;
Ian Campbell's avatar
Ian Campbell committed
847
848

	/* Skip first skb fragment if it is on same page as header fragment. */
849
	start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
Ian Campbell's avatar
Ian Campbell committed
850

851
852
	for (shinfo->nr_frags = start; shinfo->nr_frags < nr_slots;
	     shinfo->nr_frags++, txp++, gop++) {
853
854
		index = pending_index(vif->pending_cons++);
		pending_idx = vif->pending_ring[index];
855
856
		xenvif_tx_create_gop(vif, pending_idx, txp, gop);
		frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx);
Ian Campbell's avatar
Ian Campbell committed
857
858
	}

859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
	if (frag_overflow) {
		struct sk_buff *nskb = xenvif_alloc_skb(0);
		if (unlikely(nskb == NULL)) {
			if (net_ratelimit())
				netdev_err(vif->dev,
					   "Can't allocate the frag_list skb.\n");
			return NULL;
		}

		shinfo = skb_shinfo(nskb);
		frags = shinfo->frags;

		for (shinfo->nr_frags = 0; shinfo->nr_frags < frag_overflow;
		     shinfo->nr_frags++, txp++, gop++) {
			index = pending_index(vif->pending_cons++);
			pending_idx = vif->pending_ring[index];
			xenvif_tx_create_gop(vif, pending_idx, txp, gop);
			frag_set_pending_idx(&frags[shinfo->nr_frags],
					     pending_idx);
		}

		skb_shinfo(skb)->frag_list = nskb;
	}
882

Ian Campbell's avatar
Ian Campbell committed
883
884
885
	return gop;
}

886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
static inline void xenvif_grant_handle_set(struct xenvif *vif,
					   u16 pending_idx,
					   grant_handle_t handle)
{
	if (unlikely(vif->grant_tx_handle[pending_idx] !=
		     NETBACK_INVALID_HANDLE)) {
		netdev_err(vif->dev,
			   "Trying to overwrite active handle! pending_idx: %x\n",
			   pending_idx);
		BUG();
	}
	vif->grant_tx_handle[pending_idx] = handle;
}

static inline void xenvif_grant_handle_reset(struct xenvif *vif,
					     u16 pending_idx)
{
	if (unlikely(vif->grant_tx_handle[pending_idx] ==
		     NETBACK_INVALID_HANDLE)) {
		netdev_err(vif->dev,
			   "Trying to unmap invalid handle! pending_idx: %x\n",
			   pending_idx);
		BUG();
	}
	vif->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE;
}

Wei Liu's avatar
Wei Liu committed
913
914
static int xenvif_tx_check_gop(struct xenvif *vif,
			       struct sk_buff *skb,
915
			       struct gnttab_map_grant_ref **gopp)
Ian Campbell's avatar
Ian Campbell committed
916
{
917
	struct gnttab_map_grant_ref *gop = *gopp;
918
	u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
Ian Campbell's avatar
Ian Campbell committed
919
	struct skb_shared_info *shinfo = skb_shinfo(skb);
920
	struct pending_tx_info *tx_info;
Ian Campbell's avatar
Ian Campbell committed
921
922
	int nr_frags = shinfo->nr_frags;
	int i, err, start;
923
	struct sk_buff *first_skb = NULL;
Ian Campbell's avatar
Ian Campbell committed
924
925
926

	/* Check status of header. */
	err = gop->status;
927
	if (unlikely(err))
Wei Liu's avatar
Wei Liu committed
928
		xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
929
930
	else
		xenvif_grant_handle_set(vif, pending_idx , gop->handle);
Ian Campbell's avatar
Ian Campbell committed
931
932

	/* Skip first skb fragment if it is on same page as header fragment. */
933
	start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
Ian Campbell's avatar
Ian Campbell committed
934

935
check_frags:
Ian Campbell's avatar
Ian Campbell committed
936
937
938
	for (i = start; i < nr_frags; i++) {
		int j, newerr;

939
		pending_idx = frag_get_pending_idx(&shinfo->frags[i]);
940
		tx_info = &vif->pending_tx_info[pending_idx];
Ian Campbell's avatar
Ian Campbell committed
941
942

		/* Check error status: if okay then remember grant handle. */
943
		newerr = (++gop)->status;
944

Ian Campbell's avatar
Ian Campbell committed
945
		if (likely(!newerr)) {
946
			xenvif_grant_handle_set(vif, pending_idx , gop->handle);
Ian Campbell's avatar
Ian Campbell committed
947
948
			/* Had a previous error? Invalidate this fragment. */
			if (unlikely(err))
949
				xenvif_idx_unmap(vif, pending_idx);
Ian Campbell's avatar
Ian Campbell committed
950
951
952
953
			continue;
		}

		/* Error on this fragment: respond to client with an error. */
Wei Liu's avatar
Wei Liu committed
954
		xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
Ian Campbell's avatar
Ian Campbell committed
955
956
957
958
959

		/* Not the first error? Preceding frags already invalidated. */
		if (err)
			continue;
		/* First error: invalidate header and preceding fragments. */
960
961
962
963
		if (!first_skb)
			pending_idx = XENVIF_TX_CB(skb)->pending_idx;
		else
			pending_idx = XENVIF_TX_CB(skb)->pending_idx;
964
		xenvif_idx_unmap(vif, pending_idx);
Ian Campbell's avatar
Ian Campbell committed
965
		for (j = start; j < i; j++) {
966
			pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
967
			xenvif_idx_unmap(vif, pending_idx);
Ian Campbell's avatar
Ian Campbell committed
968
969
970
971
972
973
		}

		/* Remember the error: invalidate all subsequent fragments. */
		err = newerr;
	}

974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
	if (skb_has_frag_list(skb)) {
		first_skb = skb;
		skb = shinfo->frag_list;
		shinfo = skb_shinfo(skb);
		nr_frags = shinfo->nr_frags;
		start = 0;

		goto check_frags;
	}

	/* There was a mapping error in the frag_list skb. We have to unmap
	 * the first skb's frags
	 */
	if (first_skb && err) {
		int j;
		shinfo = skb_shinfo(first_skb);
		pending_idx = XENVIF_TX_CB(skb)->pending_idx;
		start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
		for (j = start; j < shinfo->nr_frags; j++) {
			pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
			xenvif_idx_unmap(vif, pending_idx);
		}
	}

Ian Campbell's avatar
Ian Campbell committed
998
999
1000
1001
	*gopp = gop + 1;
	return err;
}

Wei Liu's avatar
Wei Liu committed
1002
static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
Ian Campbell's avatar
Ian Campbell committed
1003
1004
1005
1006
{
	struct skb_shared_info *shinfo = skb_shinfo(skb);
	int nr_frags = shinfo->nr_frags;
	int i;
1007
1008
1009
1010
	u16 prev_pending_idx = INVALID_PENDING_IDX;

	if (skb_shinfo(skb)->destructor_arg)
		prev_pending_idx = XENVIF_TX_CB(skb)->pending_idx;
Ian Campbell's avatar
Ian Campbell committed
1011
1012
1013
1014

	for (i = 0; i < nr_frags; i++) {
		skb_frag_t *frag = shinfo->frags + i;
		struct xen_netif_tx_request *txp;
1015
1016
		struct page *page;
		u16 pending_idx;
Ian Campbell's avatar
Ian Campbell committed
1017

1018
		pending_idx = frag_get_pending_idx(frag);
Ian Campbell's avatar
Ian Campbell committed
1019

1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
		/* If this is not the first frag, chain it to the previous*/
		if (unlikely(prev_pending_idx == INVALID_PENDING_IDX))
			skb_shinfo(skb)->destructor_arg =
				&vif->pending_tx_info[pending_idx].callback_struct;
		else if (likely(pending_idx != prev_pending_idx))
			vif->pending_tx_info[prev_pending_idx].callback_struct.ctx =
				&(vif->pending_tx_info[pending_idx].callback_struct);

		vif->pending_tx_info[pending_idx].callback_struct.ctx = NULL;
		prev_pending_idx = pending_idx;

1031
1032
		txp = &vif->pending_tx_info[pending_idx].req;
		page = virt_to_page(idx_to_kaddr(vif, pending_idx));
1033
		__skb_fill_page_desc(skb, i, page, txp->offset, txp->size);
Ian Campbell's avatar
Ian Campbell committed
1034
1035
1036
1037
		skb->len += txp->size;
		skb->data_len += txp->size;
		skb->truesize += txp->size;

1038
		/* Take an extra reference to offset network stack's put_page */
1039
		get_page(vif->mmap_pages[pending_idx]);
Ian Campbell's avatar
Ian Campbell committed
1040
	}
1041
1042
1043
1044
1045
1046
	/* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc
	 * overlaps with "index", and "mapping" is not set. I think mapping
	 * should be set. If delivered to local stack, it would drop this
	 * skb in sk_filter unless the socket has the right to use it.
	 */
	skb->pfmemalloc	= false;
Ian Campbell's avatar
Ian Campbell committed
1047
1048
}

Wei Liu's avatar
Wei Liu committed
1049
static int xenvif_get_extras(struct xenvif *vif,
Ian Campbell's avatar
Ian Campbell committed
1050
1051
1052
1053
1054
1055
1056
1057
				struct xen_netif_extra_info *extras,
				int work_to_do)
{
	struct xen_netif_extra_info extra;
	RING_IDX cons = vif->tx.req_cons;

	do {
		if (unlikely(work_to_do-- <= 0)) {
1058
			netdev_err(vif->dev, "Missing extra info\n");
Wei Liu's avatar
Wei Liu committed
1059
			xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1060
1061
1062
1063
1064
1065
1066
1067
			return -EBADR;
		}

		memcpy(&extra, RING_GET_REQUEST(&vif->tx, cons),
		       sizeof(extra));
		if (unlikely(!extra.type ||
			     extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
			vif->tx.req_cons = ++cons;
1068
			netdev_err(vif->dev,
Ian Campbell's avatar
Ian Campbell committed
1069
				   "Invalid extra type: %d\n", extra.type);
Wei Liu's avatar
Wei Liu committed
1070
			xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
			return -EINVAL;
		}

		memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
		vif->tx.req_cons = ++cons;
	} while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);

	return work_to_do;
}

Wei Liu's avatar
Wei Liu committed
1081
1082
1083
static int xenvif_set_skb_gso(struct xenvif *vif,
			      struct sk_buff *skb,
			      struct xen_netif_extra_info *gso)
Ian Campbell's avatar
Ian Campbell committed
1084
1085
{
	if (!gso->u.gso.size) {
1086
		netdev_err(vif->dev, "GSO size must not be zero.\n");
Wei Liu's avatar
Wei Liu committed
1087
		xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1088
1089
1090
		return -EINVAL;
	}

1091
1092
1093
1094
1095
1096
1097
1098
	switch (gso->u.gso.type) {
	case XEN_NETIF_GSO_TYPE_TCPV4:
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
		break;
	case XEN_NETIF_GSO_TYPE_TCPV6:
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
		break;
	default:
1099
		netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type);
Wei Liu's avatar
Wei Liu committed
1100
		xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1101
1102
1103
1104
		return -EINVAL;
	}

	skb_shinfo(skb)->gso_size = gso->u.gso.size;
1105
	/* gso_segs will be calculated later */
Ian Campbell's avatar
Ian Campbell committed
1106
1107
1108
1109

	return 0;
}

1110
1111
static int checksum_setup(struct xenvif *vif, struct sk_buff *skb)
{
1112
	bool recalculate_partial_csum = false;
1113
1114
1115
1116
1117
1118
1119
1120
1121

	/* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
	 * peers can fail to set NETRXF_csum_blank when sending a GSO
	 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
	 * recalculate the partial checksum.
	 */
	if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
		vif->rx_gso_checksum_fixup++;
		skb->ip_summed = CHECKSUM_PARTIAL;
1122
		recalculate_partial_csum = true;
1123
1124
1125
1126
1127
1128
	}

	/* A non-CHECKSUM_PARTIAL SKB does not require setup. */
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

1129
	return skb_checksum_setup(skb, recalculate_partial_csum);
1130
1131
}

Ian Campbell's avatar
Ian Campbell committed
1132
1133
static bool tx_credit_exceeded(struct xenvif *vif, unsigned size)
{
1134
1135
	u64 now = get_jiffies_64();
	u64 next_credit = vif->credit_window_start +
Ian Campbell's avatar
Ian Campbell committed
1136
1137
1138
1139
1140
1141
1142
		msecs_to_jiffies(vif->credit_usec / 1000);

	/* Timer could already be pending in rare cases. */
	if (timer_pending(&vif->credit_timeout))
		return true;

	/* Passed the point where we can replenish credit? */
1143
1144
	if (time_after_eq64(now, next_credit)) {
		vif->credit_window_start = now;
Ian Campbell's avatar
Ian Campbell committed
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
		tx_add_credit(vif);
	}

	/* Still too big to send right now? Set a callback. */
	if (size > vif->remaining_credit) {
		vif->credit_timeout.data     =
			(unsigned long)vif;
		vif->credit_timeout.function =
			tx_credit_callback;
		mod_timer(&vif->credit_timeout,
			  next_credit);
1156
		vif->credit_window_start = next_credit;
Ian Campbell's avatar
Ian Campbell committed
1157
1158
1159
1160
1161
1162
1163

		return true;
	}

	return false;
}

1164
static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
Ian Campbell's avatar
Ian Campbell committed
1165
{
1166
	struct gnttab_map_grant_ref *gop = vif->tx_map_ops, *request_gop;
Ian Campbell's avatar
Ian Campbell committed
1167
1168
1169
	struct sk_buff *skb;
	int ret;

1170
	while (skb_queue_len(&vif->tx_queue) < budget) {
Ian Campbell's avatar
Ian Campbell committed
1171
		struct xen_netif_tx_request txreq;
1172
		struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX];
Ian Campbell's avatar
Ian Campbell committed
1173
1174
1175
1176
1177
1178
1179
		struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1];
		u16 pending_idx;
		RING_IDX idx;
		int work_to_do;
		unsigned int data_len;
		pending_ring_idx_t index;

1180
1181
1182
1183
1184
1185
1186
		if (vif->tx.sring->req_prod - vif->tx.req_cons >
		    XEN_NETIF_TX_RING_SIZE) {
			netdev_err(vif->dev,
				   "Impossible number of requests. "
				   "req_prod %d, req_cons %d, size %ld\n",
				   vif->tx.sring->req_prod, vif->tx.req_cons,
				   XEN_NETIF_TX_RING_SIZE);
Wei Liu's avatar
Wei Liu committed
1187
			xenvif_fatal_tx_err(vif);
1188
1189
1190
			continue;
		}

1191
		work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&vif->tx);
1192
1193
		if (!work_to_do)
			break;
Ian Campbell's avatar
Ian Campbell committed
1194
1195
1196
1197
1198
1199
1200

		idx = vif->tx.req_cons;
		rmb(); /* Ensure that we see the request before we copy it. */
		memcpy(&txreq, RING_GET_REQUEST(&vif->tx, idx), sizeof(txreq));

		/* Credit-based scheduling. */
		if (txreq.size > vif->remaining_credit &&
1201
1202
		    tx_credit_exceeded(vif, txreq.size))
			break;
Ian Campbell's avatar
Ian Campbell committed
1203
1204
1205
1206
1207
1208
1209
1210

		vif->remaining_credit -= txreq.size;

		work_to_do--;
		vif->tx.req_cons = ++idx;

		memset(extras, 0, sizeof(extras));
		if (txreq.flags & XEN_NETTXF_extra_info) {
Wei Liu's avatar
Wei Liu committed
1211
1212
			work_to_do = xenvif_get_extras(vif, extras,
						       work_to_do);
Ian Campbell's avatar
Ian Campbell committed
1213
			idx = vif->tx.req_cons;
1214
			if (unlikely(work_to_do < 0))