netback.c 47.9 KB
Newer Older
Ian Campbell's avatar
Ian Campbell committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/*
 * Back-end of the driver for virtual network devices. This portion of the
 * driver exports a 'unified' network-device interface that can be accessed
 * by any operating system that implements a compatible front end. A
 * reference front-end implementation can be found in:
 *  drivers/net/xen-netfront.c
 *
 * Copyright (c) 2002-2005, K A Fraser
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation; or, when distributed
 * separately from the Linux kernel or incorporated into other
 * software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "common.h"

#include <linux/kthread.h>
#include <linux/if_vlan.h>
#include <linux/udp.h>
40
#include <linux/highmem.h>
Ian Campbell's avatar
Ian Campbell committed
41
42
43

#include <net/tcp.h>

Stefano Stabellini's avatar
Stefano Stabellini committed
44
#include <xen/xen.h>
Ian Campbell's avatar
Ian Campbell committed
45
46
47
48
49
50
#include <xen/events.h>
#include <xen/interface/memory.h>

#include <asm/xen/hypercall.h>
#include <asm/xen/page.h>

51
52
53
54
55
56
57
/* Provide an option to disable split event channels at load time as
 * event channels are limited resource. Split event channels are
 * enabled by default.
 */
bool separate_tx_rx_irq = 1;
module_param(separate_tx_rx_irq, bool, 0644);

58
/* When guest ring is filled up, qdisc queues the packets for us, but we have
59
 * to timeout them, otherwise other guests' packets can get stuck there
60
61
62
63
64
 */
unsigned int rx_drain_timeout_msecs = 10000;
module_param(rx_drain_timeout_msecs, uint, 0444);
unsigned int rx_drain_timeout_jiffies;

65
66
67
68
/*
 * This is the maximum slots a skb can have. If a guest sends a skb
 * which exceeds this limit it is considered malicious.
 */
69
70
71
72
#define FATAL_SKB_SLOTS_DEFAULT 20
static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT;
module_param(fatal_skb_slots, uint, 0444);

Wei Liu's avatar
Wei Liu committed
73
74
75
static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx,
			       u8 status);

Ian Campbell's avatar
Ian Campbell committed
76
77
78
static void make_tx_response(struct xenvif *vif,
			     struct xen_netif_tx_request *txp,
			     s8       st);
79
80
81
82

static inline int tx_work_todo(struct xenvif *vif);
static inline int rx_work_todo(struct xenvif *vif);

Ian Campbell's avatar
Ian Campbell committed
83
84
85
86
87
88
89
static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
					     u16      id,
					     s8       st,
					     u16      offset,
					     u16      size,
					     u16      flags);

90
static inline unsigned long idx_to_pfn(struct xenvif *vif,
91
				       u16 idx)
Ian Campbell's avatar
Ian Campbell committed
92
{
93
	return page_to_pfn(vif->mmap_pages[idx]);
Ian Campbell's avatar
Ian Campbell committed
94
95
}

96
static inline unsigned long idx_to_kaddr(struct xenvif *vif,
97
					 u16 idx)
Ian Campbell's avatar
Ian Campbell committed
98
{
99
	return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif, idx));
Ian Campbell's avatar
Ian Campbell committed
100
101
}

102
103
104
#define callback_param(vif, pending_idx) \
	(vif->pending_tx_info[pending_idx].callback_struct)

105
106
/* Find the containing VIF's structure from a pointer in pending_tx_info array
 */
107
108
static inline struct xenvif* ubuf_to_vif(struct ubuf_info *ubuf)
{
109
110
111
112
113
114
	u16 pending_idx = ubuf->desc;
	struct pending_tx_info *temp =
		container_of(ubuf, struct pending_tx_info, callback_struct);
	return container_of(temp - pending_idx,
			    struct xenvif,
			    pending_tx_info[0]);
115
}
116

117
118
119
120
/* This is a miniumum size for the linear area to avoid lots of
 * calls to __pskb_pull_tail() as we set up checksum offsets. The
 * value 128 was chosen as it covers all IPv4 and most likely
 * IPv6 headers.
Ian Campbell's avatar
Ian Campbell committed
121
 */
122
#define PKT_PROT_LEN 128
Ian Campbell's avatar
Ian Campbell committed
123

124
125
126
127
128
129
130
131
132
133
static u16 frag_get_pending_idx(skb_frag_t *frag)
{
	return (u16)frag->page_offset;
}

static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx)
{
	frag->page_offset = pending_idx;
}

Ian Campbell's avatar
Ian Campbell committed
134
135
136
137
138
static inline pending_ring_idx_t pending_index(unsigned i)
{
	return i & (MAX_PENDING_REQS-1);
}

139
bool xenvif_rx_ring_slots_available(struct xenvif *vif, int needed)
Ian Campbell's avatar
Ian Campbell committed
140
{
141
	RING_IDX prod, cons;
Ian Campbell's avatar
Ian Campbell committed
142

143
144
145
	do {
		prod = vif->rx.sring->req_prod;
		cons = vif->rx.req_cons;
Ian Campbell's avatar
Ian Campbell committed
146

147
148
		if (prod - cons >= needed)
			return true;
Ian Campbell's avatar
Ian Campbell committed
149

150
		vif->rx.sring->req_event = prod + 1;
Ian Campbell's avatar
Ian Campbell committed
151

152
153
154
155
156
		/* Make sure event is visible before we check prod
		 * again.
		 */
		mb();
	} while (vif->rx.sring->req_prod != prod);
Ian Campbell's avatar
Ian Campbell committed
157

158
	return false;
Ian Campbell's avatar
Ian Campbell committed
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
}

/*
 * Returns true if we should start a new receive buffer instead of
 * adding 'size' bytes to a buffer which currently contains 'offset'
 * bytes.
 */
static bool start_new_rx_buffer(int offset, unsigned long size, int head)
{
	/* simple case: we have completely filled the current buffer. */
	if (offset == MAX_BUFFER_OFFSET)
		return true;

	/*
	 * complex case: start a fresh buffer if the current frag
	 * would overflow the current buffer but only if:
	 *     (i)   this frag would fit completely in the next buffer
	 * and (ii)  there is already some data in the current buffer
	 * and (iii) this is not the head buffer.
	 *
	 * Where:
	 * - (i) stops us splitting a frag into two copies
	 *   unless the frag is too large for a single buffer.
	 * - (ii) stops us from leaving a buffer pointlessly empty.
	 * - (iii) stops us leaving the first buffer
	 *   empty. Strictly speaking this is already covered
	 *   by (ii) but is explicitly checked because
	 *   netfront relies on the first buffer being
	 *   non-empty and can crash otherwise.
	 *
	 * This means we will effectively linearise small
	 * frags but do not needlessly split large buffers
	 * into multiple copies tend to give large frags their
	 * own buffers as before.
	 */
	if ((offset + size > MAX_BUFFER_OFFSET) &&
	    (size <= MAX_BUFFER_OFFSET) && offset && !head)
		return true;

	return false;
}

struct netrx_pending_operations {
	unsigned copy_prod, copy_cons;
	unsigned meta_prod, meta_cons;
	struct gnttab_copy *copy;
205
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
206
207
208
209
	int copy_off;
	grant_ref_t copy_gref;
};

210
211
static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif *vif,
						 struct netrx_pending_operations *npo)
Ian Campbell's avatar
Ian Campbell committed
212
{
213
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
214
215
216
217
218
	struct xen_netif_rx_request *req;

	req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);

	meta = npo->meta + npo->meta_prod++;
219
	meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
Ian Campbell's avatar
Ian Campbell committed
220
221
222
223
224
225
226
227
228
229
	meta->gso_size = 0;
	meta->size = 0;
	meta->id = req->id;

	npo->copy_off = 0;
	npo->copy_gref = req->gref;

	return meta;
}

230
231
232
233
/*
 * Set up the grant operations for this fragment. If it's a flipping
 * interface, we also set up the unmap request from here.
 */
Wei Liu's avatar
Wei Liu committed
234
235
236
static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
				 struct netrx_pending_operations *npo,
				 struct page *page, unsigned long size,
237
238
239
				 unsigned long offset, int *head,
				 struct xenvif *foreign_vif,
				 grant_ref_t foreign_gref)
Ian Campbell's avatar
Ian Campbell committed
240
241
{
	struct gnttab_copy *copy_gop;
242
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
243
	unsigned long bytes;
244
	int gso_type = XEN_NETIF_GSO_TYPE_NONE;
Ian Campbell's avatar
Ian Campbell committed
245
246

	/* Data must not cross a page boundary. */
247
	BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
Ian Campbell's avatar
Ian Campbell committed
248
249
250

	meta = npo->meta + npo->meta_prod - 1;

251
252
253
254
	/* Skip unused frames from start of page */
	page += offset >> PAGE_SHIFT;
	offset &= ~PAGE_MASK;

Ian Campbell's avatar
Ian Campbell committed
255
	while (size > 0) {
256
		BUG_ON(offset >= PAGE_SIZE);
Ian Campbell's avatar
Ian Campbell committed
257
258
		BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);

259
260
261
262
263
		bytes = PAGE_SIZE - offset;

		if (bytes > size)
			bytes = size;

264
		if (start_new_rx_buffer(npo->copy_off, bytes, *head)) {
Ian Campbell's avatar
Ian Campbell committed
265
266
267
268
			/*
			 * Netfront requires there to be some data in the head
			 * buffer.
			 */
269
			BUG_ON(*head);
Ian Campbell's avatar
Ian Campbell committed
270
271
272
273
274
275
276
277
278

			meta = get_next_rx_buffer(vif, npo);
		}

		if (npo->copy_off + bytes > MAX_BUFFER_OFFSET)
			bytes = MAX_BUFFER_OFFSET - npo->copy_off;

		copy_gop = npo->copy + npo->copy_prod++;
		copy_gop->flags = GNTCOPY_dest_gref;
279
280
		copy_gop->len = bytes;

281
282
283
284
285
286
287
288
289
		if (foreign_vif) {
			copy_gop->source.domid = foreign_vif->domid;
			copy_gop->source.u.ref = foreign_gref;
			copy_gop->flags |= GNTCOPY_source_gref;
		} else {
			copy_gop->source.domid = DOMID_SELF;
			copy_gop->source.u.gmfn =
				virt_to_mfn(page_address(page));
		}
Ian Campbell's avatar
Ian Campbell committed
290
291
		copy_gop->source.offset = offset;

292
		copy_gop->dest.domid = vif->domid;
Ian Campbell's avatar
Ian Campbell committed
293
294
295
296
297
298
299
300
301
		copy_gop->dest.offset = npo->copy_off;
		copy_gop->dest.u.ref = npo->copy_gref;

		npo->copy_off += bytes;
		meta->size += bytes;

		offset += bytes;
		size -= bytes;

302
303
304
305
306
307
308
		/* Next frame */
		if (offset == PAGE_SIZE && size) {
			BUG_ON(!PageCompound(page));
			page++;
			offset = 0;
		}

Ian Campbell's avatar
Ian Campbell committed
309
		/* Leave a gap for the GSO descriptor. */
310
311
312
313
314
315
		if (skb_is_gso(skb)) {
			if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
				gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
			else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
				gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
		}
316
317

		if (*head && ((1 << gso_type) & vif->gso_mask))
Ian Campbell's avatar
Ian Campbell committed
318
319
			vif->rx.req_cons++;

320
		*head = 0; /* There must be something in this buffer now. */
Ian Campbell's avatar
Ian Campbell committed
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336

	}
}

/*
 * Prepare an SKB to be transmitted to the frontend.
 *
 * This function is responsible for allocating grant operations, meta
 * structures, etc.
 *
 * It returns the number of meta structures consumed. The number of
 * ring slots used is always equal to the number of meta slots used
 * plus the number of GSO descriptors used. Currently, we use either
 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
 * frontend-side LRO).
 */
Wei Liu's avatar
Wei Liu committed
337
338
static int xenvif_gop_skb(struct sk_buff *skb,
			  struct netrx_pending_operations *npo)
Ian Campbell's avatar
Ian Campbell committed
339
340
341
342
343
{
	struct xenvif *vif = netdev_priv(skb->dev);
	int nr_frags = skb_shinfo(skb)->nr_frags;
	int i;
	struct xen_netif_rx_request *req;
344
	struct xenvif_rx_meta *meta;
Ian Campbell's avatar
Ian Campbell committed
345
	unsigned char *data;
346
	int head = 1;
Ian Campbell's avatar
Ian Campbell committed
347
	int old_meta_prod;
348
	int gso_type;
349
350
351
	struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg;
	grant_ref_t foreign_grefs[MAX_SKB_FRAGS];
	struct xenvif *foreign_vif = NULL;
Ian Campbell's avatar
Ian Campbell committed
352
353
354

	old_meta_prod = npo->meta_prod;

355
356
357
358
359
360
	gso_type = XEN_NETIF_GSO_TYPE_NONE;
	if (skb_is_gso(skb)) {
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
			gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
			gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
361
362
	}

Ian Campbell's avatar
Ian Campbell committed
363
	/* Set up a GSO prefix descriptor, if necessary */
364
	if ((1 << gso_type) & vif->gso_prefix_mask) {
Ian Campbell's avatar
Ian Campbell committed
365
366
		req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
		meta = npo->meta + npo->meta_prod++;
367
		meta->gso_type = gso_type;
368
		meta->gso_size = skb_shinfo(skb)->gso_size;
Ian Campbell's avatar
Ian Campbell committed
369
370
371
372
373
374
375
		meta->size = 0;
		meta->id = req->id;
	}

	req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
	meta = npo->meta + npo->meta_prod++;

376
377
	if ((1 << gso_type) & vif->gso_mask) {
		meta->gso_type = gso_type;
378
		meta->gso_size = skb_shinfo(skb)->gso_size;
379
380
	} else {
		meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
Ian Campbell's avatar
Ian Campbell committed
381
		meta->gso_size = 0;
382
	}
Ian Campbell's avatar
Ian Campbell committed
383
384
385
386
387
388

	meta->size = 0;
	meta->id = req->id;
	npo->copy_off = 0;
	npo->copy_gref = req->gref;

389
390
391
392
393
394
395
396
397
398
399
400
401
	if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
		 (ubuf->callback == &xenvif_zerocopy_callback)) {
		int i = 0;
		foreign_vif = ubuf_to_vif(ubuf);

		do {
			u16 pending_idx = ubuf->desc;
			foreign_grefs[i++] =
				foreign_vif->pending_tx_info[pending_idx].req.gref;
			ubuf = (struct ubuf_info *) ubuf->ctx;
		} while (ubuf);
	}

Ian Campbell's avatar
Ian Campbell committed
402
403
404
405
406
407
408
409
	data = skb->data;
	while (data < skb_tail_pointer(skb)) {
		unsigned int offset = offset_in_page(data);
		unsigned int len = PAGE_SIZE - offset;

		if (data + len > skb_tail_pointer(skb))
			len = skb_tail_pointer(skb) - data;

Wei Liu's avatar
Wei Liu committed
410
		xenvif_gop_frag_copy(vif, skb, npo,
411
412
413
				     virt_to_page(data), len, offset, &head,
				     NULL,
				     0);
Ian Campbell's avatar
Ian Campbell committed
414
415
416
417
		data += len;
	}

	for (i = 0; i < nr_frags; i++) {
Wei Liu's avatar
Wei Liu committed
418
419
420
421
		xenvif_gop_frag_copy(vif, skb, npo,
				     skb_frag_page(&skb_shinfo(skb)->frags[i]),
				     skb_frag_size(&skb_shinfo(skb)->frags[i]),
				     skb_shinfo(skb)->frags[i].page_offset,
422
423
424
				     &head,
				     foreign_vif,
				     foreign_grefs[i]);
Ian Campbell's avatar
Ian Campbell committed
425
426
427
428
429
430
	}

	return npo->meta_prod - old_meta_prod;
}

/*
Wei Liu's avatar
Wei Liu committed
431
 * This is a twin to xenvif_gop_skb.  Assume that xenvif_gop_skb was
Ian Campbell's avatar
Ian Campbell committed
432
433
434
435
 * used to set up the operations on the top of
 * netrx_pending_operations, which have since been done.  Check that
 * they didn't give any errors and advance over them.
 */
Wei Liu's avatar
Wei Liu committed
436
437
static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
			    struct netrx_pending_operations *npo)
Ian Campbell's avatar
Ian Campbell committed
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
{
	struct gnttab_copy     *copy_op;
	int status = XEN_NETIF_RSP_OKAY;
	int i;

	for (i = 0; i < nr_meta_slots; i++) {
		copy_op = npo->copy + npo->copy_cons++;
		if (copy_op->status != GNTST_okay) {
			netdev_dbg(vif->dev,
				   "Bad status %d from copy to DOM%d.\n",
				   copy_op->status, vif->domid);
			status = XEN_NETIF_RSP_ERROR;
		}
	}

	return status;
}

Wei Liu's avatar
Wei Liu committed
456
457
458
static void xenvif_add_frag_responses(struct xenvif *vif, int status,
				      struct xenvif_rx_meta *meta,
				      int nr_meta_slots)
Ian Campbell's avatar
Ian Campbell committed
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
{
	int i;
	unsigned long offset;

	/* No fragments used */
	if (nr_meta_slots <= 1)
		return;

	nr_meta_slots--;

	for (i = 0; i < nr_meta_slots; i++) {
		int flags;
		if (i == nr_meta_slots - 1)
			flags = 0;
		else
			flags = XEN_NETRXF_more_data;

		offset = 0;
		make_rx_response(vif, meta[i].id, status, offset,
				 meta[i].size, flags);
	}
}

482
struct xenvif_rx_cb {
483
484
485
	int meta_slots_used;
};

486
487
#define XENVIF_RX_CB(skb) ((struct xenvif_rx_cb *)(skb)->cb)

488
void xenvif_kick_thread(struct xenvif *vif)
489
490
491
492
{
	wake_up(&vif->wq);
}

493
static void xenvif_rx_action(struct xenvif *vif)
Ian Campbell's avatar
Ian Campbell committed
494
495
{
	s8 status;
496
	u16 flags;
Ian Campbell's avatar
Ian Campbell committed
497
498
499
500
501
502
	struct xen_netif_rx_response *resp;
	struct sk_buff_head rxq;
	struct sk_buff *skb;
	LIST_HEAD(notify);
	int ret;
	unsigned long offset;
503
	bool need_to_notify = false;
Ian Campbell's avatar
Ian Campbell committed
504
505

	struct netrx_pending_operations npo = {
506
507
		.copy  = vif->grant_copy_op,
		.meta  = vif->meta,
Ian Campbell's avatar
Ian Campbell committed
508
509
510
511
	};

	skb_queue_head_init(&rxq);

512
	while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) {
513
		RING_IDX max_slots_needed;
514
515
516
517
518
519
520
521
522
523
524
525
526
527
		int i;

		/* We need a cheap worse case estimate for the number of
		 * slots we'll use.
		 */

		max_slots_needed = DIV_ROUND_UP(offset_in_page(skb->data) +
						skb_headlen(skb),
						PAGE_SIZE);
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
			unsigned int size;
			size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
			max_slots_needed += DIV_ROUND_UP(size, PAGE_SIZE);
		}
528
529
530
		if (skb_is_gso(skb) &&
		   (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 ||
		    skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
531
532
533
534
535
			max_slots_needed++;

		/* If the skb may not fit then bail out now */
		if (!xenvif_rx_ring_slots_available(vif, max_slots_needed)) {
			skb_queue_head(&vif->rx_queue, skb);
536
			need_to_notify = true;
537
			vif->rx_last_skb_slots = max_slots_needed;
538
			break;
539
540
		} else
			vif->rx_last_skb_slots = 0;
Ian Campbell's avatar
Ian Campbell committed
541

542
543
		XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo);
		BUG_ON(XENVIF_RX_CB(skb)->meta_slots_used > max_slots_needed);
Ian Campbell's avatar
Ian Campbell committed
544
545
546
547

		__skb_queue_tail(&rxq, skb);
	}

548
	BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta));
Ian Campbell's avatar
Ian Campbell committed
549
550

	if (!npo.copy_prod)
551
		goto done;
Ian Campbell's avatar
Ian Campbell committed
552

553
	BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS);
554
	gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod);
Ian Campbell's avatar
Ian Campbell committed
555
556
557

	while ((skb = __skb_dequeue(&rxq)) != NULL) {

558
559
		if ((1 << vif->meta[npo.meta_cons].gso_type) &
		    vif->gso_prefix_mask) {
Ian Campbell's avatar
Ian Campbell committed
560
			resp = RING_GET_RESPONSE(&vif->rx,
561
						 vif->rx.rsp_prod_pvt++);
Ian Campbell's avatar
Ian Campbell committed
562
563
564

			resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;

565
566
			resp->offset = vif->meta[npo.meta_cons].gso_size;
			resp->id = vif->meta[npo.meta_cons].id;
567
			resp->status = XENVIF_RX_CB(skb)->meta_slots_used;
Ian Campbell's avatar
Ian Campbell committed
568
569

			npo.meta_cons++;
570
			XENVIF_RX_CB(skb)->meta_slots_used--;
Ian Campbell's avatar
Ian Campbell committed
571
572
573
574
575
576
		}


		vif->dev->stats.tx_bytes += skb->len;
		vif->dev->stats.tx_packets++;

577
578
579
		status = xenvif_check_gop(vif,
					  XENVIF_RX_CB(skb)->meta_slots_used,
					  &npo);
Ian Campbell's avatar
Ian Campbell committed
580

581
		if (XENVIF_RX_CB(skb)->meta_slots_used == 1)
Ian Campbell's avatar
Ian Campbell committed
582
583
584
585
586
587
588
589
590
591
592
			flags = 0;
		else
			flags = XEN_NETRXF_more_data;

		if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */
			flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated;
		else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
			/* remote but checksummed. */
			flags |= XEN_NETRXF_data_validated;

		offset = 0;
593
		resp = make_rx_response(vif, vif->meta[npo.meta_cons].id,
Ian Campbell's avatar
Ian Campbell committed
594
					status, offset,
595
					vif->meta[npo.meta_cons].size,
Ian Campbell's avatar
Ian Campbell committed
596
597
					flags);

598
599
		if ((1 << vif->meta[npo.meta_cons].gso_type) &
		    vif->gso_mask) {
Ian Campbell's avatar
Ian Campbell committed
600
601
602
603
604
605
606
			struct xen_netif_extra_info *gso =
				(struct xen_netif_extra_info *)
				RING_GET_RESPONSE(&vif->rx,
						  vif->rx.rsp_prod_pvt++);

			resp->flags |= XEN_NETRXF_extra_info;

607
			gso->u.gso.type = vif->meta[npo.meta_cons].gso_type;
608
			gso->u.gso.size = vif->meta[npo.meta_cons].gso_size;
Ian Campbell's avatar
Ian Campbell committed
609
610
611
612
613
614
615
			gso->u.gso.pad = 0;
			gso->u.gso.features = 0;

			gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
			gso->flags = 0;
		}

Wei Liu's avatar
Wei Liu committed
616
617
		xenvif_add_frag_responses(vif, status,
					  vif->meta + npo.meta_cons + 1,
618
					  XENVIF_RX_CB(skb)->meta_slots_used);
Ian Campbell's avatar
Ian Campbell committed
619
620
621

		RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret);

622
		need_to_notify |= !!ret;
623

624
		npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used;
Ian Campbell's avatar
Ian Campbell committed
625
626
627
		dev_kfree_skb(skb);
	}

628
done:
629
	if (need_to_notify)
630
		notify_remote_via_irq(vif->rx_irq);
Ian Campbell's avatar
Ian Campbell committed
631
632
}

Wei Liu's avatar
Wei Liu committed
633
void xenvif_check_rx_xenvif(struct xenvif *vif)
Ian Campbell's avatar
Ian Campbell committed
634
635
636
637
638
639
{
	int more_to_do;

	RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do);

	if (more_to_do)
640
		napi_schedule(&vif->napi);
Ian Campbell's avatar
Ian Campbell committed
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
}

static void tx_add_credit(struct xenvif *vif)
{
	unsigned long max_burst, max_credit;

	/*
	 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
	 * Otherwise the interface can seize up due to insufficient credit.
	 */
	max_burst = RING_GET_REQUEST(&vif->tx, vif->tx.req_cons)->size;
	max_burst = min(max_burst, 131072UL);
	max_burst = max(max_burst, vif->credit_bytes);

	/* Take care that adding a new chunk of credit doesn't wrap to zero. */
	max_credit = vif->remaining_credit + vif->credit_bytes;
	if (max_credit < vif->remaining_credit)
		max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */

	vif->remaining_credit = min(max_credit, max_burst);
}

static void tx_credit_callback(unsigned long data)
{
	struct xenvif *vif = (struct xenvif *)data;
	tx_add_credit(vif);
Wei Liu's avatar
Wei Liu committed
667
	xenvif_check_rx_xenvif(vif);
Ian Campbell's avatar
Ian Campbell committed
668
669
}

Wei Liu's avatar
Wei Liu committed
670
671
static void xenvif_tx_err(struct xenvif *vif,
			  struct xen_netif_tx_request *txp, RING_IDX end)
Ian Campbell's avatar
Ian Campbell committed
672
673
{
	RING_IDX cons = vif->tx.req_cons;
674
	unsigned long flags;
Ian Campbell's avatar
Ian Campbell committed
675
676

	do {
677
		spin_lock_irqsave(&vif->response_lock, flags);
Ian Campbell's avatar
Ian Campbell committed
678
		make_tx_response(vif, txp, XEN_NETIF_RSP_ERROR);
679
		spin_unlock_irqrestore(&vif->response_lock, flags);
680
		if (cons == end)
Ian Campbell's avatar
Ian Campbell committed
681
682
683
684
685
686
			break;
		txp = RING_GET_REQUEST(&vif->tx, cons++);
	} while (1);
	vif->tx.req_cons = cons;
}

Wei Liu's avatar
Wei Liu committed
687
static void xenvif_fatal_tx_err(struct xenvif *vif)
688
689
690
691
692
{
	netdev_err(vif->dev, "fatal error; disabling device\n");
	xenvif_carrier_off(vif);
}

Wei Liu's avatar
Wei Liu committed
693
694
695
696
static int xenvif_count_requests(struct xenvif *vif,
				 struct xen_netif_tx_request *first,
				 struct xen_netif_tx_request *txp,
				 int work_to_do)
Ian Campbell's avatar
Ian Campbell committed
697
698
{
	RING_IDX cons = vif->tx.req_cons;
699
700
	int slots = 0;
	int drop_err = 0;
701
	int more_data;
Ian Campbell's avatar
Ian Campbell committed
702
703
704
705
706

	if (!(first->flags & XEN_NETTXF_more_data))
		return 0;

	do {
707
708
		struct xen_netif_tx_request dropped_tx = { 0 };

709
710
711
712
		if (slots >= work_to_do) {
			netdev_err(vif->dev,
				   "Asked for %d slots but exceeds this limit\n",
				   work_to_do);
Wei Liu's avatar
Wei Liu committed
713
			xenvif_fatal_tx_err(vif);
714
			return -ENODATA;
Ian Campbell's avatar
Ian Campbell committed
715
716
		}

717
718
719
		/* This guest is really using too many slots and
		 * considered malicious.
		 */
720
		if (unlikely(slots >= fatal_skb_slots)) {
721
722
			netdev_err(vif->dev,
				   "Malicious frontend using %d slots, threshold %u\n",
723
				   slots, fatal_skb_slots);
Wei Liu's avatar
Wei Liu committed
724
			xenvif_fatal_tx_err(vif);
725
			return -E2BIG;
Ian Campbell's avatar
Ian Campbell committed
726
727
		}

728
		/* Xen network protocol had implicit dependency on
729
730
731
732
733
		 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
		 * the historical MAX_SKB_FRAGS value 18 to honor the
		 * same behavior as before. Any packet using more than
		 * 18 slots but less than fatal_skb_slots slots is
		 * dropped
734
		 */
735
		if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) {
736
737
738
			if (net_ratelimit())
				netdev_dbg(vif->dev,
					   "Too many slots (%d) exceeding limit (%d), dropping packet\n",
739
					   slots, XEN_NETBK_LEGACY_SLOTS_MAX);
740
741
742
			drop_err = -E2BIG;
		}

743
744
745
		if (drop_err)
			txp = &dropped_tx;

746
		memcpy(txp, RING_GET_REQUEST(&vif->tx, cons + slots),
Ian Campbell's avatar
Ian Campbell committed
747
		       sizeof(*txp));
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763

		/* If the guest submitted a frame >= 64 KiB then
		 * first->size overflowed and following slots will
		 * appear to be larger than the frame.
		 *
		 * This cannot be fatal error as there are buggy
		 * frontends that do this.
		 *
		 * Consume all slots and drop the packet.
		 */
		if (!drop_err && txp->size > first->size) {
			if (net_ratelimit())
				netdev_dbg(vif->dev,
					   "Invalid tx request, slot size %u > remaining size %u\n",
					   txp->size, first->size);
			drop_err = -EIO;
Ian Campbell's avatar
Ian Campbell committed
764
765
766
		}

		first->size -= txp->size;
767
		slots++;
Ian Campbell's avatar
Ian Campbell committed
768
769

		if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
770
			netdev_err(vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n",
Ian Campbell's avatar
Ian Campbell committed
771
				 txp->offset, txp->size);
Wei Liu's avatar
Wei Liu committed
772
			xenvif_fatal_tx_err(vif);
773
			return -EINVAL;
Ian Campbell's avatar
Ian Campbell committed
774
		}
775
776
777
778
779
780
781

		more_data = txp->flags & XEN_NETTXF_more_data;

		if (!drop_err)
			txp++;

	} while (more_data);
782
783

	if (drop_err) {
Wei Liu's avatar
Wei Liu committed
784
		xenvif_tx_err(vif, first, cons + slots);
785
786
787
788
		return drop_err;
	}

	return slots;
Ian Campbell's avatar
Ian Campbell committed
789
790
}

791
792
793
794
795
796
797

struct xenvif_tx_cb {
	u16 pending_idx;
};

#define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)

798
799
800
801
802
803
804
805
806
807
808
809
810
811
static inline void xenvif_tx_create_gop(struct xenvif *vif,
					u16 pending_idx,
					struct xen_netif_tx_request *txp,
					struct gnttab_map_grant_ref *gop)
{
	vif->pages_to_map[gop-vif->tx_map_ops] = vif->mmap_pages[pending_idx];
	gnttab_set_map_op(gop, idx_to_kaddr(vif, pending_idx),
			  GNTMAP_host_map | GNTMAP_readonly,
			  txp->gref, vif->domid);

	memcpy(&vif->pending_tx_info[pending_idx].req, txp,
	       sizeof(*txp));
}

812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
static inline struct sk_buff *xenvif_alloc_skb(unsigned int size)
{
	struct sk_buff *skb =
		alloc_skb(size + NET_SKB_PAD + NET_IP_ALIGN,
			  GFP_ATOMIC | __GFP_NOWARN);
	if (unlikely(skb == NULL))
		return NULL;

	/* Packets passed to netif_rx() must have some headroom. */
	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);

	/* Initialize it here to avoid later surprises */
	skb_shinfo(skb)->destructor_arg = NULL;

	return skb;
}

829
830
831
832
static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
							struct sk_buff *skb,
							struct xen_netif_tx_request *txp,
							struct gnttab_map_grant_ref *gop)
Ian Campbell's avatar
Ian Campbell committed
833
834
835
{
	struct skb_shared_info *shinfo = skb_shinfo(skb);
	skb_frag_t *frags = shinfo->frags;
836
	u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
837
838
	int start;
	pending_ring_idx_t index;
839
	unsigned int nr_slots, frag_overflow = 0;
840
841

	/* At this point shinfo->nr_frags is in fact the number of
842
	 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
843
	 */
844
845
846
847
848
	if (shinfo->nr_frags > MAX_SKB_FRAGS) {
		frag_overflow = shinfo->nr_frags - MAX_SKB_FRAGS;
		BUG_ON(frag_overflow > MAX_SKB_FRAGS);
		shinfo->nr_frags = MAX_SKB_FRAGS;
	}
849
	nr_slots = shinfo->nr_frags;
Ian Campbell's avatar
Ian Campbell committed
850
851

	/* Skip first skb fragment if it is on same page as header fragment. */
852
	start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
Ian Campbell's avatar
Ian Campbell committed
853

854
855
	for (shinfo->nr_frags = start; shinfo->nr_frags < nr_slots;
	     shinfo->nr_frags++, txp++, gop++) {
856
857
		index = pending_index(vif->pending_cons++);
		pending_idx = vif->pending_ring[index];
858
859
		xenvif_tx_create_gop(vif, pending_idx, txp, gop);
		frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx);
Ian Campbell's avatar
Ian Campbell committed
860
861
	}

862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
	if (frag_overflow) {
		struct sk_buff *nskb = xenvif_alloc_skb(0);
		if (unlikely(nskb == NULL)) {
			if (net_ratelimit())
				netdev_err(vif->dev,
					   "Can't allocate the frag_list skb.\n");
			return NULL;
		}

		shinfo = skb_shinfo(nskb);
		frags = shinfo->frags;

		for (shinfo->nr_frags = 0; shinfo->nr_frags < frag_overflow;
		     shinfo->nr_frags++, txp++, gop++) {
			index = pending_index(vif->pending_cons++);
			pending_idx = vif->pending_ring[index];
			xenvif_tx_create_gop(vif, pending_idx, txp, gop);
			frag_set_pending_idx(&frags[shinfo->nr_frags],
					     pending_idx);
		}

		skb_shinfo(skb)->frag_list = nskb;
	}
885

Ian Campbell's avatar
Ian Campbell committed
886
887
888
	return gop;
}

889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
static inline void xenvif_grant_handle_set(struct xenvif *vif,
					   u16 pending_idx,
					   grant_handle_t handle)
{
	if (unlikely(vif->grant_tx_handle[pending_idx] !=
		     NETBACK_INVALID_HANDLE)) {
		netdev_err(vif->dev,
			   "Trying to overwrite active handle! pending_idx: %x\n",
			   pending_idx);
		BUG();
	}
	vif->grant_tx_handle[pending_idx] = handle;
}

static inline void xenvif_grant_handle_reset(struct xenvif *vif,
					     u16 pending_idx)
{
	if (unlikely(vif->grant_tx_handle[pending_idx] ==
		     NETBACK_INVALID_HANDLE)) {
		netdev_err(vif->dev,
			   "Trying to unmap invalid handle! pending_idx: %x\n",
			   pending_idx);
		BUG();
	}
	vif->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE;
}

Wei Liu's avatar
Wei Liu committed
916
917
static int xenvif_tx_check_gop(struct xenvif *vif,
			       struct sk_buff *skb,
918
			       struct gnttab_map_grant_ref **gopp)
Ian Campbell's avatar
Ian Campbell committed
919
{
920
	struct gnttab_map_grant_ref *gop = *gopp;
921
	u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
Ian Campbell's avatar
Ian Campbell committed
922
	struct skb_shared_info *shinfo = skb_shinfo(skb);
923
	struct pending_tx_info *tx_info;
Ian Campbell's avatar
Ian Campbell committed
924
925
	int nr_frags = shinfo->nr_frags;
	int i, err, start;
926
	struct sk_buff *first_skb = NULL;
Ian Campbell's avatar
Ian Campbell committed
927
928
929

	/* Check status of header. */
	err = gop->status;
930
	if (unlikely(err))
Wei Liu's avatar
Wei Liu committed
931
		xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
932
933
	else
		xenvif_grant_handle_set(vif, pending_idx , gop->handle);
Ian Campbell's avatar
Ian Campbell committed
934
935

	/* Skip first skb fragment if it is on same page as header fragment. */
936
	start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
Ian Campbell's avatar
Ian Campbell committed
937

938
check_frags:
Ian Campbell's avatar
Ian Campbell committed
939
940
941
	for (i = start; i < nr_frags; i++) {
		int j, newerr;

942
		pending_idx = frag_get_pending_idx(&shinfo->frags[i]);
943
		tx_info = &vif->pending_tx_info[pending_idx];
Ian Campbell's avatar
Ian Campbell committed
944
945

		/* Check error status: if okay then remember grant handle. */
946
		newerr = (++gop)->status;
947

Ian Campbell's avatar
Ian Campbell committed
948
		if (likely(!newerr)) {
949
			xenvif_grant_handle_set(vif, pending_idx , gop->handle);
Ian Campbell's avatar
Ian Campbell committed
950
951
			/* Had a previous error? Invalidate this fragment. */
			if (unlikely(err))
952
				xenvif_idx_unmap(vif, pending_idx);
Ian Campbell's avatar
Ian Campbell committed
953
954
955
956
			continue;
		}

		/* Error on this fragment: respond to client with an error. */
Wei Liu's avatar
Wei Liu committed
957
		xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
Ian Campbell's avatar
Ian Campbell committed
958
959
960
961
962

		/* Not the first error? Preceding frags already invalidated. */
		if (err)
			continue;
		/* First error: invalidate header and preceding fragments. */
963
964
965
966
		if (!first_skb)
			pending_idx = XENVIF_TX_CB(skb)->pending_idx;
		else
			pending_idx = XENVIF_TX_CB(skb)->pending_idx;
967
		xenvif_idx_unmap(vif, pending_idx);
Ian Campbell's avatar
Ian Campbell committed
968
		for (j = start; j < i; j++) {
969
			pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
970
			xenvif_idx_unmap(vif, pending_idx);
Ian Campbell's avatar
Ian Campbell committed
971
972
973
974
975
976
		}

		/* Remember the error: invalidate all subsequent fragments. */
		err = newerr;
	}

977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
	if (skb_has_frag_list(skb)) {
		first_skb = skb;
		skb = shinfo->frag_list;
		shinfo = skb_shinfo(skb);
		nr_frags = shinfo->nr_frags;
		start = 0;

		goto check_frags;
	}

	/* There was a mapping error in the frag_list skb. We have to unmap
	 * the first skb's frags
	 */
	if (first_skb && err) {
		int j;
		shinfo = skb_shinfo(first_skb);
		pending_idx = XENVIF_TX_CB(skb)->pending_idx;
		start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
		for (j = start; j < shinfo->nr_frags; j++) {
			pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
			xenvif_idx_unmap(vif, pending_idx);
		}
	}

Ian Campbell's avatar
Ian Campbell committed
1001
1002
1003
1004
	*gopp = gop + 1;
	return err;
}

Wei Liu's avatar
Wei Liu committed
1005
static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
Ian Campbell's avatar
Ian Campbell committed
1006
1007
1008
1009
{
	struct skb_shared_info *shinfo = skb_shinfo(skb);
	int nr_frags = shinfo->nr_frags;
	int i;
1010
1011
1012
1013
	u16 prev_pending_idx = INVALID_PENDING_IDX;

	if (skb_shinfo(skb)->destructor_arg)
		prev_pending_idx = XENVIF_TX_CB(skb)->pending_idx;
Ian Campbell's avatar
Ian Campbell committed
1014
1015
1016
1017

	for (i = 0; i < nr_frags; i++) {
		skb_frag_t *frag = shinfo->frags + i;
		struct xen_netif_tx_request *txp;
1018
1019
		struct page *page;
		u16 pending_idx;
Ian Campbell's avatar
Ian Campbell committed
1020

1021
		pending_idx = frag_get_pending_idx(frag);
Ian Campbell's avatar
Ian Campbell committed
1022

1023
1024
1025
		/* If this is not the first frag, chain it to the previous*/
		if (unlikely(prev_pending_idx == INVALID_PENDING_IDX))
			skb_shinfo(skb)->destructor_arg =
1026
				&callback_param(vif, pending_idx);
1027
		else if (likely(pending_idx != prev_pending_idx))
1028
1029
			callback_param(vif, prev_pending_idx).ctx =
				&callback_param(vif, pending_idx);
1030

1031
		callback_param(vif, pending_idx).ctx = NULL;
1032
1033
		prev_pending_idx = pending_idx;

1034
1035
		txp = &vif->pending_tx_info[pending_idx].req;
		page = virt_to_page(idx_to_kaddr(vif, pending_idx));
1036
		__skb_fill_page_desc(skb, i, page, txp->offset, txp->size);
Ian Campbell's avatar
Ian Campbell committed
1037
1038
1039
1040
		skb->len += txp->size;
		skb->data_len += txp->size;
		skb->truesize += txp->size;

1041
		/* Take an extra reference to offset network stack's put_page */
1042
		get_page(vif->mmap_pages[pending_idx]);
Ian Campbell's avatar
Ian Campbell committed
1043
	}
1044
1045
1046
1047
1048
1049
	/* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc
	 * overlaps with "index", and "mapping" is not set. I think mapping
	 * should be set. If delivered to local stack, it would drop this
	 * skb in sk_filter unless the socket has the right to use it.
	 */
	skb->pfmemalloc	= false;
Ian Campbell's avatar
Ian Campbell committed
1050
1051
}

Wei Liu's avatar
Wei Liu committed
1052
static int xenvif_get_extras(struct xenvif *vif,
Ian Campbell's avatar
Ian Campbell committed
1053
1054
1055
1056
1057
1058
1059
1060
				struct xen_netif_extra_info *extras,
				int work_to_do)
{
	struct xen_netif_extra_info extra;
	RING_IDX cons = vif->tx.req_cons;

	do {
		if (unlikely(work_to_do-- <= 0)) {
1061
			netdev_err(vif->dev, "Missing extra info\n");
Wei Liu's avatar
Wei Liu committed
1062
			xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1063
1064
1065
1066
1067
1068
1069
1070
			return -EBADR;
		}

		memcpy(&extra, RING_GET_REQUEST(&vif->tx, cons),
		       sizeof(extra));
		if (unlikely(!extra.type ||
			     extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
			vif->tx.req_cons = ++cons;
1071
			netdev_err(vif->dev,
Ian Campbell's avatar
Ian Campbell committed
1072
				   "Invalid extra type: %d\n", extra.type);
Wei Liu's avatar
Wei Liu committed
1073
			xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
			return -EINVAL;
		}

		memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
		vif->tx.req_cons = ++cons;
	} while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);

	return work_to_do;
}

Wei Liu's avatar
Wei Liu committed
1084
1085
1086
static int xenvif_set_skb_gso(struct xenvif *vif,
			      struct sk_buff *skb,
			      struct xen_netif_extra_info *gso)
Ian Campbell's avatar
Ian Campbell committed
1087
1088
{
	if (!gso->u.gso.size) {
1089
		netdev_err(vif->dev, "GSO size must not be zero.\n");
Wei Liu's avatar
Wei Liu committed
1090
		xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1091
1092
1093
		return -EINVAL;
	}

1094
1095
1096
1097
1098
1099
1100
1101
	switch (gso->u.gso.type) {
	case XEN_NETIF_GSO_TYPE_TCPV4:
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
		break;
	case XEN_NETIF_GSO_TYPE_TCPV6:
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
		break;
	default:
1102
		netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type);
Wei Liu's avatar
Wei Liu committed
1103
		xenvif_fatal_tx_err(vif);
Ian Campbell's avatar
Ian Campbell committed
1104
1105
1106
1107
		return -EINVAL;
	}

	skb_shinfo(skb)->gso_size = gso->u.gso.size;
1108
	/* gso_segs will be calculated later */
Ian Campbell's avatar
Ian Campbell committed
1109
1110
1111
1112

	return 0;
}

1113
1114
static int checksum_setup(struct xenvif *vif, struct sk_buff *skb)
{
1115
	bool recalculate_partial_csum = false;
1116
1117
1118
1119
1120
1121
1122
1123
1124

	/* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
	 * peers can fail to set NETRXF_csum_blank when sending a GSO
	 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
	 * recalculate the partial checksum.
	 */
	if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
		vif->rx_gso_checksum_fixup++;
		skb->ip_summed = CHECKSUM_PARTIAL;
1125
		recalculate_partial_csum = true;
1126
1127
1128
1129
1130
1131
	}

	/* A non-CHECKSUM_PARTIAL SKB does not require setup. */
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

1132
	return skb_checksum_setup(skb, recalculate_partial_csum);
1133
1134
}

Ian Campbell's avatar
Ian Campbell committed
1135
1136
static bool tx_credit_exceeded(struct xenvif *vif, unsigned size)
{
1137
1138
	u64 now = get_jiffies_64();
	u64 next_credit = vif->credit_window_start +
Ian Campbell's avatar
Ian Campbell committed
1139
1140
1141
1142
1143
1144
1145
		msecs_to_jiffies(vif->credit_usec / 1000);

	/* Timer could already be pending in rare cases. */
	if (timer_pending(&vif->credit_timeout))
		return true;

	/* Passed the point where we can replenish credit? */
1146
1147
	if (time_after_eq64(now, next_credit)) {
		vif->credit_window_start = now;
Ian Campbell's avatar
Ian Campbell committed
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
		tx_add_credit(vif);
	}

	/* Still too big to send right now? Set a callback. */
	if (size > vif->remaining_credit) {
		vif->credit_timeout.data     =
			(unsigned long)vif;
		vif->credit_timeout.function =
			tx_credit_callback;
		mod_timer(&vif->credit_timeout,
			  next_credit);
1159
		vif->credit_window_start = next_credit;
Ian Campbell's avatar
Ian Campbell committed
1160
1161
1162
1163
1164
1165
1166

		return true;
	}

	return false;
}

1167
static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
Ian Campbell's avatar
Ian Campbell committed
1168
{
1169
	struct gnttab_map_grant_ref *gop = vif->tx_map_ops, *request_gop;
Ian Campbell's avatar
Ian Campbell committed
1170
1171
1172
	struct sk_buff *skb;
	int ret;

1173
	while (skb_queue_len(&vif->tx_queue) < budget) {
Ian Campbell's avatar
Ian Campbell committed
1174
		struct xen_netif_tx_request txreq;
1175
		struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX];
Ian Campbell's avatar
Ian Campbell committed
1176
1177
1178
1179
1180
1181
1182
		struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1];
		u16 pending_idx;
		RING_IDX idx;
		int work_to_do;
		unsigned int data_len;
		pending_ring_idx_t index;

1183
1184
1185
1186
1187
1188
1189
		if (vif->tx.sring->req_prod - vif->tx.req_cons >
		    XEN_NETIF_TX_RING_SIZE) {
			netdev_err(vif->dev,
				   "Impossible number of requests. "
				   "req_prod %d, req_cons %d, size %ld\n",
				   vif->tx.sring->req_prod, vif->tx.req_cons,
				   XEN_NETIF_TX_RING_SIZE);
Wei Liu's avatar
Wei Liu committed
1190
			xenvif_fatal_tx_err(vif);
1191
1192
1193
			continue;
		}

1194
		work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&vif->tx);
1195
1196
		if (!work_to_do)
			break;
Ian Campbell's avatar
Ian Campbell committed
1197
1198
1199
1200
1201
1202
1203

		idx = vif->tx.req_cons;
		rmb(); /* Ensure that we see the request before we copy it. */
		memcpy(&txreq, RING_GET_REQUEST(&vif->tx, idx), sizeof(txreq));

		/* Credit-based scheduling. */
		if (txreq.size > vif->remaining_credit &&
1204
1205
		    tx_credit_exceeded(vif, txreq.size))
			break;