dpaa2-eth.c 76.4 KB
Newer Older
1
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2
3
4
5
6
7
8
9
10
11
12
/* Copyright 2014-2016 Freescale Semiconductor Inc.
 * Copyright 2016-2017 NXP
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/etherdevice.h>
#include <linux/of_net.h>
#include <linux/interrupt.h>
#include <linux/msi.h>
#include <linux/kthread.h>
13
#include <linux/iommu.h>
14
#include <linux/net_tstamp.h>
15
#include <linux/fsl/mc.h>
16
17
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
18
19
#include <net/sock.h>

20
21
#include "dpaa2-eth.h"

22
23
24
25
26
27
/* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files
 * using trace events only need to #include <trace/events/sched.h>
 */
#define CREATE_TRACE_POINTS
#include "dpaa2-eth-trace.h"

28
29
30
31
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Freescale Semiconductor, Inc");
MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver");

32
33
34
35
36
37
38
39
40
41
static void *dpaa2_iova_to_virt(struct iommu_domain *domain,
				dma_addr_t iova_addr)
{
	phys_addr_t phys_addr;

	phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;

	return phys_to_virt(phys_addr);
}

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
static void validate_rx_csum(struct dpaa2_eth_priv *priv,
			     u32 fd_status,
			     struct sk_buff *skb)
{
	skb_checksum_none_assert(skb);

	/* HW checksum validation is disabled, nothing to do here */
	if (!(priv->net_dev->features & NETIF_F_RXCSUM))
		return;

	/* Read checksum validation bits */
	if (!((fd_status & DPAA2_FAS_L3CV) &&
	      (fd_status & DPAA2_FAS_L4CV)))
		return;

	/* Inform the stack there's no need to compute L3/L4 csum anymore */
	skb->ip_summed = CHECKSUM_UNNECESSARY;
}

/* Free a received FD.
 * Not to be used for Tx conf FDs or on any other paths.
 */
static void free_rx_fd(struct dpaa2_eth_priv *priv,
		       const struct dpaa2_fd *fd,
		       void *vaddr)
{
	struct device *dev = priv->net_dev->dev.parent;
	dma_addr_t addr = dpaa2_fd_get_addr(fd);
	u8 fd_format = dpaa2_fd_get_format(fd);
	struct dpaa2_sg_entry *sgt;
	void *sg_vaddr;
	int i;

	/* If single buffer frame, just free the data buffer */
	if (fd_format == dpaa2_fd_single)
		goto free_buf;
	else if (fd_format != dpaa2_fd_sg)
		/* We don't support any other format */
		return;

82
83
84
	/* For S/G frames, we first need to free all SG entries
	 * except the first one, which was taken care of already
	 */
85
	sgt = vaddr + dpaa2_fd_get_offset(fd);
86
	for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
87
		addr = dpaa2_sg_get_addr(&sgt[i]);
88
		sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
89
90
91
92
93
94
95
96
97
98
99
100
101
		dma_unmap_single(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
				 DMA_FROM_DEVICE);

		skb_free_frag(sg_vaddr);
		if (dpaa2_sg_is_final(&sgt[i]))
			break;
	}

free_buf:
	skb_free_frag(vaddr);
}

/* Build a linear skb based on a single-buffer frame descriptor */
102
static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch,
103
104
105
106
107
108
109
					const struct dpaa2_fd *fd,
					void *fd_vaddr)
{
	struct sk_buff *skb = NULL;
	u16 fd_offset = dpaa2_fd_get_offset(fd);
	u32 fd_length = dpaa2_fd_get_len(fd);

110
111
	ch->buf_count--;

112
	skb = build_skb(fd_vaddr, DPAA2_ETH_SKB_SIZE);
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
	if (unlikely(!skb))
		return NULL;

	skb_reserve(skb, fd_offset);
	skb_put(skb, fd_length);

	return skb;
}

/* Build a non linear (fragmented) skb based on a S/G table */
static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv,
				      struct dpaa2_eth_channel *ch,
				      struct dpaa2_sg_entry *sgt)
{
	struct sk_buff *skb = NULL;
	struct device *dev = priv->net_dev->dev.parent;
	void *sg_vaddr;
	dma_addr_t sg_addr;
	u16 sg_offset;
	u32 sg_length;
	struct page *page, *head_page;
	int page_offset;
	int i;

	for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
		struct dpaa2_sg_entry *sge = &sgt[i];

		/* NOTE: We only support SG entries in dpaa2_sg_single format,
		 * but this is the only format we may receive from HW anyway
		 */

		/* Get the address and length from the S/G entry */
		sg_addr = dpaa2_sg_get_addr(sge);
146
		sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr);
147
148
149
150
151
152
153
		dma_unmap_single(dev, sg_addr, DPAA2_ETH_RX_BUF_SIZE,
				 DMA_FROM_DEVICE);

		sg_length = dpaa2_sg_get_len(sge);

		if (i == 0) {
			/* We build the skb around the first data buffer */
154
			skb = build_skb(sg_vaddr, DPAA2_ETH_SKB_SIZE);
155
			if (unlikely(!skb)) {
156
157
158
159
160
				/* Free the first SG entry now, since we already
				 * unmapped it and obtained the virtual address
				 */
				skb_free_frag(sg_vaddr);

161
162
163
164
165
166
167
168
				/* We still need to subtract the buffers used
				 * by this FD from our software counter
				 */
				while (!dpaa2_sg_is_final(&sgt[i]) &&
				       i < DPAA2_ETH_MAX_SG_ENTRIES)
					i++;
				break;
			}
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194

			sg_offset = dpaa2_sg_get_offset(sge);
			skb_reserve(skb, sg_offset);
			skb_put(skb, sg_length);
		} else {
			/* Rest of the data buffers are stored as skb frags */
			page = virt_to_page(sg_vaddr);
			head_page = virt_to_head_page(sg_vaddr);

			/* Offset in page (which may be compound).
			 * Data in subsequent SG entries is stored from the
			 * beginning of the buffer, so we don't need to add the
			 * sg_offset.
			 */
			page_offset = ((unsigned long)sg_vaddr &
				(PAGE_SIZE - 1)) +
				(page_address(page) - page_address(head_page));

			skb_add_rx_frag(skb, i - 1, head_page, page_offset,
					sg_length, DPAA2_ETH_RX_BUF_SIZE);
		}

		if (dpaa2_sg_is_final(sge))
			break;
	}

195
196
	WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT");

197
198
199
200
201
202
	/* Count all data buffers + SG table buffer */
	ch->buf_count -= i + 2;

	return skb;
}

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
static u32 run_xdp(struct dpaa2_eth_priv *priv,
		   struct dpaa2_eth_channel *ch,
		   struct dpaa2_fd *fd, void *vaddr)
{
	struct bpf_prog *xdp_prog;
	struct xdp_buff xdp;
	u32 xdp_act = XDP_PASS;

	rcu_read_lock();

	xdp_prog = READ_ONCE(ch->xdp.prog);
	if (!xdp_prog)
		goto out;

	xdp.data = vaddr + dpaa2_fd_get_offset(fd);
	xdp.data_end = xdp.data + dpaa2_fd_get_len(fd);
	xdp.data_hard_start = xdp.data;
	xdp_set_data_meta_invalid(&xdp);

	xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);

	switch (xdp_act) {
	case XDP_PASS:
		break;
	default:
		bpf_warn_invalid_xdp_action(xdp_act);
	case XDP_ABORTED:
		trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
	case XDP_DROP:
		ch->buf_count--;
		free_rx_fd(priv, fd, vaddr);
		break;
	}

out:
	rcu_read_unlock();
	return xdp_act;
}

242
243
244
245
/* Main Rx frame processing routine */
static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
			 struct dpaa2_eth_channel *ch,
			 const struct dpaa2_fd *fd,
246
			 struct dpaa2_eth_fq *fq)
247
248
249
250
251
252
{
	dma_addr_t addr = dpaa2_fd_get_addr(fd);
	u8 fd_format = dpaa2_fd_get_format(fd);
	void *vaddr;
	struct sk_buff *skb;
	struct rtnl_link_stats64 *percpu_stats;
253
	struct dpaa2_eth_drv_stats *percpu_extras;
254
255
	struct device *dev = priv->net_dev->dev.parent;
	struct dpaa2_fas *fas;
256
	void *buf_data;
257
	u32 status = 0;
258
	u32 xdp_act;
259

260
261
262
	/* Tracing point */
	trace_dpaa2_rx_fd(priv->net_dev, fd);

263
	vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
264
265
	dma_unmap_single(dev, addr, DPAA2_ETH_RX_BUF_SIZE, DMA_FROM_DEVICE);

266
	fas = dpaa2_get_fas(vaddr, false);
267
268
269
	prefetch(fas);
	buf_data = vaddr + dpaa2_fd_get_offset(fd);
	prefetch(buf_data);
270
271

	percpu_stats = this_cpu_ptr(priv->percpu_stats);
272
	percpu_extras = this_cpu_ptr(priv->percpu_extras);
273
274

	if (fd_format == dpaa2_fd_single) {
275
276
277
278
279
280
281
		xdp_act = run_xdp(priv, ch, (struct dpaa2_fd *)fd, vaddr);
		if (xdp_act != XDP_PASS) {
			percpu_stats->rx_packets++;
			percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
			return;
		}

282
		skb = build_linear_skb(ch, fd, vaddr);
283
	} else if (fd_format == dpaa2_fd_sg) {
284
285
		WARN_ON(priv->xdp_prog);

286
		skb = build_frag_skb(priv, ch, buf_data);
287
		skb_free_frag(vaddr);
288
289
		percpu_extras->rx_sg_frames++;
		percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd);
290
291
292
293
294
295
296
297
298
299
	} else {
		/* We don't support any other format */
		goto err_frame_format;
	}

	if (unlikely(!skb))
		goto err_build_skb;

	prefetch(skb->data);

300
301
302
303
304
305
306
307
308
309
310
311
	/* Get the timestamp value */
	if (priv->rx_tstamp) {
		struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
		__le64 *ts = dpaa2_get_ts(vaddr, false);
		u64 ns;

		memset(shhwtstamps, 0, sizeof(*shhwtstamps));

		ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
		shhwtstamps->hwtstamp = ns_to_ktime(ns);
	}

312
313
314
315
316
317
318
	/* Check if we need to validate the L4 csum */
	if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) {
		status = le32_to_cpu(fas->status);
		validate_rx_csum(priv, status, skb);
	}

	skb->protocol = eth_type_trans(skb, priv->net_dev);
319
	skb_record_rx_queue(skb, fq->flowid);
320
321
322
323

	percpu_stats->rx_packets++;
	percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);

324
	napi_gro_receive(&ch->napi, skb);
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339

	return;

err_build_skb:
	free_rx_fd(priv, fd, vaddr);
err_frame_format:
	percpu_stats->rx_dropped++;
}

/* Consume all frames pull-dequeued into the store. This is the simplest way to
 * make sure we don't accidentally issue another volatile dequeue which would
 * overwrite (leak) frames already in the store.
 *
 * Observance of NAPI budget is not our concern, leaving that to the caller.
 */
340
static int consume_frames(struct dpaa2_eth_channel *ch,
341
			  struct dpaa2_eth_fq **src)
342
343
{
	struct dpaa2_eth_priv *priv = ch->priv;
344
	struct dpaa2_eth_fq *fq = NULL;
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
	struct dpaa2_dq *dq;
	const struct dpaa2_fd *fd;
	int cleaned = 0;
	int is_last;

	do {
		dq = dpaa2_io_store_next(ch->store, &is_last);
		if (unlikely(!dq)) {
			/* If we're here, we *must* have placed a
			 * volatile dequeue comnmand, so keep reading through
			 * the store until we get some sort of valid response
			 * token (either a valid frame or an "empty dequeue")
			 */
			continue;
		}

		fd = dpaa2_dq_fd(dq);
362
		fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq);
363

364
		fq->consume(priv, ch, fd, fq);
365
366
367
		cleaned++;
	} while (!is_last);

368
369
370
371
372
373
374
	if (!cleaned)
		return 0;

	fq->stats.frames += cleaned;
	ch->stats.frames += cleaned;

	/* A dequeue operation only pulls frames from a single queue
375
	 * into the store. Return the frame queue as an out param.
376
	 */
377
378
	if (src)
		*src = fq;
379

380
381
382
	return cleaned;
}

383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
/* Configure the egress frame annotation for timestamp update */
static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start)
{
	struct dpaa2_faead *faead;
	u32 ctrl, frc;

	/* Mark the egress frame annotation area as valid */
	frc = dpaa2_fd_get_frc(fd);
	dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);

	/* Set hardware annotation size */
	ctrl = dpaa2_fd_get_ctrl(fd);
	dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL);

	/* enable UPD (update prepanded data) bit in FAEAD field of
	 * hardware frame annotation area
	 */
	ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD;
	faead = dpaa2_get_faead(buf_start, true);
	faead->ctrl = cpu_to_le32(ctrl);
}

405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
/* Create a frame descriptor based on a fragmented skb */
static int build_sg_fd(struct dpaa2_eth_priv *priv,
		       struct sk_buff *skb,
		       struct dpaa2_fd *fd)
{
	struct device *dev = priv->net_dev->dev.parent;
	void *sgt_buf = NULL;
	dma_addr_t addr;
	int nr_frags = skb_shinfo(skb)->nr_frags;
	struct dpaa2_sg_entry *sgt;
	int i, err;
	int sgt_buf_size;
	struct scatterlist *scl, *crt_scl;
	int num_sg;
	int num_dma_bufs;
	struct dpaa2_eth_swa *swa;

	/* Create and map scatterlist.
	 * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have
	 * to go beyond nr_frags+1.
	 * Note: We don't support chained scatterlists
	 */
	if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1))
		return -EINVAL;

	scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC);
	if (unlikely(!scl))
		return -ENOMEM;

	sg_init_table(scl, nr_frags + 1);
	num_sg = skb_to_sgvec(skb, scl, 0, skb->len);
436
	num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
437
438
439
440
441
442
443
	if (unlikely(!num_dma_bufs)) {
		err = -ENOMEM;
		goto dma_map_sg_failed;
	}

	/* Prepare the HW SGT structure */
	sgt_buf_size = priv->tx_data_offset +
444
		       sizeof(struct dpaa2_sg_entry) *  num_dma_bufs;
445
	sgt_buf = netdev_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN);
446
447
448
449
450
	if (unlikely(!sgt_buf)) {
		err = -ENOMEM;
		goto sgt_buf_alloc_failed;
	}
	sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN);
451
452
	memset(sgt_buf, 0, sgt_buf_size);

453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
	sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);

	/* Fill in the HW SGT structure.
	 *
	 * sgt_buf is zeroed out, so the following fields are implicit
	 * in all sgt entries:
	 *   - offset is 0
	 *   - format is 'dpaa2_sg_single'
	 */
	for_each_sg(scl, crt_scl, num_dma_bufs, i) {
		dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl));
		dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl));
	}
	dpaa2_sg_set_final(&sgt[i - 1], true);

	/* Store the skb backpointer in the SGT buffer.
	 * Fit the scatterlist and the number of buffers alongside the
	 * skb backpointer in the software annotation area. We'll need
	 * all of them on Tx Conf.
	 */
	swa = (struct dpaa2_eth_swa *)sgt_buf;
	swa->skb = skb;
	swa->scl = scl;
	swa->num_sg = num_sg;
477
	swa->sgt_size = sgt_buf_size;
478
479

	/* Separately map the SGT buffer */
480
	addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);
481
482
483
484
485
486
487
488
	if (unlikely(dma_mapping_error(dev, addr))) {
		err = -ENOMEM;
		goto dma_map_single_failed;
	}
	dpaa2_fd_set_offset(fd, priv->tx_data_offset);
	dpaa2_fd_set_format(fd, dpaa2_fd_sg);
	dpaa2_fd_set_addr(fd, addr);
	dpaa2_fd_set_len(fd, skb->len);
489
	dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
490

491
492
493
	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
		enable_tx_tstamp(fd, sgt_buf);

494
495
496
	return 0;

dma_map_single_failed:
497
	skb_free_frag(sgt_buf);
498
sgt_buf_alloc_failed:
499
	dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
500
501
502
503
504
505
506
507
508
509
510
dma_map_sg_failed:
	kfree(scl);
	return err;
}

/* Create a frame descriptor based on a linear skb */
static int build_single_fd(struct dpaa2_eth_priv *priv,
			   struct sk_buff *skb,
			   struct dpaa2_fd *fd)
{
	struct device *dev = priv->net_dev->dev.parent;
511
	u8 *buffer_start, *aligned_start;
512
513
514
	struct sk_buff **skbh;
	dma_addr_t addr;

515
516
517
518
519
520
521
522
523
	buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb);

	/* If there's enough room to align the FD address, do it.
	 * It will help hardware optimize accesses.
	 */
	aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN,
				  DPAA2_ETH_TX_BUF_ALIGN);
	if (aligned_start >= skb->head)
		buffer_start = aligned_start;
524
525
526
527
528
529
530
531
532
533

	/* Store a backpointer to the skb at the beginning of the buffer
	 * (in the private data area) such that we can release it
	 * on Tx confirm
	 */
	skbh = (struct sk_buff **)buffer_start;
	*skbh = skb;

	addr = dma_map_single(dev, buffer_start,
			      skb_tail_pointer(skb) - buffer_start,
534
			      DMA_BIDIRECTIONAL);
535
536
537
538
539
540
541
	if (unlikely(dma_mapping_error(dev, addr)))
		return -ENOMEM;

	dpaa2_fd_set_addr(fd, addr);
	dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start));
	dpaa2_fd_set_len(fd, skb->len);
	dpaa2_fd_set_format(fd, dpaa2_fd_single);
542
	dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
543

544
545
546
	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
		enable_tx_tstamp(fd, buffer_start);

547
548
549
550
551
552
553
554
555
556
557
	return 0;
}

/* FD freeing routine on the Tx path
 *
 * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb
 * back-pointed to is also freed.
 * This can be called either from dpaa2_eth_tx_conf() or on the error path of
 * dpaa2_eth_tx().
 */
static void free_tx_fd(const struct dpaa2_eth_priv *priv,
558
		       const struct dpaa2_fd *fd)
559
560
561
562
563
564
565
566
567
{
	struct device *dev = priv->net_dev->dev.parent;
	dma_addr_t fd_addr;
	struct sk_buff **skbh, *skb;
	unsigned char *buffer_start;
	struct dpaa2_eth_swa *swa;
	u8 fd_format = dpaa2_fd_get_format(fd);

	fd_addr = dpaa2_fd_get_addr(fd);
568
	skbh = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr);
569
570
571
572
573
574
575
576
577

	if (fd_format == dpaa2_fd_single) {
		skb = *skbh;
		buffer_start = (unsigned char *)skbh;
		/* Accessing the skb buffer is safe before dma unmap, because
		 * we didn't map the actual skb shell.
		 */
		dma_unmap_single(dev, fd_addr,
				 skb_tail_pointer(skb) - buffer_start,
578
				 DMA_BIDIRECTIONAL);
579
580
581
582
583
	} else if (fd_format == dpaa2_fd_sg) {
		swa = (struct dpaa2_eth_swa *)skbh;
		skb = swa->skb;

		/* Unmap the scatterlist */
584
585
		dma_unmap_sg(dev, swa->scl, swa->num_sg, DMA_BIDIRECTIONAL);
		kfree(swa->scl);
586
587

		/* Unmap the SGT buffer */
588
589
		dma_unmap_single(dev, fd_addr, swa->sgt_size,
				 DMA_BIDIRECTIONAL);
590
	} else {
591
		netdev_dbg(priv->net_dev, "Invalid FD format\n");
592
593
594
		return;
	}

595
596
597
598
599
600
601
602
603
604
605
606
607
	/* Get the timestamp value */
	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
		struct skb_shared_hwtstamps shhwtstamps;
		__le64 *ts = dpaa2_get_ts(skbh, true);
		u64 ns;

		memset(&shhwtstamps, 0, sizeof(shhwtstamps));

		ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
		shhwtstamps.hwtstamp = ns_to_ktime(ns);
		skb_tstamp_tx(skb, &shhwtstamps);
	}

608
	/* Free SGT buffer allocated on tx */
609
	if (fd_format != dpaa2_fd_single)
610
		skb_free_frag(skbh);
611
612
613
614
615

	/* Move on with skb release */
	dev_kfree_skb(skb);
}

616
static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev)
617
618
619
620
{
	struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
	struct dpaa2_fd fd;
	struct rtnl_link_stats64 *percpu_stats;
621
	struct dpaa2_eth_drv_stats *percpu_extras;
622
	struct dpaa2_eth_fq *fq;
623
	struct netdev_queue *nq;
624
	u16 queue_mapping;
625
	unsigned int needed_headroom;
626
	u32 fd_len;
627
628
629
	int err, i;

	percpu_stats = this_cpu_ptr(priv->percpu_stats);
630
	percpu_extras = this_cpu_ptr(priv->percpu_extras);
631

632
633
	needed_headroom = dpaa2_eth_needed_headroom(priv, skb);
	if (skb_headroom(skb) < needed_headroom) {
634
635
		struct sk_buff *ns;

636
		ns = skb_realloc_headroom(skb, needed_headroom);
637
638
639
640
		if (unlikely(!ns)) {
			percpu_stats->tx_dropped++;
			goto err_alloc_headroom;
		}
641
		percpu_extras->tx_reallocs++;
642
643
644
645

		if (skb->sk)
			skb_set_owner_w(ns, skb->sk);

646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
		dev_kfree_skb(skb);
		skb = ns;
	}

	/* We'll be holding a back-reference to the skb until Tx Confirmation;
	 * we don't want that overwritten by a concurrent Tx with a cloned skb.
	 */
	skb = skb_unshare(skb, GFP_ATOMIC);
	if (unlikely(!skb)) {
		/* skb_unshare() has already freed the skb */
		percpu_stats->tx_dropped++;
		return NETDEV_TX_OK;
	}

	/* Setup the FD fields */
	memset(&fd, 0, sizeof(fd));

663
	if (skb_is_nonlinear(skb)) {
664
		err = build_sg_fd(priv, skb, &fd);
665
666
667
		percpu_extras->tx_sg_frames++;
		percpu_extras->tx_sg_bytes += skb->len;
	} else {
668
		err = build_single_fd(priv, skb, &fd);
669
670
	}

671
672
673
674
675
	if (unlikely(err)) {
		percpu_stats->tx_dropped++;
		goto err_build_fd;
	}

676
677
678
	/* Tracing point */
	trace_dpaa2_tx_fd(net_dev, &fd);

679
680
681
	/* TxConf FQ selection relies on queue id from the stack.
	 * In case of a forwarded frame from another DPNI interface, we choose
	 * a queue affined to the same core that processed the Rx frame
682
	 */
683
	queue_mapping = skb_get_queue_mapping(skb);
684
685
	fq = &priv->fq[queue_mapping];
	for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
686
687
		err = dpaa2_io_service_enqueue_qd(fq->channel->dpio,
						  priv->tx_qdid, 0,
688
689
690
691
						  fq->tx_qdbin, &fd);
		if (err != -EBUSY)
			break;
	}
692
	percpu_extras->tx_portal_busy += i;
693
694
695
	if (unlikely(err < 0)) {
		percpu_stats->tx_errors++;
		/* Clean up everything, including freeing the skb */
696
		free_tx_fd(priv, &fd);
697
	} else {
698
		fd_len = dpaa2_fd_get_len(&fd);
699
		percpu_stats->tx_packets++;
700
701
702
703
		percpu_stats->tx_bytes += fd_len;

		nq = netdev_get_tx_queue(net_dev, queue_mapping);
		netdev_tx_sent_queue(nq, fd_len);
704
705
706
707
708
709
710
711
712
713
714
715
716
	}

	return NETDEV_TX_OK;

err_build_fd:
err_alloc_headroom:
	dev_kfree_skb(skb);

	return NETDEV_TX_OK;
}

/* Tx confirmation frame processing routine */
static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv,
717
			      struct dpaa2_eth_channel *ch __always_unused,
718
			      const struct dpaa2_fd *fd,
719
			      struct dpaa2_eth_fq *fq)
720
721
{
	struct rtnl_link_stats64 *percpu_stats;
722
	struct dpaa2_eth_drv_stats *percpu_extras;
723
	u32 fd_len = dpaa2_fd_get_len(fd);
724
	u32 fd_errors;
725

726
727
728
	/* Tracing point */
	trace_dpaa2_tx_conf_fd(priv->net_dev, fd);

729
730
	percpu_extras = this_cpu_ptr(priv->percpu_extras);
	percpu_extras->tx_conf_frames++;
731
732
733
734
	percpu_extras->tx_conf_bytes += fd_len;

	fq->dq_frames++;
	fq->dq_bytes += fd_len;
735

736
737
	/* Check frame errors in the FD field */
	fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK;
738
	free_tx_fd(priv, fd);
739
740
741
742

	if (likely(!fd_errors))
		return;

743
744
745
746
	if (net_ratelimit())
		netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n",
			   fd_errors);

747
748
749
	percpu_stats = this_cpu_ptr(priv->percpu_stats);
	/* Tx-conf logically pertains to the egress path. */
	percpu_stats->tx_errors++;
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
}

static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable)
{
	int err;

	err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
			       DPNI_OFF_RX_L3_CSUM, enable);
	if (err) {
		netdev_err(priv->net_dev,
			   "dpni_set_offload(RX_L3_CSUM) failed\n");
		return err;
	}

	err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
			       DPNI_OFF_RX_L4_CSUM, enable);
	if (err) {
		netdev_err(priv->net_dev,
			   "dpni_set_offload(RX_L4_CSUM) failed\n");
		return err;
	}

	return 0;
}

static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable)
{
	int err;

	err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
			       DPNI_OFF_TX_L3_CSUM, enable);
	if (err) {
		netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n");
		return err;
	}

	err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
			       DPNI_OFF_TX_L4_CSUM, enable);
	if (err) {
		netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n");
		return err;
	}

	return 0;
}

796
797
798
799
800
801
802
803
804
805
806
807
/* Free buffers acquired from the buffer pool or which were meant to
 * be released in the pool
 */
static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count)
{
	struct device *dev = priv->net_dev->dev.parent;
	void *vaddr;
	int i;

	for (i = 0; i < count; i++) {
		vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]);
		dma_unmap_single(dev, buf_array[i], DPAA2_ETH_RX_BUF_SIZE,
808
				 DMA_FROM_DEVICE);
809
810
811
812
		skb_free_frag(vaddr);
	}
}

813
814
815
/* Perform a single release command to add buffers
 * to the specified buffer pool
 */
816
817
static int add_bufs(struct dpaa2_eth_priv *priv,
		    struct dpaa2_eth_channel *ch, u16 bpid)
818
819
820
821
822
{
	struct device *dev = priv->net_dev->dev.parent;
	u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
	void *buf;
	dma_addr_t addr;
823
	int i, err;
824
825
826
827
828

	for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) {
		/* Allocate buffer visible to WRIOP + skb shared info +
		 * alignment padding
		 */
829
		buf = napi_alloc_frag(dpaa2_eth_buf_raw_size(priv));
830
831
832
		if (unlikely(!buf))
			goto err_alloc;

833
		buf = PTR_ALIGN(buf, priv->rx_buf_align);
834
835
836
837
838
839
840

		addr = dma_map_single(dev, buf, DPAA2_ETH_RX_BUF_SIZE,
				      DMA_FROM_DEVICE);
		if (unlikely(dma_mapping_error(dev, addr)))
			goto err_map;

		buf_array[i] = addr;
841
842
843

		/* tracing point */
		trace_dpaa2_eth_buf_seed(priv->net_dev,
844
					 buf, dpaa2_eth_buf_raw_size(priv),
845
846
					 addr, DPAA2_ETH_RX_BUF_SIZE,
					 bpid);
847
848
849
	}

release_bufs:
850
	/* In case the portal is busy, retry until successful */
851
	while ((err = dpaa2_io_service_release(ch->dpio, bpid,
852
					       buf_array, i)) == -EBUSY)
853
		cpu_relax();
854
855
856
857
858
859
860
861
862

	/* If release command failed, clean up and bail out;
	 * not much else we can do about it
	 */
	if (err) {
		free_bufs(priv, buf_array, i);
		return 0;
	}

863
864
865
866
867
	return i;

err_map:
	skb_free_frag(buf);
err_alloc:
868
869
870
	/* If we managed to allocate at least some buffers,
	 * release them to hardware
	 */
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
	if (i)
		goto release_bufs;

	return 0;
}

static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid)
{
	int i, j;
	int new_count;

	/* This is the lazy seeding of Rx buffer pools.
	 * dpaa2_add_bufs() is also used on the Rx hotpath and calls
	 * napi_alloc_frag(). The trouble with that is that it in turn ends up
	 * calling this_cpu_ptr(), which mandates execution in atomic context.
	 * Rather than splitting up the code, do a one-off preempt disable.
	 */
	preempt_disable();
	for (j = 0; j < priv->num_channels; j++) {
		for (i = 0; i < DPAA2_ETH_NUM_BUFS;
		     i += DPAA2_ETH_BUFS_PER_CMD) {
892
			new_count = add_bufs(priv, priv->channel[j], bpid);
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
			priv->channel[j]->buf_count += new_count;

			if (new_count < DPAA2_ETH_BUFS_PER_CMD) {
				preempt_enable();
				return -ENOMEM;
			}
		}
	}
	preempt_enable();

	return 0;
}

/**
 * Drain the specified number of buffers from the DPNI's private buffer pool.
 * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD
 */
static void drain_bufs(struct dpaa2_eth_priv *priv, int count)
{
	u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
913
	int ret;
914
915

	do {
916
		ret = dpaa2_io_service_acquire(NULL, priv->bpid,
917
918
919
920
921
					       buf_array, count);
		if (ret < 0) {
			netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n");
			return;
		}
922
		free_bufs(priv, buf_array, ret);
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
	} while (ret);
}

static void drain_pool(struct dpaa2_eth_priv *priv)
{
	int i;

	drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD);
	drain_bufs(priv, 1);

	for (i = 0; i < priv->num_channels; i++)
		priv->channel[i]->buf_count = 0;
}

/* Function is called from softirq context only, so we don't need to guard
 * the access to percpu count
 */
static int refill_pool(struct dpaa2_eth_priv *priv,
		       struct dpaa2_eth_channel *ch,
		       u16 bpid)
{
	int new_count;

	if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH))
		return 0;

	do {
950
		new_count = add_bufs(priv, ch, bpid);
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
		if (unlikely(!new_count)) {
			/* Out of memory; abort for now, we'll try later on */
			break;
		}
		ch->buf_count += new_count;
	} while (ch->buf_count < DPAA2_ETH_NUM_BUFS);

	if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS))
		return -ENOMEM;

	return 0;
}

static int pull_channel(struct dpaa2_eth_channel *ch)
{
	int err;
967
	int dequeues = -1;
968
969
970

	/* Retry while portal is busy */
	do {
971
972
		err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id,
						    ch->store);
973
		dequeues++;
974
975
976
		cpu_relax();
	} while (err == -EBUSY);

977
978
979
980
	ch->stats.dequeue_portal_busy += dequeues;
	if (unlikely(err))
		ch->stats.pull_err++;

981
982
983
984
985
986
987
988
989
990
991
992
993
	return err;
}

/* NAPI poll routine
 *
 * Frames are dequeued from the QMan channel associated with this NAPI context.
 * Rx, Tx confirmation and (if configured) Rx error frames all count
 * towards the NAPI budget.
 */
static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
{
	struct dpaa2_eth_channel *ch;
	struct dpaa2_eth_priv *priv;
994
	int rx_cleaned = 0, txconf_cleaned = 0;
995
996
997
	struct dpaa2_eth_fq *fq, *txc_fq = NULL;
	struct netdev_queue *nq;
	int store_cleaned, work_done;
998
999
1000
	int err;

	ch = container_of(napi, struct dpaa2_eth_channel, napi);
For faster browsing, not all history is shown. View entire blame