ixgbe_main.c 313 KB
Newer Older
1
2
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 1999 - 2018 Intel Corporation. */
3
4
5
6
7
8
9
10

#include <linux/types.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/in.h>
11
#include <linux/interrupt.h>
12
13
#include <linux/ip.h>
#include <linux/tcp.h>
14
#include <linux/sctp.h>
15
#include <linux/pkt_sched.h>
16
#include <linux/ipv6.h>
17
#include <linux/slab.h>
18
19
#include <net/checksum.h>
#include <net/ip6_checksum.h>
20
#include <linux/etherdevice.h>
21
#include <linux/ethtool.h>
22
#include <linux/if.h>
23
#include <linux/if_vlan.h>
24
#include <linux/if_macvlan.h>
25
#include <linux/if_bridge.h>
26
#include <linux/prefetch.h>
27
28
29
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/atomic.h>
30
#include <linux/numa.h>
31
#include <scsi/fc/fc_fcoe.h>
32
#include <net/udp_tunnel.h>
33
34
#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
35
#include <net/tc_act/tc_mirred.h>
36
#include <net/vxlan.h>
37
#include <net/mpls.h>
38
#include <net/xdp_sock.h>
39
#include <net/xfrm.h>
40
41
42

#include "ixgbe.h"
#include "ixgbe_common.h"
43
#include "ixgbe_dcb_82599.h"
Steve Douthit's avatar
Steve Douthit committed
44
#include "ixgbe_phy.h"
45
#include "ixgbe_sriov.h"
46
#include "ixgbe_model.h"
47
#include "ixgbe_txrx_common.h"
48
49

char ixgbe_driver_name[] = "ixgbe";
Stephen Hemminger's avatar
Stephen Hemminger committed
50
static const char ixgbe_driver_string[] =
51
			      "Intel(R) 10 Gigabit PCI Express Network Driver";
52
#ifdef IXGBE_FCOE
53
54
char ixgbe_default_device_descr[] =
			      "Intel(R) 10 Gigabit Network Connection";
55
56
57
58
#else
static char ixgbe_default_device_descr[] =
			      "Intel(R) 10 Gigabit Network Connection";
#endif
Tony Nguyen's avatar
Tony Nguyen committed
59
#define DRV_VERSION "5.1.0-k"
Stephen Hemminger's avatar
Stephen Hemminger committed
60
const char ixgbe_driver_version[] = DRV_VERSION;
61
static const char ixgbe_copyright[] =
62
				"Copyright (c) 1999-2016 Intel Corporation.";
63

64
65
static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";

66
static const struct ixgbe_info *ixgbe_info_tbl[] = {
67
68
69
70
71
	[board_82598]		= &ixgbe_82598_info,
	[board_82599]		= &ixgbe_82599_info,
	[board_X540]		= &ixgbe_X540_info,
	[board_X550]		= &ixgbe_X550_info,
	[board_X550EM_x]	= &ixgbe_X550EM_x_info,
72
	[board_x550em_x_fw]	= &ixgbe_x550em_x_fw_info,
73
	[board_x550em_a]	= &ixgbe_x550em_a_info,
74
	[board_x550em_a_fw]	= &ixgbe_x550em_a_fw_info,
75
76
77
78
79
80
81
82
83
84
};

/* ixgbe_pci_tbl - PCI Device ID Table
 *
 * Wildcard entries (PCI_ANY_ID) should come last
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
85
static const struct pci_device_id ixgbe_pci_tbl[] = {
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
112
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
113
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
114
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
115
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
116
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
117
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
118
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
119
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), board_X550EM_x},
120
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
121
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
122
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
123
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), board_x550em_x_fw},
124
125
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
126
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
127
128
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
129
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
130
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
131
132
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
133
134
135
136
137
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);

138
#ifdef CONFIG_IXGBE_DCA
139
static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
140
			    void *p);
141
142
143
144
145
146
147
static struct notifier_block dca_notifier = {
	.notifier_call = ixgbe_notify_dca,
	.next          = NULL,
	.priority      = 0
};
#endif

148
149
150
#ifdef CONFIG_PCI_IOV
static unsigned int max_vfs;
module_param(max_vfs, uint, 0);
151
MODULE_PARM_DESC(max_vfs,
152
		 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
153
154
#endif /* CONFIG_PCI_IOV */

155
156
157
158
159
static unsigned int allow_unsupported_sfp;
module_param(allow_unsupported_sfp, uint, 0);
MODULE_PARM_DESC(allow_unsupported_sfp,
		 "Allow unsupported and untested SFP+ modules on 82599-based adapters");

160
161
162
163
164
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

165
166
MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
167
MODULE_LICENSE("GPL v2");
168
169
MODULE_VERSION(DRV_VERSION);

170
171
static struct workqueue_struct *ixgbe_wq;

172
static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
173
static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);
174

175
176
177
178
179
180
181
static const struct net_device_ops ixgbe_netdev_ops;

static bool netif_is_ixgbe(struct net_device *dev)
{
	return dev && (dev->netdev_ops == &ixgbe_netdev_ops);
}

182
183
184
185
186
187
188
189
190
191
192
193
194
195
static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
					  u32 reg, u16 *value)
{
	struct pci_dev *parent_dev;
	struct pci_bus *parent_bus;

	parent_bus = adapter->pdev->bus->parent;
	if (!parent_bus)
		return -1;

	parent_dev = parent_bus->self;
	if (!parent_dev)
		return -1;

196
	if (!pci_is_pcie(parent_dev))
197
198
		return -1;

199
	pcie_capability_read_word(parent_dev, reg, value);
200
201
202
	if (*value == IXGBE_FAILED_READ_CFG_WORD &&
	    ixgbe_check_cfg_remove(&adapter->hw, parent_dev))
		return -1;
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
	return 0;
}

static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u16 link_status = 0;
	int err;

	hw->bus.type = ixgbe_bus_type_pci_express;

	/* Get the negotiated link width and speed from PCI config space of the
	 * parent, as this device is behind a switch
	 */
	err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status);

	/* assume caller will handle error case */
	if (err)
		return err;

	hw->bus.width = ixgbe_convert_bus_width(link_status);
	hw->bus.speed = ixgbe_convert_bus_speed(link_status);

	return 0;
}

229
230
231
232
233
234
235
236
237
238
239
240
241
/**
 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent
 * @hw: hw specific details
 *
 * This function is used by probe to determine whether a device's PCI-Express
 * bandwidth details should be gathered from the parent bus instead of from the
 * device. Used to ensure that various locations all have the correct device ID
 * checks.
 */
static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
{
	switch (hw->device_id) {
	case IXGBE_DEV_ID_82599_SFP_SF_QP:
242
	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
243
244
245
246
247
248
249
250
251
		return true;
	default:
		return false;
	}
}

static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
				     int expected_gts)
{
252
	struct ixgbe_hw *hw = &adapter->hw;
253
254
	struct pci_dev *pdev;

255
256
257
258
259
260
261
	/* Some devices are not connected over PCIe and thus do not negotiate
	 * speed. These devices do not have valid bus info, and thus any report
	 * we generate may not be correct.
	 */
	if (hw->bus.type == ixgbe_bus_type_internal)
		return;

262
	/* determine whether to use the parent device */
263
264
265
266
267
	if (ixgbe_pcie_from_parent(&adapter->hw))
		pdev = adapter->pdev->bus->parent->self;
	else
		pdev = adapter->pdev;

268
	pcie_print_link_status(pdev);
269
270
}

271
272
273
static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
{
	if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
274
	    !test_bit(__IXGBE_REMOVING, &adapter->state) &&
275
	    !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
276
		queue_work(ixgbe_wq, &adapter->service_task);
277
278
}

279
280
281
282
283
284
285
286
static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
{
	struct ixgbe_adapter *adapter = hw->back;

	if (!hw->hw_addr)
		return;
	hw->hw_addr = NULL;
	e_dev_err("Adapter removed\n");
287
288
	if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
		ixgbe_service_event_schedule(adapter);
289
290
}

291
static u32 ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
292
{
293
	u8 __iomem *reg_addr;
294
	u32 value;
295
	int i;
296

297
298
299
300
301
302
303
	reg_addr = READ_ONCE(hw->hw_addr);
	if (ixgbe_removed(reg_addr))
		return IXGBE_FAILED_READ_REG;

	/* Register read of 0xFFFFFFF can indicate the adapter has been removed,
	 * so perform several status register reads to determine if the adapter
	 * has been removed.
304
	 */
305
306
307
308
309
	for (i = 0; i < IXGBE_FAILED_READ_RETRIES; i++) {
		value = readl(reg_addr + IXGBE_STATUS);
		if (value != IXGBE_FAILED_READ_REG)
			break;
		mdelay(3);
310
	}
311

312
313
	if (value == IXGBE_FAILED_READ_REG)
		ixgbe_remove_adapter(hw);
314
315
316
	else
		value = readl(reg_addr + reg);
	return value;
317
318
}

319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
/**
 * ixgbe_read_reg - Read from device register
 * @hw: hw specific details
 * @reg: offset of register to read
 *
 * Returns : value read or IXGBE_FAILED_READ_REG if removed
 *
 * This function is used to read device registers. It checks for device
 * removal by confirming any read that returns all ones by checking the
 * status register value for all ones. This function avoids reading from
 * the hardware if a removal was previously detected in which case it
 * returns IXGBE_FAILED_READ_REG (all ones).
 */
u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
{
334
	u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
335
336
337
338
	u32 value;

	if (ixgbe_removed(reg_addr))
		return IXGBE_FAILED_READ_REG;
339
	if (unlikely(hw->phy.nw_mng_if_sel &
340
		     IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
		struct ixgbe_adapter *adapter;
		int i;

		for (i = 0; i < 200; ++i) {
			value = readl(reg_addr + IXGBE_MAC_SGMII_BUSY);
			if (likely(!value))
				goto writes_completed;
			if (value == IXGBE_FAILED_READ_REG) {
				ixgbe_remove_adapter(hw);
				return IXGBE_FAILED_READ_REG;
			}
			udelay(5);
		}

		adapter = hw->back;
		e_warn(hw, "register writes incomplete %08x\n", value);
	}

writes_completed:
360
361
	value = readl(reg_addr + reg);
	if (unlikely(value == IXGBE_FAILED_READ_REG))
362
		value = ixgbe_check_remove(hw, reg);
363
364
365
	return value;
}

366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
{
	u16 value;

	pci_read_config_word(pdev, PCI_VENDOR_ID, &value);
	if (value == IXGBE_FAILED_READ_CFG_WORD) {
		ixgbe_remove_adapter(hw);
		return true;
	}
	return false;
}

u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
{
	struct ixgbe_adapter *adapter = hw->back;
	u16 value;

	if (ixgbe_removed(hw->hw_addr))
		return IXGBE_FAILED_READ_CFG_WORD;
	pci_read_config_word(adapter->pdev, reg, &value);
	if (value == IXGBE_FAILED_READ_CFG_WORD &&
	    ixgbe_check_cfg_remove(hw, adapter->pdev))
		return IXGBE_FAILED_READ_CFG_WORD;
	return value;
}

#ifdef CONFIG_PCI_IOV
static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
{
	struct ixgbe_adapter *adapter = hw->back;
	u32 value;

	if (ixgbe_removed(hw->hw_addr))
		return IXGBE_FAILED_READ_CFG_DWORD;
	pci_read_config_dword(adapter->pdev, reg, &value);
	if (value == IXGBE_FAILED_READ_CFG_DWORD &&
	    ixgbe_check_cfg_remove(hw, adapter->pdev))
		return IXGBE_FAILED_READ_CFG_DWORD;
	return value;
}
#endif /* CONFIG_PCI_IOV */

408
409
410
411
412
413
414
415
416
void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
{
	struct ixgbe_adapter *adapter = hw->back;

	if (ixgbe_removed(hw->hw_addr))
		return;
	pci_write_config_word(adapter->pdev, reg, value);
}

417
418
419
420
static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
{
	BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));

Stephen Hemminger's avatar
Stephen Hemminger committed
421
	/* flush memory to make sure state is correct before next watchdog */
422
	smp_mb__before_atomic();
423
424
425
	clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
}

426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
struct ixgbe_reg_info {
	u32 ofs;
	char *name;
};

static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {

	/* General Registers */
	{IXGBE_CTRL, "CTRL"},
	{IXGBE_STATUS, "STATUS"},
	{IXGBE_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{IXGBE_EICR, "EICR"},

	/* RX Registers */
	{IXGBE_SRRCTL(0), "SRRCTL"},
	{IXGBE_DCA_RXCTRL(0), "DRXCTL"},
	{IXGBE_RDLEN(0), "RDLEN"},
	{IXGBE_RDH(0), "RDH"},
	{IXGBE_RDT(0), "RDT"},
	{IXGBE_RXDCTL(0), "RXDCTL"},
	{IXGBE_RDBAL(0), "RDBAL"},
	{IXGBE_RDBAH(0), "RDBAH"},

	/* TX Registers */
	{IXGBE_TDBAL(0), "TDBAL"},
	{IXGBE_TDBAH(0), "TDBAH"},
	{IXGBE_TDLEN(0), "TDLEN"},
	{IXGBE_TDH(0), "TDH"},
	{IXGBE_TDT(0), "TDT"},
	{IXGBE_TXDCTL(0), "TXDCTL"},

	/* List Terminator */
460
	{ .name = NULL }
461
462
463
464
465
466
467
468
};


/*
 * ixgbe_regdump - register printout routine
 */
static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
{
Joe Perches's avatar
Joe Perches committed
469
	int i;
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
	char rname[16];
	u32 regs[64];

	switch (reginfo->ofs) {
	case IXGBE_SRRCTL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
		break;
	case IXGBE_DCA_RXCTRL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
		break;
	case IXGBE_RDLEN(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
		break;
	case IXGBE_RDH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
		break;
	case IXGBE_RDT(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
		break;
	case IXGBE_RXDCTL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
		break;
	case IXGBE_RDBAL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
		break;
	case IXGBE_RDBAH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
		break;
	case IXGBE_TDBAL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
		break;
	case IXGBE_TDBAH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
		break;
	case IXGBE_TDLEN(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
		break;
	case IXGBE_TDH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
		break;
	case IXGBE_TDT(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
		break;
	case IXGBE_TXDCTL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
		break;
	default:
Joe Perches's avatar
Joe Perches committed
531
532
		pr_info("%-15s %08x\n",
			reginfo->name, IXGBE_READ_REG(hw, reginfo->ofs));
533
534
535
		return;
	}

Joe Perches's avatar
Joe Perches committed
536
537
538
539
540
541
542
	i = 0;
	while (i < 64) {
		int j;
		char buf[9 * 8 + 1];
		char *p = buf;

		snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i, i + 7);
543
		for (j = 0; j < 8; j++)
Joe Perches's avatar
Joe Perches committed
544
545
			p += sprintf(p, " %08x", regs[i++]);
		pr_err("%-15s%s\n", rname, buf);
546
547
548
549
	}

}

550
551
552
553
554
555
556
557
558
559
560
561
562
static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n)
{
	struct ixgbe_tx_buffer *tx_buffer;

	tx_buffer = &ring->tx_buffer_info[ring->next_to_clean];
	pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
		n, ring->next_to_use, ring->next_to_clean,
		(u64)dma_unmap_addr(tx_buffer, dma),
		dma_unmap_len(tx_buffer, len),
		tx_buffer->next_to_watch,
		(u64)tx_buffer->time_stamp);
}

563
564
565
566
567
568
569
570
571
/*
 * ixgbe_dump - Print registers, tx-rings and rx-rings
 */
static void ixgbe_dump(struct ixgbe_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct ixgbe_hw *hw = &adapter->hw;
	struct ixgbe_reg_info *reginfo;
	int n = 0;
572
	struct ixgbe_ring *ring;
573
	struct ixgbe_tx_buffer *tx_buffer;
574
575
576
577
578
579
580
581
582
583
584
585
586
	union ixgbe_adv_tx_desc *tx_desc;
	struct my_u0 { u64 a; u64 b; } *u0;
	struct ixgbe_ring *rx_ring;
	union ixgbe_adv_rx_desc *rx_desc;
	struct ixgbe_rx_buffer *rx_buffer_info;
	int i = 0;

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
587
		pr_info("Device Name     state            "
588
589
			"trans_start\n");
		pr_info("%-15s %016lX %016lX\n",
590
591
			netdev->name,
			netdev->state,
592
			dev_trans_start(netdev));
593
594
595
596
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
597
	pr_info(" Register Name   Value\n");
598
599
600
601
602
603
604
	for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
	     reginfo->name; reginfo++) {
		ixgbe_regdump(hw, reginfo);
	}

	/* Print TX Ring Summary */
	if (!netdev || !netif_running(netdev))
605
		return;
606
607

	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
608
609
610
	pr_info(" %s     %s              %s        %s\n",
		"Queue [NTU] [NTC] [bi(ntc)->dma  ]",
		"leng", "ntw", "timestamp");
611
	for (n = 0; n < adapter->num_tx_queues; n++) {
612
613
614
615
616
617
618
		ring = adapter->tx_ring[n];
		ixgbe_print_buffer(ring, n);
	}

	for (n = 0; n < adapter->num_xdp_queues; n++) {
		ring = adapter->xdp_ring[n];
		ixgbe_print_buffer(ring, n);
619
620
621
622
623
624
625
626
627
628
	}

	/* Print TX Rings */
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

	/* Transmit Descriptor Formats
	 *
629
	 * 82598 Advanced Transmit Descriptor
630
631
632
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0]                                |
	 *   +--------------------------------------------------------------+
633
	 * 8 |  PAYLEN  | POPTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
634
635
	 *   +--------------------------------------------------------------+
	 *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
	 *
	 * 82598 Advanced Transmit Descriptor (Write-Back Format)
	 *   +--------------------------------------------------------------+
	 * 0 |                          RSV [63:0]                          |
	 *   +--------------------------------------------------------------+
	 * 8 |            RSV           |  STA  |          NXTSEQ           |
	 *   +--------------------------------------------------------------+
	 *   63                       36 35   32 31                         0
	 *
	 * 82599+ Advanced Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0]                                |
	 *   +--------------------------------------------------------------+
	 * 8 |PAYLEN  |POPTS|CC|IDX  |STA  |DCMD  |DTYP |MAC  |RSV  |DTALEN |
	 *   +--------------------------------------------------------------+
	 *   63     46 45 40 39 38 36 35 32 31  24 23 20 19 18 17 16 15     0
	 *
	 * 82599+ Advanced Transmit Descriptor (Write-Back Format)
	 *   +--------------------------------------------------------------+
	 * 0 |                          RSV [63:0]                          |
	 *   +--------------------------------------------------------------+
	 * 8 |            RSV           |  STA  |           RSV             |
	 *   +--------------------------------------------------------------+
	 *   63                       36 35   32 31                         0
660
661
662
	 */

	for (n = 0; n < adapter->num_tx_queues; n++) {
663
		ring = adapter->tx_ring[n];
664
		pr_info("------------------------------------\n");
665
		pr_info("TX QUEUE INDEX = %d\n", ring->queue_index);
666
		pr_info("------------------------------------\n");
667
668
669
670
		pr_info("%s%s    %s              %s        %s          %s\n",
			"T [desc]     [address 63:0  ] ",
			"[PlPOIdStDDt Ln] [bi->dma       ] ",
			"leng", "ntw", "timestamp", "bi->skb");
671

672
673
674
		for (i = 0; ring->desc && (i < ring->count); i++) {
			tx_desc = IXGBE_TX_DESC(ring, i);
			tx_buffer = &ring->tx_buffer_info[i];
675
			u0 = (struct my_u0 *)tx_desc;
676
			if (dma_unmap_len(tx_buffer, len) > 0) {
Joe Perches's avatar
Joe Perches committed
677
678
				const char *ring_desc;

679
680
				if (i == ring->next_to_use &&
				    i == ring->next_to_clean)
Joe Perches's avatar
Joe Perches committed
681
					ring_desc = " NTC/U";
682
				else if (i == ring->next_to_use)
Joe Perches's avatar
Joe Perches committed
683
					ring_desc = " NTU";
684
				else if (i == ring->next_to_clean)
Joe Perches's avatar
Joe Perches committed
685
686
687
688
					ring_desc = " NTC";
				else
					ring_desc = "";
				pr_info("T [0x%03X]    %016llX %016llX %016llX %08X %p %016llX %p%s",
689
					i,
Cathy Zhou's avatar
Cathy Zhou committed
690
691
					le64_to_cpu((__force __le64)u0->a),
					le64_to_cpu((__force __le64)u0->b),
692
					(u64)dma_unmap_addr(tx_buffer, dma),
693
					dma_unmap_len(tx_buffer, len),
694
695
					tx_buffer->next_to_watch,
					(u64)tx_buffer->time_stamp,
Joe Perches's avatar
Joe Perches committed
696
697
					tx_buffer->skb,
					ring_desc);
698
699
700
701
702
703
704
705
706

				if (netif_msg_pktdata(adapter) &&
				    tx_buffer->skb)
					print_hex_dump(KERN_INFO, "",
						DUMP_PREFIX_ADDRESS, 16, 1,
						tx_buffer->skb->data,
						dma_unmap_len(tx_buffer, len),
						true);
			}
707
708
709
710
711
712
		}
	}

	/* Print RX Rings Summary */
rx_ring_summary:
	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
713
	pr_info("Queue [NTU] [NTC]\n");
714
715
	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
716
717
		pr_info("%5d %5X %5X\n",
			n, rx_ring->next_to_use, rx_ring->next_to_clean);
718
719
720
721
	}

	/* Print RX Rings */
	if (!netif_msg_rx_status(adapter))
722
		return;
723
724
725

	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

726
727
728
	/* Receive Descriptor Formats
	 *
	 * 82598 Advanced Receive Descriptor (Read) Format
729
730
731
732
733
734
735
736
	 *    63                                           1        0
	 *    +-----------------------------------------------------+
	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
	 *    +----------------------------------------------+------+
	 *  8 |       Header Buffer Address [63:1]           |  DD  |
	 *    +-----------------------------------------------------+
	 *
	 *
737
	 * 82598 Advanced Receive Descriptor (Write-Back) Format
738
739
740
	 *
	 *   63       48 47    32 31  30      21 20 16 15   4 3     0
	 *   +------------------------------------------------------+
741
742
743
	 * 0 |       RSS Hash /  |SPH| HDR_LEN  | RSV |Packet|  RSS |
	 *   | Packet   | IP     |   |          |     | Type | Type |
	 *   | Checksum | Ident  |   |          |     |      |      |
744
745
746
747
	 *   +------------------------------------------------------+
	 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
	 *   +------------------------------------------------------+
	 *   63       48 47    32 31            20 19               0
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
	 *
	 * 82599+ Advanced Receive Descriptor (Read) Format
	 *    63                                           1        0
	 *    +-----------------------------------------------------+
	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
	 *    +----------------------------------------------+------+
	 *  8 |       Header Buffer Address [63:1]           |  DD  |
	 *    +-----------------------------------------------------+
	 *
	 *
	 * 82599+ Advanced Receive Descriptor (Write-Back) Format
	 *
	 *   63       48 47    32 31  30      21 20 17 16   4 3     0
	 *   +------------------------------------------------------+
	 * 0 |RSS / Frag Checksum|SPH| HDR_LEN  |RSC- |Packet|  RSS |
	 *   |/ RTT / PCoE_PARAM |   |          | CNT | Type | Type |
	 *   |/ Flow Dir Flt ID  |   |          |     |      |      |
	 *   +------------------------------------------------------+
	 * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
	 *   +------------------------------------------------------+
	 *   63       48 47    32 31          20 19                 0
769
	 */
770

771
772
	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
773
774
775
		pr_info("------------------------------------\n");
		pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
		pr_info("------------------------------------\n");
Joe Perches's avatar
Joe Perches committed
776
		pr_info("%s%s%s\n",
777
778
			"R  [desc]      [ PktBuf     A0] ",
			"[  HeadBuf   DD] [bi->dma       ] [bi->skb       ] ",
Joe Perches's avatar
Joe Perches committed
779
780
			"<-- Adv Rx Read format");
		pr_info("%s%s%s\n",
781
782
			"RWB[desc]      [PcsmIpSHl PtRs] ",
			"[vl er S cks ln] ---------------- [bi->skb       ] ",
Joe Perches's avatar
Joe Perches committed
783
			"<-- Adv Rx Write-Back format");
784
785

		for (i = 0; i < rx_ring->count; i++) {
Joe Perches's avatar
Joe Perches committed
786
787
788
789
790
791
792
793
794
			const char *ring_desc;

			if (i == rx_ring->next_to_use)
				ring_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				ring_desc = " NTC";
			else
				ring_desc = "";

795
			rx_buffer_info = &rx_ring->rx_buffer_info[i];
796
			rx_desc = IXGBE_RX_DESC(rx_ring, i);
797
			u0 = (struct my_u0 *)rx_desc;
798
			if (rx_desc->wb.upper.length) {
799
				/* Descriptor Done */
Joe Perches's avatar
Joe Perches committed
800
801
				pr_info("RWB[0x%03X]     %016llX %016llX ---------------- %p%s\n",
					i,
Cathy Zhou's avatar
Cathy Zhou committed
802
803
					le64_to_cpu((__force __le64)u0->a),
					le64_to_cpu((__force __le64)u0->b),
Joe Perches's avatar
Joe Perches committed
804
805
					rx_buffer_info->skb,
					ring_desc);
806
			} else {
Joe Perches's avatar
Joe Perches committed
807
808
				pr_info("R  [0x%03X]     %016llX %016llX %016llX %p%s\n",
					i,
Cathy Zhou's avatar
Cathy Zhou committed
809
810
					le64_to_cpu((__force __le64)u0->a),
					le64_to_cpu((__force __le64)u0->b),
811
					(u64)rx_buffer_info->dma,
Joe Perches's avatar
Joe Perches committed
812
813
					rx_buffer_info->skb,
					ring_desc);
814

815
816
				if (netif_msg_pktdata(adapter) &&
				    rx_buffer_info->dma) {
817
818
					print_hex_dump(KERN_INFO, "",
					   DUMP_PREFIX_ADDRESS, 16, 1,
819
820
					   page_address(rx_buffer_info->page) +
						    rx_buffer_info->page_offset,
821
					   ixgbe_rx_bufsz(rx_ring), true);
822
823
824
825
826
827
				}
			}
		}
	}
}

828
829
830
831
832
833
834
static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
{
	u32 ctrl_ext;

	/* Let firmware take over control of h/w */
	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
835
			ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
836
837
838
839
840
841
842
843
844
}

static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
{
	u32 ctrl_ext;

	/* Let firmware know the driver has taken over */
	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
845
			ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
846
}
847

848
/**
849
850
851
852
853
854
855
856
 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 * @adapter: pointer to adapter struct
 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 * @queue: queue to map the corresponding interrupt to
 * @msix_vector: the vector to map to the corresponding queue
 *
 */
static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
857
			   u8 queue, u8 msix_vector)
858
859
{
	u32 ivar, index;
860
861
862
863
864
865
866
867
868
869
870
871
872
	struct ixgbe_hw *hw = &adapter->hw;
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
		if (direction == -1)
			direction = 0;
		index = (((direction * 64) + queue) >> 2) & 0x1F;
		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
		ivar &= ~(0xFF << (8 * (queue & 0x3)));
		ivar |= (msix_vector << (8 * (queue & 0x3)));
		IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
		break;
	case ixgbe_mac_82599EB:
Don Skidmore's avatar
Don Skidmore committed
873
	case ixgbe_mac_X540:
874
875
	case ixgbe_mac_X550:
	case ixgbe_mac_X550EM_x:
876
	case ixgbe_mac_x550em_a:
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
		if (direction == -1) {
			/* other causes */
			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
			index = ((queue & 1) * 8);
			ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
			ivar &= ~(0xFF << index);
			ivar |= (msix_vector << index);
			IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
			break;
		} else {
			/* tx or rx causes */
			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
			index = ((16 * (queue & 1)) + (8 * direction));
			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
			ivar &= ~(0xFF << index);
			ivar |= (msix_vector << index);
			IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
			break;
		}
	default:
		break;
	}
899
900
}

901
902
void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
			    u64 qmask)
903
904
905
{
	u32 mask;

906
907
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82598EB:
908
909
		mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
910
911
		break;
	case ixgbe_mac_82599EB:
Don Skidmore's avatar
Don Skidmore committed
912
	case ixgbe_mac_X540:
913
914
	case ixgbe_mac_X550:
	case ixgbe_mac_X550EM_x:
915
	case ixgbe_mac_x550em_a:
916
917
918
919
		mask = (qmask & 0xFFFFFFFF);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
		mask = (qmask >> 32);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
920
921
922
		break;
	default:
		break;
923
924
925
	}
}

926
static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
927
928
929
930
{
	struct ixgbe_hw *hw = &adapter->hw;
	struct ixgbe_hw_stats *hwstats = &adapter->stats;
	int i;
931
	u32 data;
932

933
934
935
	if ((hw->fc.current_mode != ixgbe_fc_full) &&
	    (hw->fc.current_mode != ixgbe_fc_rx_pause))
		return;
936

937
938
939
940
941
942
943
944
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
		break;
	default:
		data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
	}
	hwstats->lxoffrxc += data;
945

946
947
	/* refill credits (no tx hang) if we received xoff */
	if (!data)
948
		return;
949
950
951
952

	for (i = 0; i < adapter->num_tx_queues; i++)
		clear_bit(__IXGBE_HANG_CHECK_ARMED,
			  &adapter->tx_ring[i]->state);
953
954
955
956

	for (i = 0; i < adapter->num_xdp_queues; i++)
		clear_bit(__IXGBE_HANG_CHECK_ARMED,
			  &adapter->xdp_ring[i]->state);
957
958
959
960
961
962
963
}

static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	struct ixgbe_hw_stats *hwstats = &adapter->stats;
	u32 xoff[8] = {0};
964
	u8 tc;
965
966
967
968
969
970
971
972
	int i;
	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

	if (adapter->ixgbe_ieee_pfc)
		pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

	if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
		ixgbe_update_xoff_rx_lfc(adapter);
973
		return;
974
	}
975
976
977

	/* update stats for each tc, only valid with PFC enabled */
	for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
978
979
		u32 pxoffrxc;

980
981
		switch (hw->mac.type) {
		case ixgbe_mac_82598EB:
982
			pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
983
			break;
984
		default:
985
			pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
986
		}
987
988
989
990
		hwstats->pxoffrxc[i] += pxoffrxc;
		/* Get the TC for given UP */
		tc = netdev_get_prio_tc_map(adapter->netdev, i);
		xoff[tc] += pxoffrxc;
991
992
993
994
995
996
	}

	/* disarm tx queues that have received xoff frames */
	for (i = 0; i < adapter->num_tx_queues; i++) {
		struct ixgbe_ring *tx_ring = adapter->tx_ring[i];

997
		tc = tx_ring->dcb_tc;
998
999
		if (xoff[tc])
			clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
1000
	}
1001
1002
1003
1004
1005
1006
1007
1008

	for (i = 0; i < adapter->num_xdp_queues; i++) {
		struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];

		tc = xdp_ring->dcb_tc;
		if (xoff[tc])
			clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state);
	}
1009
1010
}

1011
static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
1012
{
1013
	return ring->stats.packets;
1014
1015
1016
1017
}

static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
{
1018
	unsigned int head, tail;
1019

1020
1021
	head = ring->next_to_clean;
	tail = ring->next_to_use;
1022

1023
	return ((head <= tail) ? tail : tail + ring->count) - head;
1024
1025
1026
1027
1028
1029
1030
1031
}

static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
{
	u32 tx_done = ixgbe_get_tx_completed(tx_ring);
	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
	u32 tx_pending = ixgbe_get_tx_pending(tx_ring);

1032
	clear_check_for_tx_hang(tx_ring);
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045

	/*
	 * Check for a hung queue, but be thorough. This verifies
	 * that a transmit has been completed since the previous
	 * check AND there is at least one packet pending. The
	 * ARMED bit is set to indicate a potential hang. The
	 * bit is cleared if a pause frame is received to remove
	 * false hang detection due to PFC or 802.3x frames. By
	 * requiring this to fail twice we avoid races with
	 * pfc clearing the ARMED bit and conditions where we
	 * run the check_tx_hang logic with a transmit completion
	 * pending but without time to complete it yet.
	 */
1046
	if (tx_done_old == tx_done && tx_pending)
1047
		/* make sure it is true for two checks in a row */
1048
1049
1050
1051
1052
1053
		return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
					&tx_ring->state);
	/* update completed stats and continue */
	tx_ring->tx_stats.tx_done_old = tx_done;
	/* reset the countdown */
	clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
1054

1055
	return false;
1056
1057
}

1058
1059
1060
1061
1062
1063
1064
1065
1066
/**
 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
 * @adapter: driver private struct
 **/
static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
{

	/* Do the reset outside of interrupt context */
	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1067
		set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
1068
		e_warn(drv, "initiating reset due to tx timeout\n");
1069
1070
1071
		ixgbe_service_event_schedule(adapter);
	}
}
1072

1073
1074
/**
 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
1075
1076
1077
 * @netdev: network interface device structure
 * @queue_index: Tx queue to set
 * @maxrate: desired maximum transmit bitrate
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
 **/
static int ixgbe_tx_maxrate(struct net_device *netdev,
			    int queue_index, u32 maxrate)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;
	u32 bcnrc_val = ixgbe_link_mbps(adapter);

	if (!maxrate)
		return 0;

	/* Calculate the rate factor values to set */
	bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
	bcnrc_val /= maxrate;

	/* clear everything but the rate factor */
	bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
	IXGBE_RTTBCNRC_RF_DEC_MASK;

	/* enable the rate scheduler */
	bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;

	IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
	IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);

	return 0;
}

1106
1107
/**
 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
1108
 * @q_vector: structure containing interrupt and ring information
1109
 * @tx_ring: tx ring to clean
1110
 * @napi_budget: Used to determine if we are in netpoll
1111
 **/
1112
static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
1113
			       struct ixgbe_ring *tx_ring, int napi_budget)
1114
{
1115
	struct ixgbe_adapter *adapter = q_vector->adapter;
1116
1117
	struct ixgbe_tx_buffer *tx_buffer;
	union ixgbe_adv_tx_desc *tx_desc;
Shannon Nelson's avatar
Shannon Nelson committed
1118
	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
1119
	unsigned int budget = q_vector->tx.work_limit;
1120
1121
1122
1123
	unsigned int i = tx_ring->next_to_clean;

	if (test_bit(__IXGBE_DOWN, &adapter->state))
		return true;
1124

1125
	tx_buffer = &tx_ring->tx_buffer_info[i];
1126
	tx_desc = IXGBE_TX_DESC(tx_ring, i);
1127
	i -= tx_ring->count;
1128

1129
	do {
1130
1131
1132
1133
1134
1135
		union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;

1136
		/* prevent any other reads prior to eop_desc */
1137
		smp_rmb();
1138

1139
1140
1141
		/* if DD is not set pending work has not been completed */
		if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
			break;
1142

1143
1144
		/* clear next_to_watch to prevent false hangs */
		tx_buffer->next_to_watch = NULL;
1145

1146
1147
1148
		/* update the statistics for this packet */
		total_bytes += tx_buffer->bytecount;
		total_packets += tx_buffer->gso_segs;
Shannon Nelson's avatar
Shannon Nelson committed
1149
1150
		if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
			total_ipsec++;
1151

1152
		/* free the skb */
1153
		if (ring_is_xdp(tx_ring))
1154
			xdp_return_frame(tx_buffer->xdpf);
1155
1156
		else
			napi_consume_skb(tx_buffer->skb, napi_budget);
1157

1158
1159
1160
1161
1162
1163
		/* unmap skb header data */
		dma_unmap_single(tx_ring->dev,
				 dma_unmap_addr(tx_buffer, dma),
				 dma_unmap_len(tx_buffer, len),
				 DMA_TO_DEVICE);

1164
		/* clear tx_buffer data */
1165
		dma_unmap_len_set(tx_buffer, len, 0);
1166

1167
1168
		/* unmap remaining buffers */
		while (tx_desc != eop_desc) {
1169
1170
			tx_buffer++;
			tx_desc++;
1171
			i++;
1172
1173
			if (unlikely(!i)) {
				i -= tx_ring->count;
1174
				tx_buffer = tx_ring->tx_buffer_info;
1175
				tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1176
			}
1177

1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199