pci.c 146 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
5
6
7
8
9
10
/*
 *	PCI Bus Services, see include/linux/pci.h for further explanation.
 *
 *	Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
 *	David Mosberger-Tang
 *
 *	Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
 */

11
#include <linux/acpi.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
13
#include <linux/kernel.h>
#include <linux/delay.h>
14
#include <linux/dmi.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/init.h>
16
17
#include <linux/of.h>
#include <linux/of_pci.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/pci.h>
19
#include <linux/pm.h>
20
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
22
#include <linux/module.h>
#include <linux/spinlock.h>
Tim Schmielau's avatar
Tim Schmielau committed
23
#include <linux/string.h>
24
#include <linux/log2.h>
Shaohua Li's avatar
Shaohua Li committed
25
#include <linux/pci-aspm.h>
26
#include <linux/pm_wakeup.h>
27
#include <linux/interrupt.h>
28
#include <linux/device.h>
29
#include <linux/pm_runtime.h>
30
#include <linux/pci_hotplug.h>
31
#include <linux/vmalloc.h>
32
#include <linux/pci-ats.h>
33
#include <asm/setup.h>
34
#include <asm/dma.h>
35
#include <linux/aer.h>
36
#include "pci.h"
Linus Torvalds's avatar
Linus Torvalds committed
37

Alan Stern's avatar
Alan Stern committed
38
39
40
41
42
const char *pci_power_names[] = {
	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
};
EXPORT_SYMBOL_GPL(pci_power_names);

43
44
45
46
47
48
int isa_dma_bridge_buggy;
EXPORT_SYMBOL(isa_dma_bridge_buggy);

int pci_pci_problems;
EXPORT_SYMBOL(pci_pci_problems);

49
50
unsigned int pci_pm_d3_delay;

51
52
53
54
55
56
57
58
59
60
61
62
63
static void pci_pme_list_scan(struct work_struct *work);

static LIST_HEAD(pci_pme_list);
static DEFINE_MUTEX(pci_pme_list_mutex);
static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);

struct pci_pme_device {
	struct list_head list;
	struct pci_dev *dev;
};

#define PME_TIMEOUT 1000 /* How long between PME checks */

64
65
66
67
68
69
70
static void pci_dev_d3_sleep(struct pci_dev *dev)
{
	unsigned int delay = dev->d3_delay;

	if (delay < pci_pm_d3_delay)
		delay = pci_pm_d3_delay;

71
72
	if (delay)
		msleep(delay);
73
}
Linus Torvalds's avatar
Linus Torvalds committed
74

75
76
77
78
#ifdef CONFIG_PCI_DOMAINS
int pci_domains_supported = 1;
#endif

79
80
81
82
83
84
#define DEFAULT_CARDBUS_IO_SIZE		(256)
#define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
/* pci=cbmemsize=nnM,cbiosize=nn can override this */
unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;

85
86
87
88
89
90
#define DEFAULT_HOTPLUG_IO_SIZE		(256)
#define DEFAULT_HOTPLUG_MEM_SIZE	(2*1024*1024)
/* pci=hpmemsize=nnM,hpiosize=nn can override this */
unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE;

91
92
93
#define DEFAULT_HOTPLUG_BUS_SIZE	1
unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;

94
enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
95

96
97
98
99
100
101
/*
 * The default CLS is used if arch didn't set CLS explicitly and not
 * all pci devices agree on the same value.  Arch can override either
 * the dfl or actual value as it sees fit.  Don't forget this is
 * measured in 32-bit words, not bytes.
 */
Bill Pemberton's avatar
Bill Pemberton committed
102
u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
103
104
u8 pci_cache_line_size;

105
106
107
108
109
110
/*
 * If we set up a device for bus mastering, we need to check the latency
 * timer as certain BIOSes forget to set it properly.
 */
unsigned int pcibios_max_latency = 255;

111
112
113
/* If set, the PCIe ARI capability will not be used. */
static bool pcie_ari_disabled;

114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
/* Disable bridge_d3 for all PCIe ports */
static bool pci_bridge_d3_disable;
/* Force bridge_d3 for all PCIe ports */
static bool pci_bridge_d3_force;

static int __init pcie_port_pm_setup(char *str)
{
	if (!strcmp(str, "off"))
		pci_bridge_d3_disable = true;
	else if (!strcmp(str, "force"))
		pci_bridge_d3_force = true;
	return 1;
}
__setup("pcie_port_pm=", pcie_port_pm_setup);

Linus Torvalds's avatar
Linus Torvalds committed
129
130
131
132
133
134
135
/**
 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
 * @bus: pointer to PCI bus structure to search
 *
 * Given a PCI bus, returns the highest PCI bus number present in the set
 * including the given PCI bus and its list of child PCI buses.
 */
136
unsigned char pci_bus_max_busnr(struct pci_bus *bus)
Linus Torvalds's avatar
Linus Torvalds committed
137
{
138
	struct pci_bus *tmp;
Linus Torvalds's avatar
Linus Torvalds committed
139
140
	unsigned char max, n;

141
	max = bus->busn_res.end;
142
143
	list_for_each_entry(tmp, &bus->children, node) {
		n = pci_bus_max_busnr(tmp);
Ryan Desfosses's avatar
Ryan Desfosses committed
144
		if (n > max)
Linus Torvalds's avatar
Linus Torvalds committed
145
146
147
148
			max = n;
	}
	return max;
}
149
EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
Linus Torvalds's avatar
Linus Torvalds committed
150

151
152
153
#ifdef CONFIG_HAS_IOMEM
void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
{
154
155
	struct resource *res = &pdev->resource[bar];

156
157
158
	/*
	 * Make sure the BAR is actually a memory resource, not an IO resource
	 */
159
	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
160
		dev_warn(&pdev->dev, "can't ioremap BAR %d: %pR\n", bar, res);
161
162
		return NULL;
	}
163
	return ioremap_nocache(res->start, resource_size(res));
164
165
}
EXPORT_SYMBOL_GPL(pci_ioremap_bar);
166
167
168
169
170
171
172
173
174
175
176
177
178
179

void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
{
	/*
	 * Make sure the BAR is actually a memory resource, not an IO resource
	 */
	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
		WARN_ON(1);
		return NULL;
	}
	return ioremap_wc(pci_resource_start(pdev, bar),
			  pci_resource_len(pdev, bar));
}
EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
180
181
#endif

182
183
184

static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
				   u8 pos, int cap, int *ttl)
185
186
{
	u8 id;
187
188
189
	u16 ent;

	pci_bus_read_config_byte(bus, devfn, pos, &pos);
190

191
	while ((*ttl)--) {
192
193
194
		if (pos < 0x40)
			break;
		pos &= ~3;
195
196
197
		pci_bus_read_config_word(bus, devfn, pos, &ent);

		id = ent & 0xff;
198
199
200
201
		if (id == 0xff)
			break;
		if (id == cap)
			return pos;
202
		pos = (ent >> 8);
203
204
205
206
	}
	return 0;
}

207
208
209
210
211
212
213
214
static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
			       u8 pos, int cap)
{
	int ttl = PCI_FIND_CAP_TTL;

	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
}

215
216
217
218
219
220
221
int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
{
	return __pci_find_next_cap(dev->bus, dev->devfn,
				   pos + PCI_CAP_LIST_NEXT, cap);
}
EXPORT_SYMBOL_GPL(pci_find_next_capability);

222
223
static int __pci_bus_find_cap_start(struct pci_bus *bus,
				    unsigned int devfn, u8 hdr_type)
Linus Torvalds's avatar
Linus Torvalds committed
224
225
226
227
228
229
230
231
232
233
{
	u16 status;

	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
	if (!(status & PCI_STATUS_CAP_LIST))
		return 0;

	switch (hdr_type) {
	case PCI_HEADER_TYPE_NORMAL:
	case PCI_HEADER_TYPE_BRIDGE:
234
		return PCI_CAPABILITY_LIST;
Linus Torvalds's avatar
Linus Torvalds committed
235
	case PCI_HEADER_TYPE_CARDBUS:
236
		return PCI_CB_CAPABILITY_LIST;
Linus Torvalds's avatar
Linus Torvalds committed
237
	}
238
239

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
240
241
242
}

/**
243
 * pci_find_capability - query for devices' capabilities
Linus Torvalds's avatar
Linus Torvalds committed
244
245
246
247
248
249
250
251
 * @dev: PCI device to query
 * @cap: capability code
 *
 * Tell if a device supports a given PCI capability.
 * Returns the address of the requested capability structure within the
 * device's PCI configuration space or 0 in case the device does not
 * support it.  Possible values for @cap:
 *
252
253
254
255
 *  %PCI_CAP_ID_PM           Power Management
 *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
 *  %PCI_CAP_ID_VPD          Vital Product Data
 *  %PCI_CAP_ID_SLOTID       Slot Identification
Linus Torvalds's avatar
Linus Torvalds committed
256
 *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
257
 *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
Linus Torvalds's avatar
Linus Torvalds committed
258
259
260
261
262
 *  %PCI_CAP_ID_PCIX         PCI-X
 *  %PCI_CAP_ID_EXP          PCI Express
 */
int pci_find_capability(struct pci_dev *dev, int cap)
{
263
264
265
266
267
268
269
	int pos;

	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
	if (pos)
		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);

	return pos;
Linus Torvalds's avatar
Linus Torvalds committed
270
}
271
EXPORT_SYMBOL(pci_find_capability);
Linus Torvalds's avatar
Linus Torvalds committed
272
273

/**
274
 * pci_bus_find_capability - query for devices' capabilities
Linus Torvalds's avatar
Linus Torvalds committed
275
276
277
278
279
 * @bus:   the PCI bus to query
 * @devfn: PCI device to query
 * @cap:   capability code
 *
 * Like pci_find_capability() but works for pci devices that do not have a
280
 * pci_dev structure set up yet.
Linus Torvalds's avatar
Linus Torvalds committed
281
282
283
284
285
286
287
 *
 * Returns the address of the requested capability structure within the
 * device's PCI configuration space or 0 in case the device does not
 * support it.
 */
int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
{
288
	int pos;
Linus Torvalds's avatar
Linus Torvalds committed
289
290
291
292
	u8 hdr_type;

	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);

293
294
295
296
297
	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f);
	if (pos)
		pos = __pci_find_next_cap(bus, devfn, pos, cap);

	return pos;
Linus Torvalds's avatar
Linus Torvalds committed
298
}
299
EXPORT_SYMBOL(pci_bus_find_capability);
Linus Torvalds's avatar
Linus Torvalds committed
300
301

/**
302
 * pci_find_next_ext_capability - Find an extended capability
Linus Torvalds's avatar
Linus Torvalds committed
303
 * @dev: PCI device to query
304
 * @start: address at which to start looking (0 to start at beginning of list)
Linus Torvalds's avatar
Linus Torvalds committed
305
306
 * @cap: capability code
 *
307
 * Returns the address of the next matching extended capability structure
Linus Torvalds's avatar
Linus Torvalds committed
308
 * within the device's PCI configuration space or 0 if the device does
309
310
 * not support it.  Some capabilities can occur several times, e.g., the
 * vendor-specific capability, and this provides a way to find them all.
Linus Torvalds's avatar
Linus Torvalds committed
311
 */
312
int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap)
Linus Torvalds's avatar
Linus Torvalds committed
313
314
{
	u32 header;
315
316
	int ttl;
	int pos = PCI_CFG_SPACE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
317

318
319
320
321
	/* minimum 8 bytes per capability */
	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;

	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
322
323
		return 0;

324
325
326
	if (start)
		pos = start;

Linus Torvalds's avatar
Linus Torvalds committed
327
328
329
330
331
332
333
334
335
336
337
	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
		return 0;

	/*
	 * If we have no capabilities, this is indicated by cap ID,
	 * cap version and next pointer all being 0.
	 */
	if (header == 0)
		return 0;

	while (ttl-- > 0) {
338
		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
Linus Torvalds's avatar
Linus Torvalds committed
339
340
341
			return pos;

		pos = PCI_EXT_CAP_NEXT(header);
342
		if (pos < PCI_CFG_SPACE_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
343
344
345
346
347
348
349
350
			break;

		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
			break;
	}

	return 0;
}
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);

/**
 * pci_find_ext_capability - Find an extended capability
 * @dev: PCI device to query
 * @cap: capability code
 *
 * Returns the address of the requested extended capability structure
 * within the device's PCI configuration space or 0 if the device does
 * not support it.  Possible values for @cap:
 *
 *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
 *  %PCI_EXT_CAP_ID_VC		Virtual Channel
 *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
 *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
 */
int pci_find_ext_capability(struct pci_dev *dev, int cap)
{
	return pci_find_next_ext_capability(dev, 0, cap);
}
371
EXPORT_SYMBOL_GPL(pci_find_ext_capability);
Linus Torvalds's avatar
Linus Torvalds committed
372

373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap)
{
	int rc, ttl = PCI_FIND_CAP_TTL;
	u8 cap, mask;

	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
		mask = HT_3BIT_CAP_MASK;
	else
		mask = HT_5BIT_CAP_MASK;

	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
				      PCI_CAP_ID_HT, &ttl);
	while (pos) {
		rc = pci_read_config_byte(dev, pos + 3, &cap);
		if (rc != PCIBIOS_SUCCESSFUL)
			return 0;

		if ((cap & mask) == ht_cap)
			return pos;

393
394
		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
					      pos + PCI_CAP_LIST_NEXT,
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
					      PCI_CAP_ID_HT, &ttl);
	}

	return 0;
}
/**
 * pci_find_next_ht_capability - query a device's Hypertransport capabilities
 * @dev: PCI device to query
 * @pos: Position from which to continue searching
 * @ht_cap: Hypertransport capability code
 *
 * To be used in conjunction with pci_find_ht_capability() to search for
 * all capabilities matching @ht_cap. @pos should always be a value returned
 * from pci_find_ht_capability().
 *
 * NB. To be 100% safe against broken PCI devices, the caller should take
 * steps to avoid an infinite loop.
 */
int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap)
{
	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
}
EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);

/**
 * pci_find_ht_capability - query a device's Hypertransport capabilities
 * @dev: PCI device to query
 * @ht_cap: Hypertransport capability code
 *
 * Tell if a device supports a given Hypertransport capability.
 * Returns an address within the device's PCI configuration space
 * or 0 in case the device does not support the request capability.
 * The address points to the PCI capability, of type PCI_CAP_ID_HT,
 * which has a Hypertransport capability matching @ht_cap.
 */
int pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
{
	int pos;

	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
	if (pos)
		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);

	return pos;
}
EXPORT_SYMBOL_GPL(pci_find_ht_capability);

Linus Torvalds's avatar
Linus Torvalds committed
442
443
444
445
446
447
/**
 * pci_find_parent_resource - return resource region of parent bus of given region
 * @dev: PCI device structure contains resources to be searched
 * @res: child resource record for which parent is sought
 *
 *  For given resource region of given device, return the resource
448
 *  region of parent bus the given region is contained in.
Linus Torvalds's avatar
Linus Torvalds committed
449
 */
Ryan Desfosses's avatar
Ryan Desfosses committed
450
451
struct resource *pci_find_parent_resource(const struct pci_dev *dev,
					  struct resource *res)
Linus Torvalds's avatar
Linus Torvalds committed
452
453
{
	const struct pci_bus *bus = dev->bus;
454
	struct resource *r;
Linus Torvalds's avatar
Linus Torvalds committed
455
456
	int i;

457
	pci_bus_for_each_resource(bus, r, i) {
Linus Torvalds's avatar
Linus Torvalds committed
458
459
		if (!r)
			continue;
460
		if (resource_contains(r, res)) {
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479

			/*
			 * If the window is prefetchable but the BAR is
			 * not, the allocator made a mistake.
			 */
			if (r->flags & IORESOURCE_PREFETCH &&
			    !(res->flags & IORESOURCE_PREFETCH))
				return NULL;

			/*
			 * If we're below a transparent bridge, there may
			 * be both a positively-decoded aperture and a
			 * subtractively-decoded region that contain the BAR.
			 * We want the positively-decoded one, so this depends
			 * on pci_bus_for_each_resource() giving us those
			 * first.
			 */
			return r;
		}
Linus Torvalds's avatar
Linus Torvalds committed
480
	}
481
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
482
}
483
EXPORT_SYMBOL(pci_find_parent_resource);
Linus Torvalds's avatar
Linus Torvalds committed
484

Mika Westerberg's avatar
Mika Westerberg committed
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
/**
 * pci_find_resource - Return matching PCI device resource
 * @dev: PCI device to query
 * @res: Resource to look for
 *
 * Goes over standard PCI resources (BARs) and checks if the given resource
 * is partially or fully contained in any of them. In that case the
 * matching resource is returned, %NULL otherwise.
 */
struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
{
	int i;

	for (i = 0; i < PCI_ROM_RESOURCE; i++) {
		struct resource *r = &dev->resource[i];

		if (r->start && resource_contains(r, res))
			return r;
	}

	return NULL;
}
EXPORT_SYMBOL(pci_find_resource);

509
510
511
512
513
514
515
516
517
/**
 * pci_find_pcie_root_port - return PCIe Root Port
 * @dev: PCI device to query
 *
 * Traverse up the parent chain and return the PCIe Root Port PCI Device
 * for a given PCI Device.
 */
struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev)
{
518
	struct pci_dev *bridge, *highest_pcie_bridge = dev;
519
520
521
522
523
524
525

	bridge = pci_upstream_bridge(dev);
	while (bridge && pci_is_pcie(bridge)) {
		highest_pcie_bridge = bridge;
		bridge = pci_upstream_bridge(bridge);
	}

526
527
	if (pci_pcie_type(highest_pcie_bridge) != PCI_EXP_TYPE_ROOT_PORT)
		return NULL;
528

529
	return highest_pcie_bridge;
530
531
532
}
EXPORT_SYMBOL(pci_find_pcie_root_port);

533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
/**
 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
 * @dev: the PCI device to operate on
 * @pos: config space offset of status word
 * @mask: mask of bit(s) to care about in status word
 *
 * Return 1 when mask bit(s) in status word clear, 0 otherwise.
 */
int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
{
	int i;

	/* Wait for Transaction Pending bit clean */
	for (i = 0; i < 4; i++) {
		u16 status;
		if (i)
			msleep((1 << (i - 1)) * 100);

		pci_read_config_word(dev, pos, &status);
		if (!(status & mask))
			return 1;
	}

	return 0;
}

559
/**
Wei Yang's avatar
Wei Yang committed
560
 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
561
562
563
564
565
 * @dev: PCI device to have its BARs restored
 *
 * Restore the BAR values for a given device, so as to make it
 * accessible by its driver.
 */
Ryan Desfosses's avatar
Ryan Desfosses committed
566
static void pci_restore_bars(struct pci_dev *dev)
567
{
568
	int i;
569

570
	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
571
		pci_update_resource(dev, i);
572
573
}

574
static const struct pci_platform_pm_ops *pci_platform_pm;
575

576
int pci_set_platform_pm(const struct pci_platform_pm_ops *ops)
577
{
578
	if (!ops->is_manageable || !ops->set_state  || !ops->get_state ||
579
	    !ops->choose_state  || !ops->set_wakeup || !ops->need_resume)
580
581
582
583
584
585
586
587
588
589
590
		return -EINVAL;
	pci_platform_pm = ops;
	return 0;
}

static inline bool platform_pci_power_manageable(struct pci_dev *dev)
{
	return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false;
}

static inline int platform_pci_set_power_state(struct pci_dev *dev,
Ryan Desfosses's avatar
Ryan Desfosses committed
591
					       pci_power_t t)
592
593
594
595
{
	return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
}

596
597
598
599
600
static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
{
	return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN;
}

601
602
603
604
605
static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
{
	return pci_platform_pm ?
			pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR;
}
Randy Dunlap's avatar
Randy Dunlap committed
606

607
static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
608
609
{
	return pci_platform_pm ?
610
			pci_platform_pm->set_wakeup(dev, enable) : -ENODEV;
611
612
}

613
614
615
616
617
static inline bool platform_pci_need_resume(struct pci_dev *dev)
{
	return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false;
}

Linus Torvalds's avatar
Linus Torvalds committed
618
/**
619
620
621
622
 * pci_raw_set_power_state - Use PCI PM registers to set the power state of
 *                           given PCI device
 * @dev: PCI device to handle.
 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
Linus Torvalds's avatar
Linus Torvalds committed
623
 *
624
625
626
627
628
629
 * RETURN VALUE:
 * -EINVAL if the requested state is invalid.
 * -EIO if device does not support PCI PM or its PM capabilities register has a
 * wrong version, or device doesn't support the requested state.
 * 0 if device already is in the requested state.
 * 0 if device's power state has been successfully changed.
Linus Torvalds's avatar
Linus Torvalds committed
630
 */
631
static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
Linus Torvalds's avatar
Linus Torvalds committed
632
{
633
	u16 pmcsr;
634
	bool need_restore = false;
Linus Torvalds's avatar
Linus Torvalds committed
635

636
637
638
639
	/* Check if we're already there */
	if (dev->current_state == state)
		return 0;

640
	if (!dev->pm_cap)
641
642
		return -EIO;

643
644
645
	if (state < PCI_D0 || state > PCI_D3hot)
		return -EINVAL;

Linus Torvalds's avatar
Linus Torvalds committed
646
	/* Validate current state:
647
	 * Can enter D0 from any state, but if we can only go deeper
Linus Torvalds's avatar
Linus Torvalds committed
648
649
	 * to sleep if we're already in a low power state
	 */
650
	if (state != PCI_D0 && dev->current_state <= PCI_D3cold
651
	    && dev->current_state > state) {
652
653
		dev_err(&dev->dev, "invalid power transition (from state %d to %d)\n",
			dev->current_state, state);
Linus Torvalds's avatar
Linus Torvalds committed
654
		return -EINVAL;
655
	}
Linus Torvalds's avatar
Linus Torvalds committed
656
657

	/* check if this device supports the desired state */
658
659
	if ((state == PCI_D1 && !dev->d1_support)
	   || (state == PCI_D2 && !dev->d2_support))
660
		return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
661

662
	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
663

664
	/* If we're (effectively) in D3, force entire word to 0.
Linus Torvalds's avatar
Linus Torvalds committed
665
666
667
	 * This doesn't affect PME_Status, disables PME_En, and
	 * sets PowerState to 0.
	 */
668
	switch (dev->current_state) {
669
670
671
672
673
674
	case PCI_D0:
	case PCI_D1:
	case PCI_D2:
		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
		pmcsr |= state;
		break;
675
676
	case PCI_D3hot:
	case PCI_D3cold:
677
678
	case PCI_UNKNOWN: /* Boot-up */
		if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot
679
		 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET))
680
			need_restore = true;
681
682
		/* Fall-through: force to D0 */
	default:
683
		pmcsr = 0;
684
		break;
Linus Torvalds's avatar
Linus Torvalds committed
685
686
687
	}

	/* enter specified state */
688
	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
Linus Torvalds's avatar
Linus Torvalds committed
689
690
691
692

	/* Mandatory power management transition delays */
	/* see PCI PM 1.1 5.6.1 table 18 */
	if (state == PCI_D3hot || dev->current_state == PCI_D3hot)
693
		pci_dev_d3_sleep(dev);
Linus Torvalds's avatar
Linus Torvalds committed
694
	else if (state == PCI_D2 || dev->current_state == PCI_D2)
695
		udelay(PCI_PM_D2_DELAY);
Linus Torvalds's avatar
Linus Torvalds committed
696

697
698
699
	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
	dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
	if (dev->current_state != state && printk_ratelimit())
700
701
		dev_info(&dev->dev, "Refused to change power state, currently in D%d\n",
			 dev->current_state);
702

703
704
	/*
	 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
705
706
707
708
709
710
711
712
713
714
715
716
717
718
	 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
	 * from D3hot to D0 _may_ perform an internal reset, thereby
	 * going to "D0 Uninitialized" rather than "D0 Initialized".
	 * For example, at least some versions of the 3c905B and the
	 * 3c556B exhibit this behaviour.
	 *
	 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
	 * devices in a D3hot state at boot.  Consequently, we need to
	 * restore at least the BARs so that the device will be
	 * accessible to its driver.
	 */
	if (need_restore)
		pci_restore_bars(dev);

719
	if (dev->bus->self)
Shaohua Li's avatar
Shaohua Li committed
720
721
		pcie_aspm_pm_state_change(dev->bus->self);

Linus Torvalds's avatar
Linus Torvalds committed
722
723
724
	return 0;
}

725
/**
726
 * pci_update_current_state - Read power state of given device and cache it
727
 * @dev: PCI device to handle.
728
 * @state: State to cache in case the device doesn't have the PM capability
729
730
731
732
733
734
735
 *
 * The power state is read from the PMCSR register, which however is
 * inaccessible in D3cold.  The platform firmware is therefore queried first
 * to detect accessibility of the register.  In case the platform firmware
 * reports an incorrect state or the device isn't power manageable by the
 * platform at all, we try to detect D3cold by testing accessibility of the
 * vendor ID in config space.
736
 */
737
void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
738
{
739
740
741
742
	if (platform_pci_get_power_state(dev) == PCI_D3cold ||
	    !pci_device_is_present(dev)) {
		dev->current_state = PCI_D3cold;
	} else if (dev->pm_cap) {
743
744
		u16 pmcsr;

745
		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
746
		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
747
748
	} else {
		dev->current_state = state;
749
750
751
	}
}

752
753
754
755
756
757
758
759
760
761
762
763
764
/**
 * pci_power_up - Put the given device into D0 forcibly
 * @dev: PCI device to power up
 */
void pci_power_up(struct pci_dev *dev)
{
	if (platform_pci_power_manageable(dev))
		platform_pci_set_power_state(dev, PCI_D0);

	pci_raw_set_power_state(dev, PCI_D0);
	pci_update_current_state(dev, PCI_D0);
}

765
766
767
768
769
770
771
772
773
774
775
776
777
/**
 * pci_platform_power_transition - Use platform to change device power state
 * @dev: PCI device to handle.
 * @state: State to put the device into.
 */
static int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
{
	int error;

	if (platform_pci_power_manageable(dev)) {
		error = platform_pci_set_power_state(dev, state);
		if (!error)
			pci_update_current_state(dev, state);
778
	} else
779
		error = -ENODEV;
780
781
782

	if (error && !dev->pm_cap) /* Fall back to PCI_D0 */
		dev->current_state = PCI_D0;
783
784
785
786

	return error;
}

787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
/**
 * pci_wakeup - Wake up a PCI device
 * @pci_dev: Device to handle.
 * @ign: ignored parameter
 */
static int pci_wakeup(struct pci_dev *pci_dev, void *ign)
{
	pci_wakeup_event(pci_dev);
	pm_request_resume(&pci_dev->dev);
	return 0;
}

/**
 * pci_wakeup_bus - Walk given bus and wake up devices on it
 * @bus: Top bus of the subtree to walk.
 */
static void pci_wakeup_bus(struct pci_bus *bus)
{
	if (bus)
		pci_walk_bus(bus, pci_wakeup, NULL);
}

809
810
811
812
813
814
815
/**
 * __pci_start_power_transition - Start power transition of a PCI device
 * @dev: PCI device to handle.
 * @state: State to put the device into.
 */
static void __pci_start_power_transition(struct pci_dev *dev, pci_power_t state)
{
816
	if (state == PCI_D0) {
817
		pci_platform_power_transition(dev, PCI_D0);
818
819
820
821
822
823
824
825
		/*
		 * Mandatory power management transition delays, see
		 * PCI Express Base Specification Revision 2.0 Section
		 * 6.6.1: Conventional Reset.  Do not delay for
		 * devices powered on/off by corresponding bridge,
		 * because have already delayed for the bridge.
		 */
		if (dev->runtime_d3cold) {
826
827
			if (dev->d3cold_delay)
				msleep(dev->d3cold_delay);
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
			/*
			 * When powering on a bridge from D3cold, the
			 * whole hierarchy may be powered on into
			 * D0uninitialized state, resume them to give
			 * them a chance to suspend again
			 */
			pci_wakeup_bus(dev->subordinate);
		}
	}
}

/**
 * __pci_dev_set_current_state - Set current state of a PCI device
 * @dev: Device to handle
 * @data: pointer to state to be set
 */
static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
{
	pci_power_t state = *(pci_power_t *)data;

	dev->current_state = state;
	return 0;
}

/**
 * __pci_bus_set_current_state - Walk given bus and set current state of devices
 * @bus: Top bus of the subtree to walk.
 * @state: state to be set
 */
static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
{
	if (bus)
		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
861
862
863
864
865
866
867
868
869
870
871
}

/**
 * __pci_complete_power_transition - Complete power transition of a PCI device
 * @dev: PCI device to handle.
 * @state: State to put the device into.
 *
 * This function should not be called directly by device drivers.
 */
int __pci_complete_power_transition(struct pci_dev *dev, pci_power_t state)
{
872
873
	int ret;

874
	if (state <= PCI_D0)
875
876
877
878
879
880
		return -EINVAL;
	ret = pci_platform_power_transition(dev, state);
	/* Power off the bridge may power off the whole hierarchy */
	if (!ret && state == PCI_D3cold)
		__pci_bus_set_current_state(dev->subordinate, PCI_D3cold);
	return ret;
881
882
883
}
EXPORT_SYMBOL_GPL(__pci_complete_power_transition);

884
885
886
887
888
/**
 * pci_set_power_state - Set the power state of a PCI device
 * @dev: PCI device to handle.
 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
 *
889
 * Transition a device to a new power state, using the platform firmware and/or
890
891
892
893
894
895
 * the device's PCI PM registers.
 *
 * RETURN VALUE:
 * -EINVAL if the requested state is invalid.
 * -EIO if device does not support PCI PM or its PM capabilities register has a
 * wrong version, or device doesn't support the requested state.
896
 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
897
 * 0 if device already is in the requested state.
898
 * 0 if the transition is to D3 but D3 is not supported.
899
900
901
902
 * 0 if device's power state has been successfully changed.
 */
int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
{
903
	int error;
904
905

	/* bound the state we're entering */
906
907
	if (state > PCI_D3cold)
		state = PCI_D3cold;
908
909
910
911
912
913
914
915
916
917
	else if (state < PCI_D0)
		state = PCI_D0;
	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
		/*
		 * If the device or the parent bridge do not support PCI PM,
		 * ignore the request if we're doing anything other than putting
		 * it into D0 (which would only happen on boot).
		 */
		return 0;

918
919
920
921
	/* Check if we're already there */
	if (dev->current_state == state)
		return 0;

922
923
	__pci_start_power_transition(dev, state);

924
925
	/* This device is quirked not to be put into D3, so
	   don't put it in D3 */
926
	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
927
		return 0;
928

929
930
931
932
933
934
	/*
	 * To put device in D3cold, we put device into D3hot in native
	 * way, then put device into D3cold with platform ops
	 */
	error = pci_raw_set_power_state(dev, state > PCI_D3hot ?
					PCI_D3hot : state);
935

936
937
	if (!__pci_complete_power_transition(dev, state))
		error = 0;
938
939
940

	return error;
}
941
EXPORT_SYMBOL(pci_set_power_state);
942

Linus Torvalds's avatar
Linus Torvalds committed
943
944
945
946
947
948
949
950
951
952
953
954
/**
 * pci_choose_state - Choose the power state of a PCI device
 * @dev: PCI device to be suspended
 * @state: target sleep state for the whole system. This is the value
 *	that is passed to suspend() function.
 *
 * Returns PCI power state suitable for given device and given system
 * message.
 */

pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
{
955
	pci_power_t ret;
956

957
	if (!dev->pm_cap)
Linus Torvalds's avatar
Linus Torvalds committed
958
959
		return PCI_D0;

960
961
962
	ret = platform_pci_choose_state(dev);
	if (ret != PCI_POWER_ERROR)
		return ret;
963
964
965
966
967

	switch (state.event) {
	case PM_EVENT_ON:
		return PCI_D0;
	case PM_EVENT_FREEZE:
968
969
	case PM_EVENT_PRETHAW:
		/* REVISIT both freeze and pre-thaw "should" use D0 */
970
	case PM_EVENT_SUSPEND:
971
	case PM_EVENT_HIBERNATE:
972
		return PCI_D3hot;
Linus Torvalds's avatar
Linus Torvalds committed
973
	default:
974
975
		dev_info(&dev->dev, "unrecognized suspend event %d\n",
			 state.event);
Linus Torvalds's avatar
Linus Torvalds committed
976
977
978
979
980
981
		BUG();
	}
	return PCI_D0;
}
EXPORT_SYMBOL(pci_choose_state);

982
983
#define PCI_EXP_SAVE_REGS	7

984
985
static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
						       u16 cap, bool extended)
986
987
988
{
	struct pci_cap_saved_state *tmp;

989
	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
990
		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
991
992
993
994
995
			return tmp;
	}
	return NULL;
}

996
997
998
999
1000
struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
{
	return _pci_find_saved_cap(dev, cap, false);
}