pci-driver.c 41.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2
/*
3
4
 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
 * (C) Copyright 2007 Novell Inc.
Linus Torvalds's avatar
Linus Torvalds committed
5
6
7
8
9
10
 */

#include <linux/pci.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/device.h>
11
#include <linux/mempolicy.h>
Tim Schmielau's avatar
Tim Schmielau committed
12
13
#include <linux/string.h>
#include <linux/slab.h>
14
#include <linux/sched.h>
15
#include <linux/cpu.h>
16
#include <linux/pm_runtime.h>
17
#include <linux/suspend.h>
18
#include <linux/kexec.h>
19
20
#include <linux/of_device.h>
#include <linux/acpi.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
#include "pci.h"
22
#include "pcie/portdrv.h"
Linus Torvalds's avatar
Linus Torvalds committed
23

24
25
26
27
struct pci_dynid {
	struct list_head node;
	struct pci_device_id id;
};
Linus Torvalds's avatar
Linus Torvalds committed
28

Tejun Heo's avatar
Tejun Heo committed
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
/**
 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
 * @drv: target pci driver
 * @vendor: PCI vendor ID
 * @device: PCI device ID
 * @subvendor: PCI subvendor ID
 * @subdevice: PCI subdevice ID
 * @class: PCI class
 * @class_mask: PCI class mask
 * @driver_data: private driver data
 *
 * Adds a new dynamic pci device ID to this driver and causes the
 * driver to probe for all devices again.  @drv must have been
 * registered prior to calling this function.
 *
 * CONTEXT:
 * Does GFP_KERNEL allocation.
 *
 * RETURNS:
 * 0 on success, -errno on failure.
 */
int pci_add_dynid(struct pci_driver *drv,
		  unsigned int vendor, unsigned int device,
		  unsigned int subvendor, unsigned int subdevice,
		  unsigned int class, unsigned int class_mask,
		  unsigned long driver_data)
{
	struct pci_dynid *dynid;

	dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
	if (!dynid)
		return -ENOMEM;

	dynid->id.vendor = vendor;
	dynid->id.device = device;
	dynid->id.subvendor = subvendor;
	dynid->id.subdevice = subdevice;
	dynid->id.class = class;
	dynid->id.class_mask = class_mask;
	dynid->id.driver_data = driver_data;
69

Tejun Heo's avatar
Tejun Heo committed
70
71
72
73
	spin_lock(&drv->dynids.lock);
	list_add_tail(&dynid->node, &drv->dynids.list);
	spin_unlock(&drv->dynids.lock);

74
	return driver_attach(&drv->driver);
Tejun Heo's avatar
Tejun Heo committed
75
}
76
EXPORT_SYMBOL_GPL(pci_add_dynid);
Tejun Heo's avatar
Tejun Heo committed
77
78
79
80

static void pci_free_dynids(struct pci_driver *drv)
{
	struct pci_dynid *dynid, *n;
81

Tejun Heo's avatar
Tejun Heo committed
82
83
84
85
86
87
88
89
	spin_lock(&drv->dynids.lock);
	list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
		list_del(&dynid->node);
		kfree(dynid);
	}
	spin_unlock(&drv->dynids.lock);
}

Linus Torvalds's avatar
Linus Torvalds committed
90
/**
Tejun Heo's avatar
Tejun Heo committed
91
 * store_new_id - sysfs frontend to pci_add_dynid()
Randy Dunlap's avatar
Randy Dunlap committed
92
93
94
 * @driver: target device driver
 * @buf: buffer for scanning device ID data
 * @count: input size
Linus Torvalds's avatar
Linus Torvalds committed
95
 *
Tejun Heo's avatar
Tejun Heo committed
96
 * Allow PCI IDs to be added to an existing driver via sysfs.
Linus Torvalds's avatar
Linus Torvalds committed
97
 */
98
static ssize_t new_id_store(struct device_driver *driver, const char *buf,
Ryan Desfosses's avatar
Ryan Desfosses committed
99
			    size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
100
101
{
	struct pci_driver *pdrv = to_pci_driver(driver);
102
	const struct pci_device_id *ids = pdrv->id_table;
Ryan Desfosses's avatar
Ryan Desfosses committed
103
104
105
106
	__u32 vendor, device, subvendor = PCI_ANY_ID,
		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
	unsigned long driver_data = 0;
	int fields = 0;
107
	int retval = 0;
Linus Torvalds's avatar
Linus Torvalds committed
108

109
	fields = sscanf(buf, "%x %x %x %x %x %x %lx",
Linus Torvalds's avatar
Linus Torvalds committed
110
111
			&vendor, &device, &subvendor, &subdevice,
			&class, &class_mask, &driver_data);
112
	if (fields < 2)
Linus Torvalds's avatar
Linus Torvalds committed
113
114
		return -EINVAL;

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
	if (fields != 7) {
		struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
		if (!pdev)
			return -ENOMEM;

		pdev->vendor = vendor;
		pdev->device = device;
		pdev->subsystem_vendor = subvendor;
		pdev->subsystem_device = subdevice;
		pdev->class = class;

		if (pci_match_id(pdrv->id_table, pdev))
			retval = -EEXIST;

		kfree(pdev);

		if (retval)
			return retval;
	}

135
136
	/* Only accept driver_data values that match an existing id_table
	   entry */
137
138
139
140
141
142
143
144
	if (ids) {
		retval = -EINVAL;
		while (ids->vendor || ids->subvendor || ids->class_mask) {
			if (driver_data == ids->driver_data) {
				retval = 0;
				break;
			}
			ids++;
145
		}
146
147
		if (retval)	/* No match */
			return retval;
148
149
	}

Tejun Heo's avatar
Tejun Heo committed
150
151
	retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
			       class, class_mask, driver_data);
152
153
	if (retval)
		return retval;
Linus Torvalds's avatar
Linus Torvalds committed
154
155
	return count;
}
156
static DRIVER_ATTR_WO(new_id);
Linus Torvalds's avatar
Linus Torvalds committed
157

Chris Wright's avatar
Chris Wright committed
158
159
160
161
162
163
164
165
/**
 * store_remove_id - remove a PCI device ID from this driver
 * @driver: target device driver
 * @buf: buffer for scanning device ID data
 * @count: input size
 *
 * Removes a dynamic pci device ID to this driver.
 */
166
static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
Ryan Desfosses's avatar
Ryan Desfosses committed
167
			       size_t count)
Chris Wright's avatar
Chris Wright committed
168
169
170
171
172
173
{
	struct pci_dynid *dynid, *n;
	struct pci_driver *pdrv = to_pci_driver(driver);
	__u32 vendor, device, subvendor = PCI_ANY_ID,
		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
	int fields = 0;
174
	size_t retval = -ENODEV;
Chris Wright's avatar
Chris Wright committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191

	fields = sscanf(buf, "%x %x %x %x %x %x",
			&vendor, &device, &subvendor, &subdevice,
			&class, &class_mask);
	if (fields < 2)
		return -EINVAL;

	spin_lock(&pdrv->dynids.lock);
	list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
		struct pci_device_id *id = &dynid->id;
		if ((id->vendor == vendor) &&
		    (id->device == device) &&
		    (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
		    (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
		    !((id->class ^ class) & class_mask)) {
			list_del(&dynid->node);
			kfree(dynid);
192
			retval = count;
Chris Wright's avatar
Chris Wright committed
193
194
195
196
197
			break;
		}
	}
	spin_unlock(&pdrv->dynids.lock);

198
	return retval;
Chris Wright's avatar
Chris Wright committed
199
}
200
static DRIVER_ATTR_WO(remove_id);
Chris Wright's avatar
Chris Wright committed
201

202
203
204
205
static struct attribute *pci_drv_attrs[] = {
	&driver_attr_new_id.attr,
	&driver_attr_remove_id.attr,
	NULL,
206
};
207
ATTRIBUTE_GROUPS(pci_drv);
Chris Wright's avatar
Chris Wright committed
208

Linus Torvalds's avatar
Linus Torvalds committed
209
/**
210
 * pci_match_id - See if a pci device matches a given pci_id table
Linus Torvalds's avatar
Linus Torvalds committed
211
 * @ids: array of PCI device id structures to search in
212
213
 * @dev: the PCI device structure to match against.
 *
Linus Torvalds's avatar
Linus Torvalds committed
214
 * Used by a driver to check whether a PCI device present in the
215
 * system is in its list of supported devices.  Returns the matching
Linus Torvalds's avatar
Linus Torvalds committed
216
 * pci_device_id structure or %NULL if there is no match.
217
 *
Randy Dunlap's avatar
Randy Dunlap committed
218
 * Deprecated, don't use this as it will not catch any dynamic ids
219
 * that a driver might want to check for.
Linus Torvalds's avatar
Linus Torvalds committed
220
 */
221
222
const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
					 struct pci_dev *dev)
Linus Torvalds's avatar
Linus Torvalds committed
223
{
224
225
226
227
228
229
	if (ids) {
		while (ids->vendor || ids->subvendor || ids->class_mask) {
			if (pci_match_one_device(ids, dev))
				return ids;
			ids++;
		}
Linus Torvalds's avatar
Linus Torvalds committed
230
231
232
	}
	return NULL;
}
233
EXPORT_SYMBOL(pci_match_id);
Linus Torvalds's avatar
Linus Torvalds committed
234

235
236
237
238
239
240
241
static const struct pci_device_id pci_device_id_any = {
	.vendor = PCI_ANY_ID,
	.device = PCI_ANY_ID,
	.subvendor = PCI_ANY_ID,
	.subdevice = PCI_ANY_ID,
};

Linus Torvalds's avatar
Linus Torvalds committed
242
/**
Randy Dunlap's avatar
Randy Dunlap committed
243
 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
244
 * @drv: the PCI driver to match against
245
 * @dev: the PCI device structure to match against
246
247
248
249
 *
 * Used by a driver to check whether a PCI device present in the
 * system is in its list of supported devices.  Returns the matching
 * pci_device_id structure or %NULL if there is no match.
Linus Torvalds's avatar
Linus Torvalds committed
250
 */
251
252
static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
						    struct pci_dev *dev)
253
254
{
	struct pci_dynid *dynid;
255
256
257
258
259
	const struct pci_device_id *found_id = NULL;

	/* When driver_override is set, only bind to the matching driver */
	if (dev->driver_override && strcmp(dev->driver_override, drv->name))
		return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
260

261
	/* Look at the dynamic ids first, before the static ones */
262
263
264
	spin_lock(&drv->dynids.lock);
	list_for_each_entry(dynid, &drv->dynids.list, node) {
		if (pci_match_one_device(&dynid->id, dev)) {
265
266
			found_id = &dynid->id;
			break;
267
		}
Linus Torvalds's avatar
Linus Torvalds committed
268
	}
269
	spin_unlock(&drv->dynids.lock);
270

271
272
273
274
275
276
277
278
	if (!found_id)
		found_id = pci_match_id(drv->id_table, dev);

	/* driver_override will always match, send a dummy id */
	if (!found_id && dev->driver_override)
		found_id = &pci_device_id_any;

	return found_id;
Linus Torvalds's avatar
Linus Torvalds committed
279
280
}

281
282
283
284
285
286
287
288
289
struct drv_dev_and_id {
	struct pci_driver *drv;
	struct pci_dev *dev;
	const struct pci_device_id *id;
};

static long local_pci_probe(void *_ddi)
{
	struct drv_dev_and_id *ddi = _ddi;
290
291
292
	struct pci_dev *pci_dev = ddi->dev;
	struct pci_driver *pci_drv = ddi->drv;
	struct device *dev = &pci_dev->dev;
293
294
	int rc;

295
296
297
298
	/*
	 * Unbound PCI devices are always put in D0, regardless of
	 * runtime PM status.  During probe, the device is set to
	 * active and the usage count is incremented.  If the driver
299
300
301
302
	 * supports runtime PM, it should call pm_runtime_put_noidle(),
	 * or any other runtime PM helper function decrementing the usage
	 * count, in its probe routine and pm_runtime_get_noresume() in
	 * its remove routine.
303
	 */
304
305
306
	pm_runtime_get_sync(dev);
	pci_dev->driver = pci_drv;
	rc = pci_drv->probe(pci_dev, ddi->id);
307
308
309
	if (!rc)
		return rc;
	if (rc < 0) {
310
311
		pci_dev->driver = NULL;
		pm_runtime_put_sync(dev);
312
		return rc;
313
	}
314
315
316
317
318
319
	/*
	 * Probe function should return < 0 for failure, 0 for success
	 * Treat values > 0 as success, but warn.
	 */
	dev_warn(dev, "Driver probe function unexpectedly returned %d\n", rc);
	return 0;
320
321
}

322
323
324
325
326
327
328
329
330
static bool pci_physfn_is_probed(struct pci_dev *dev)
{
#ifdef CONFIG_PCI_IOV
	return dev->is_virtfn && dev->physfn->is_probed;
#else
	return false;
#endif
}

331
332
333
static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
			  const struct pci_device_id *id)
{
334
	int error, node, cpu;
335
336
	struct drv_dev_and_id ddi = { drv, dev, id };

337
338
339
340
341
	/*
	 * Execute driver initialization on node where the device is
	 * attached.  This way the driver likely allocates its local memory
	 * on the right node.
	 */
342
	node = dev_to_node(&dev->dev);
343
344
345
	dev->is_probed = 1;

	cpu_hotplug_disable();
346
347

	/*
348
349
	 * Prevent nesting work_on_cpu() for the case where a Virtual Function
	 * device is probed from work_on_cpu() of the Physical device.
350
	 */
351
352
353
354
	if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
	    pci_physfn_is_probed(dev))
		cpu = nr_cpu_ids;
	else
355
		cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
356
357
358
359

	if (cpu < nr_cpu_ids)
		error = work_on_cpu(cpu, local_pci_probe, &ddi);
	else
360
		error = local_pci_probe(&ddi);
361

362
363
	dev->is_probed = 0;
	cpu_hotplug_enable();
364
365
366
	return error;
}

Linus Torvalds's avatar
Linus Torvalds committed
367
/**
368
 * __pci_device_probe - check if a driver wants to claim a specific PCI device
Randy Dunlap's avatar
Randy Dunlap committed
369
370
 * @drv: driver to call to check if it wants the PCI device
 * @pci_dev: PCI device being probed
371
 *
Randy Dunlap's avatar
Randy Dunlap committed
372
 * returns 0 on success, else error.
Linus Torvalds's avatar
Linus Torvalds committed
373
374
 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
 */
Ryan Desfosses's avatar
Ryan Desfosses committed
375
static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
376
377
{
	const struct pci_device_id *id;
Linus Torvalds's avatar
Linus Torvalds committed
378
379
380
	int error = 0;

	if (!pci_dev->driver && drv->probe) {
381
382
383
384
		error = -ENODEV;

		id = pci_match_device(drv, pci_dev);
		if (id)
385
			error = pci_call_probe(drv, pci_dev, id);
Linus Torvalds's avatar
Linus Torvalds committed
386
387
388
389
	}
	return error;
}

390
391
392
393
394
395
396
397
398
int __weak pcibios_alloc_irq(struct pci_dev *dev)
{
	return 0;
}

void __weak pcibios_free_irq(struct pci_dev *dev)
{
}

399
400
401
402
403
404
405
406
407
408
409
410
#ifdef CONFIG_PCI_IOV
static inline bool pci_device_can_probe(struct pci_dev *pdev)
{
	return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe);
}
#else
static inline bool pci_device_can_probe(struct pci_dev *pdev)
{
	return true;
}
#endif

Ryan Desfosses's avatar
Ryan Desfosses committed
411
static int pci_device_probe(struct device *dev)
Linus Torvalds's avatar
Linus Torvalds committed
412
{
413
414
415
416
	int error;
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = to_pci_driver(dev->driver);

417
418
	pci_assign_irq(pci_dev);

419
420
421
	error = pcibios_alloc_irq(pci_dev);
	if (error < 0)
		return error;
Linus Torvalds's avatar
Linus Torvalds committed
422
423

	pci_dev_get(pci_dev);
424
425
426
427
428
429
	if (pci_device_can_probe(pci_dev)) {
		error = __pci_device_probe(drv, pci_dev);
		if (error) {
			pcibios_free_irq(pci_dev);
			pci_dev_put(pci_dev);
		}
430
	}
Linus Torvalds's avatar
Linus Torvalds committed
431
432
433
434

	return error;
}

Ryan Desfosses's avatar
Ryan Desfosses committed
435
static int pci_device_remove(struct device *dev)
Linus Torvalds's avatar
Linus Torvalds committed
436
{
Ryan Desfosses's avatar
Ryan Desfosses committed
437
438
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = pci_dev->driver;
Linus Torvalds's avatar
Linus Torvalds committed
439
440

	if (drv) {
441
442
		if (drv->remove) {
			pm_runtime_get_sync(dev);
Linus Torvalds's avatar
Linus Torvalds committed
443
			drv->remove(pci_dev);
444
445
			pm_runtime_put_noidle(dev);
		}
446
		pcibios_free_irq(pci_dev);
Linus Torvalds's avatar
Linus Torvalds committed
447
		pci_dev->driver = NULL;
448
		pci_iov_remove(pci_dev);
Linus Torvalds's avatar
Linus Torvalds committed
449
450
	}

451
	/* Undo the runtime PM settings in local_pci_probe() */
452
	pm_runtime_put_sync(dev);
453

454
455
456
457
458
459
460
	/*
	 * If the device is still on, set the power state as "unknown",
	 * since it might change by the next time we load the driver.
	 */
	if (pci_dev->current_state == PCI_D0)
		pci_dev->current_state = PCI_UNKNOWN;

Linus Torvalds's avatar
Linus Torvalds committed
461
462
463
464
	/*
	 * We would love to complain here if pci_dev->is_enabled is set, that
	 * the driver should have called pci_disable_device(), but the
	 * unfortunate fact is there are too many odd BIOS and bridge setups
465
	 * that don't like drivers doing that all of the time.
Linus Torvalds's avatar
Linus Torvalds committed
466
467
468
469
470
471
472
473
	 * Oh well, we can dream of sane hardware when we sleep, no matter how
	 * horrible the crap we have to deal with is when we are awake...
	 */

	pci_dev_put(pci_dev);
	return 0;
}

474
475
476
477
478
static void pci_device_shutdown(struct device *dev)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = pci_dev->driver;

479
480
	pm_runtime_resume(dev);

481
482
	if (drv && drv->shutdown)
		drv->shutdown(pci_dev);
483

484
	/*
485
486
487
488
489
	 * If this is a kexec reboot, turn off Bus Master bit on the
	 * device to tell it to not continue to do DMA. Don't touch
	 * devices in D3cold or unknown states.
	 * If it is not a kexec reboot, firmware will hit the PCI
	 * devices with big hammer and stop their DMA any way.
490
	 */
491
	if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
492
		pci_clear_master(pci_dev);
493
494
}

Rafael J. Wysocki's avatar
Rafael J. Wysocki committed
495
#ifdef CONFIG_PM
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512

/* Auxiliary functions used for system resume and run-time resume. */

/**
 * pci_restore_standard_config - restore standard config registers of PCI device
 * @pci_dev: PCI device to handle
 */
static int pci_restore_standard_config(struct pci_dev *pci_dev)
{
	pci_update_current_state(pci_dev, PCI_UNKNOWN);

	if (pci_dev->current_state != PCI_D0) {
		int error = pci_set_power_state(pci_dev, PCI_D0);
		if (error)
			return error;
	}

513
	pci_restore_state(pci_dev);
514
	pci_pme_restore(pci_dev);
515
	return 0;
516
517
}

518
519
520
521
#endif

#ifdef CONFIG_PM_SLEEP

522
523
static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
{
524
525
	pci_power_up(pci_dev);
	pci_restore_state(pci_dev);
526
	pci_pme_restore(pci_dev);
527
528
529
	pci_fixup_device(pci_fixup_resume_early, pci_dev);
}

530
531
532
/*
 * Default "suspend" method for devices that have no driver provided suspend,
 * or not even a driver at all (second part).
533
 */
534
static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
535
536
537
538
539
540
541
542
543
{
	/*
	 * mark its power state as "unknown", since we don't know if
	 * e.g. the BIOS will change its device state when we suspend.
	 */
	if (pci_dev->current_state == PCI_D0)
		pci_dev->current_state = PCI_UNKNOWN;
}

544
545
546
547
/*
 * Default "resume" method for devices that have no driver provided resume,
 * or not even a driver at all (second part).
 */
548
static int pci_pm_reenable_device(struct pci_dev *pci_dev)
549
550
551
{
	int retval;

552
553
554
555
556
557
558
559
560
561
562
563
564
	/* if the device was enabled before suspend, reenable */
	retval = pci_reenable_device(pci_dev);
	/*
	 * if the device was busmaster before the suspend, make it busmaster
	 * again
	 */
	if (pci_dev->is_busmaster)
		pci_set_master(pci_dev);

	return retval;
}

static int pci_legacy_suspend(struct device *dev, pm_message_t state)
Linus Torvalds's avatar
Linus Torvalds committed
565
{
Ryan Desfosses's avatar
Ryan Desfosses committed
566
567
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = pci_dev->driver;
568

569
	if (drv && drv->suspend) {
570
		pci_power_t prev = pci_dev->current_state;
571
		int error;
572

573
574
575
576
		error = drv->suspend(pci_dev, state);
		suspend_report_result(drv->suspend, error);
		if (error)
			return error;
577

578
		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
579
580
581
582
583
		    && pci_dev->current_state != PCI_UNKNOWN) {
			WARN_ONCE(pci_dev->current_state != prev,
				"PCI PM: Device state not saved by %pF\n",
				drv->suspend);
		}
584
	}
585
586
587

	pci_fixup_device(pci_fixup_suspend, pci_dev);

588
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
589
590
}

591
static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
Linus Torvalds's avatar
Linus Torvalds committed
592
{
Ryan Desfosses's avatar
Ryan Desfosses committed
593
594
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = pci_dev->driver;
Linus Torvalds's avatar
Linus Torvalds committed
595
596

	if (drv && drv->suspend_late) {
597
598
599
		pci_power_t prev = pci_dev->current_state;
		int error;

600
601
		error = drv->suspend_late(pci_dev, state);
		suspend_report_result(drv->suspend_late, error);
602
603
604
605
606
607
608
609
		if (error)
			return error;

		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
		    && pci_dev->current_state != PCI_UNKNOWN) {
			WARN_ONCE(pci_dev->current_state != prev,
				"PCI PM: Device state not saved by %pF\n",
				drv->suspend_late);
610
			goto Fixup;
611
		}
Linus Torvalds's avatar
Linus Torvalds committed
612
	}
613
614
615
616
617
618

	if (!pci_dev->state_saved)
		pci_save_state(pci_dev);

	pci_pm_set_unknown_state(pci_dev);

619
620
621
Fixup:
	pci_fixup_device(pci_fixup_suspend_late, pci_dev);

622
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
623
}
Linus Torvalds's avatar
Linus Torvalds committed
624

625
626
static int pci_legacy_resume_early(struct device *dev)
{
Ryan Desfosses's avatar
Ryan Desfosses committed
627
628
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = pci_dev->driver;
629

630
631
	return drv && drv->resume_early ?
			drv->resume_early(pci_dev) : 0;
632
633
}

634
static int pci_legacy_resume(struct device *dev)
Linus Torvalds's avatar
Linus Torvalds committed
635
{
Ryan Desfosses's avatar
Ryan Desfosses committed
636
637
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct pci_driver *drv = pci_dev->driver;
Linus Torvalds's avatar
Linus Torvalds committed
638

639
640
	pci_fixup_device(pci_fixup_resume, pci_dev);

641
642
	return drv && drv->resume ?
			drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
Linus Torvalds's avatar
Linus Torvalds committed
643
644
}

645
646
/* Auxiliary functions used by the new power management framework */

647
static void pci_pm_default_resume(struct pci_dev *pci_dev)
648
{
649
	pci_fixup_device(pci_fixup_resume, pci_dev);
650
	pci_enable_wake(pci_dev, PCI_D0, false);
651
652
}

653
static void pci_pm_default_suspend(struct pci_dev *pci_dev)
654
{
655
	/* Disable non-bridge devices without PM support */
656
	if (!pci_has_subordinate(pci_dev))
657
		pci_disable_enabled_device(pci_dev);
658
659
}

660
661
662
static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
{
	struct pci_driver *drv = pci_dev->driver;
663
	bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
664
		|| drv->resume_early);
665
666
667
668
669
670

	/*
	 * Legacy PM support is used by default, so warn if the new framework is
	 * supported as well.  Drivers are supposed to support either the
	 * former, or the latter, but not both at the same time.
	 */
671
672
	WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
		drv->name, pci_dev->vendor, pci_dev->device);
673
674

	return ret;
675
676
}

677
678
/* New power management framework */

679
680
681
682
static int pci_pm_prepare(struct device *dev)
{
	struct device_driver *drv = dev->driver;

683
684
	if (drv && drv->pm && drv->pm->prepare) {
		int error = drv->pm->prepare(dev);
685
		if (error < 0)
686
			return error;
687
688
689

		if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
			return 0;
690
691
	}
	return pci_dev_keep_suspended(to_pci_dev(dev));
692
693
}

694
695
static void pci_pm_complete(struct device *dev)
{
696
697
698
699
700
701
	struct pci_dev *pci_dev = to_pci_dev(dev);

	pci_dev_complete_resume(pci_dev);
	pm_generic_complete(dev);

	/* Resume device if platform firmware has put it in reset-power-on */
702
	if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
703
704
705
706
707
708
		pci_power_t pre_sleep_state = pci_dev->current_state;

		pci_update_current_state(pci_dev, pci_dev->current_state);
		if (pci_dev->current_state < pre_sleep_state)
			pm_request_resume(dev);
	}
709
}
710

711
712
713
#else /* !CONFIG_PM_SLEEP */

#define pci_pm_prepare	NULL
714
#define pci_pm_complete	NULL
715
716
717

#endif /* !CONFIG_PM_SLEEP */

718
#ifdef CONFIG_SUSPEND
719
720
721
722
723
724
725
726
static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
{
	/*
	 * Some BIOSes forget to clear Root PME Status bits after system
	 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
	 * Clear those bits now just in case (shouldn't hurt).
	 */
	if (pci_is_pcie(pci_dev) &&
727
728
	    (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
	     pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
729
730
		pcie_clear_root_pme_status(pci_dev);
}
731
732
733
734

static int pci_pm_suspend(struct device *dev)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
735
	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
736

737
738
	if (pci_has_legacy_pm_support(pci_dev))
		return pci_legacy_suspend(dev, PMSG_SUSPEND);
739

740
741
	if (!pm) {
		pci_pm_default_suspend(pci_dev);
742
		return 0;
743
744
	}

745
	/*
746
747
748
749
750
751
752
753
754
755
756
	 * PCI devices suspended at run time may need to be resumed at this
	 * point, because in general it may be necessary to reconfigure them for
	 * system suspend.  Namely, if the device is expected to wake up the
	 * system from the sleep state, it may have to be reconfigured for this
	 * purpose, or if the device is not expected to wake up the system from
	 * the sleep state, it should be prevented from signaling wakeup events
	 * going forward.
	 *
	 * Also if the driver of the device does not indicate that its system
	 * suspend callbacks can cope with runtime-suspended devices, it is
	 * better to resume the device from runtime suspend here.
757
	 */
758
	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
759
	    !pci_dev_keep_suspended(pci_dev)) {
760
		pm_runtime_resume(dev);
761
762
		pci_dev->state_saved = false;
	}
763

764
765
766
767
	if (pm->suspend) {
		pci_power_t prev = pci_dev->current_state;
		int error;

768
769
		error = pm->suspend(dev);
		suspend_report_result(pm->suspend, error);
770
771
772
		if (error)
			return error;

773
		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
774
775
776
777
778
		    && pci_dev->current_state != PCI_UNKNOWN) {
			WARN_ONCE(pci_dev->current_state != prev,
				"PCI PM: State of device not saved by %pF\n",
				pm->suspend);
		}
779
	}
780

781
	return 0;
782
783
}

784
785
786
787
788
789
790
791
792
793
static int pci_pm_suspend_late(struct device *dev)
{
	if (dev_pm_smart_suspend_and_suspended(dev))
		return 0;

	pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));

	return pm_generic_suspend_late(dev);
}

794
static int pci_pm_suspend_noirq(struct device *dev)
795
{
796
	struct pci_dev *pci_dev = to_pci_dev(dev);
797
	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
798

799
800
	if (dev_pm_smart_suspend_and_suspended(dev)) {
		dev->power.may_skip_resume = true;
801
		return 0;
802
	}
803

804
805
806
	if (pci_has_legacy_pm_support(pci_dev))
		return pci_legacy_suspend_late(dev, PMSG_SUSPEND);

807
808
	if (!pm) {
		pci_save_state(pci_dev);
809
		goto Fixup;
810
	}
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825

	if (pm->suspend_noirq) {
		pci_power_t prev = pci_dev->current_state;
		int error;

		error = pm->suspend_noirq(dev);
		suspend_report_result(pm->suspend_noirq, error);
		if (error)
			return error;

		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
		    && pci_dev->current_state != PCI_UNKNOWN) {
			WARN_ONCE(pci_dev->current_state != prev,
				"PCI PM: State of device not saved by %pF\n",
				pm->suspend_noirq);
826
			goto Fixup;
827
		}
828
829
	}

830
831
	if (!pci_dev->state_saved) {
		pci_save_state(pci_dev);
832
		if (pci_power_manageable(pci_dev))
833
834
			pci_prepare_to_sleep(pci_dev);
	}
835

836
837
838
	dev_dbg(dev, "PCI PM: Suspend power state: %s\n",
		pci_power_name(pci_dev->current_state));

839
840
	pci_pm_set_unknown_state(pci_dev);

841
842
843
844
845
846
847
848
849
850
851
852
	/*
	 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
	 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
	 * hasn't been quiesced and tries to turn it off.  If the controller
	 * is already in D3, this can hang or cause memory corruption.
	 *
	 * Since the value of the COMMAND register doesn't matter once the
	 * device has been suspended, we can safely set it to 0 here.
	 */
	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
		pci_write_config_word(pci_dev, PCI_COMMAND, 0);

853
854
855
Fixup:
	pci_fixup_device(pci_fixup_suspend_late, pci_dev);

856
857
858
859
860
861
862
863
864
865
	/*
	 * If the target system sleep state is suspend-to-idle, it is sufficient
	 * to check whether or not the device's wakeup settings are good for
	 * runtime PM.  Otherwise, the pm_resume_via_firmware() check will cause
	 * pci_pm_complete() to take care of fixing up the device's state
	 * anyway, if need be.
	 */
	dev->power.may_skip_resume = device_may_wakeup(dev) ||
					!device_can_wakeup(dev);

866
	return 0;
867
}
Linus Torvalds's avatar
Linus Torvalds committed
868

869
static int pci_pm_resume_noirq(struct device *dev)
870
871
872
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct device_driver *drv = dev->driver;
873
	int error = 0;
874

875
876
877
	if (dev_pm_may_skip_resume(dev))
		return 0;

878
879
880
881
882
883
884
885
	/*
	 * Devices with DPM_FLAG_SMART_SUSPEND may be left in runtime suspend
	 * during system suspend, so update their runtime PM status to "active"
	 * as they are going to be put into D0 shortly.
	 */
	if (dev_pm_smart_suspend_and_suspended(dev))
		pm_runtime_set_active(dev);

886
	pci_pm_default_resume_early(pci_dev);
887

888
	if (pci_has_legacy_pm_support(pci_dev))
889
		return pci_legacy_resume_early(dev);
890

891
892
	pcie_pme_root_status_cleanup(pci_dev);

893
894
	if (drv && drv->pm && drv->pm->resume_noirq)
		error = drv->pm->resume_noirq(dev);
895
896
897
898

	return error;
}

899
static int pci_pm_resume(struct device *dev)
900
{
901
	struct pci_dev *pci_dev = to_pci_dev(dev);
902
	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
903
904
	int error = 0;

905
906
907
908
909
910
911
	/*
	 * This is necessary for the suspend error path in which resume is
	 * called without restoring the standard config registers of the device.
	 */
	if (pci_dev->state_saved)
		pci_restore_standard_config(pci_dev);

912
	if (pci_has_legacy_pm_support(pci_dev))
913
		return pci_legacy_resume(dev);
914

915
	pci_pm_default_resume(pci_dev);
916

917
918
919
920
921
922
	if (pm) {
		if (pm->resume)
			error = pm->resume(dev);
	} else {
		pci_pm_reenable_device(pci_dev);
	}
923

924
	return error;
925
926
927
928
929
}

#else /* !CONFIG_SUSPEND */

#define pci_pm_suspend		NULL
930
#define pci_pm_suspend_late	NULL
931
932
933
934
935
936
#define pci_pm_suspend_noirq	NULL
#define pci_pm_resume		NULL
#define pci_pm_resume_noirq	NULL

#endif /* !CONFIG_SUSPEND */

937
#ifdef CONFIG_HIBERNATE_CALLBACKS
938

939
940
941
942
943
944
945

/*
 * pcibios_pm_ops - provide arch-specific hooks when a PCI device is doing
 * a hibernate transition
 */
struct dev_pm_ops __weak pcibios_pm_ops;

946
947
948
static int pci_pm_freeze(struct device *dev)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
949
	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
950

951
952
	if (pci_has_legacy_pm_support(pci_dev))
		return pci_legacy_suspend(dev, PMSG_FREEZE);
953

954
955
956
	if (!pm) {
		pci_pm_default_suspend(pci_dev);
		return 0;
957
958
	}

959
960
961
962
963
964
	/*
	 * This used to be done in pci_pm_prepare() for all devices and some
	 * drivers may depend on it, so do it here.  Ideally, runtime-suspended
	 * devices should not be touched during freeze/thaw transitions,
	 * however.
	 */
965
	if (!dev_pm_smart_suspend_and_suspended(dev)) {
966
		pm_runtime_resume(dev);
967
968
		pci_dev->state_saved = false;
	}
969

970
971
972
973
974
975
976
977
978
979
	if (pm->freeze) {
		int error;

		error = pm->freeze(dev);
		suspend_report_result(pm->freeze, error);
		if (error)
			return error;
	}

	return 0;
980
981
}

982
983
984
985
986
static int pci_pm_freeze_late(struct device *dev)
{
	if (dev_pm_smart_suspend_and_suspended(dev))
		return 0;

987
	return pm_generic_freeze_late(dev);
988
989
}

990
991
static int pci_pm_freeze_noirq(struct device *dev)
{
992
	struct pci_dev *pci_dev = to_pci_dev(dev);
993
	struct device_driver *drv = dev->driver;
994

995
996
997
	if (dev_pm_smart_suspend_and_suspended(dev))
		return 0;

998
999
1000
	if (pci_has_legacy_pm_support(pci_dev))
		return pci_legacy_suspend_late(dev, PMSG_FREEZE);

1001
	if (drv && drv->pm && drv->pm->freeze_noirq) {
1002
1003
		int error;

1004
1005
		error = drv->pm->freeze_noirq(dev);
		suspend_report_result(drv->pm->freeze_noirq, error);
1006
1007
		if (error)
			return error;
1008
1009
	}

1010
1011
	if (!pci_dev->state_saved)
		pci_save_state(pci_dev);
1012

1013
1014
	pci_pm_set_unknown_state(pci_dev);

1015
1016
1017
	if (pcibios_pm_ops.freeze_noirq)
		return pcibios_pm_ops.freeze_noirq(dev);

1018
	return 0;
1019
1020
}

1021
static int pci_pm_thaw_noirq(struct device *dev)
1022
{
1023
	struct pci_dev *pci_dev = to_pci_dev(dev);
1024
1025
1026
	struct device_driver *drv = dev->driver;
	int error = 0;

1027
1028
1029
1030
1031
1032
	/*
	 * If the device is in runtime suspend, the code below may not work
	 * correctly with it, so skip that code and make the PM core skip all of
	 * the subsequent "thaw" callbacks for the device.
	 */
	if (dev_pm_smart_suspend_and_suspended(dev)) {
1033
		dev_pm_skip_next_resume_phases(dev);
1034
1035
1036
		return 0;
	}

1037
1038
1039
1040
1041
1042
	if (pcibios_pm_ops.thaw_noirq) {
		error = pcibios_pm_ops.thaw_noirq(dev);
		if (error)
			return error;
	}

1043
	if (pci_has_legacy_pm_support(pci_dev))
1044
		return pci_legacy_resume_early(dev);
1045

1046
1047
1048
1049
1050
1051
	/*
	 * pci_restore_state() requires the device to be in D0 (because of MSI
	 * restoration among other things), so force it into D0 in case the
	 * driver's "freeze" callbacks put it into a low-power state directly.
	 */
	pci_set_power_state(pci_dev, PCI_D0);
1052
	pci_restore_state(pci_dev);
1053

1054
1055
	if (drv && drv->pm && drv->pm->thaw_noirq)
		error = drv->pm->thaw_noirq(dev);
1056
1057
1058
1059

	return error;
}

1060
static int pci_pm_thaw(struct device *dev)
1061
{
1062
	struct pci_dev *pci_dev = to_pci_dev(dev);
1063
	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1064
1065
	int error = 0;