pci.c 14.1 KB
Newer Older
1
2
3
4
#include "kvm/virtio-pci.h"

#include "kvm/ioport.h"
#include "kvm/kvm.h"
5
#include "kvm/kvm-cpu.h"
6
7
8
#include "kvm/virtio-pci-dev.h"
#include "kvm/irq.h"
#include "kvm/virtio.h"
9
#include "kvm/ioeventfd.h"
10

11
#include <sys/ioctl.h>
12
#include <linux/virtio_pci.h>
13
#include <linux/byteorder.h>
14
15
#include <string.h>

16
17
18
static void virtio_pci__ioevent_callback(struct kvm *kvm, void *param)
{
	struct virtio_pci_ioevent_param *ioeventfd = param;
19
	struct virtio_pci *vpci = ioeventfd->vdev->virtio;
20

21
	ioeventfd->vdev->ops->notify_vq(kvm, vpci->dev, ioeventfd->vq);
22
23
}

24
static int virtio_pci__init_ioeventfd(struct kvm *kvm, struct virtio_device *vdev, u32 vq)
25
26
{
	struct ioevent ioevent;
27
	struct virtio_pci *vpci = vdev->virtio;
28
29
	int r, flags = 0;
	int fd;
30
31

	vpci->ioeventfds[vq] = (struct virtio_pci_ioevent_param) {
32
		.vdev		= vdev,
33
34
35
36
37
38
39
40
41
42
		.vq		= vq,
	};

	ioevent = (struct ioevent) {
		.fn		= virtio_pci__ioevent_callback,
		.fn_ptr		= &vpci->ioeventfds[vq],
		.datamatch	= vq,
		.fn_kvm		= kvm,
	};

43
44
45
46
47
48
49
50
51
52
	/*
	 * Vhost will poll the eventfd in host kernel side, otherwise we
	 * need to poll in userspace.
	 */
	if (!vdev->use_vhost)
		flags |= IOEVENTFD_FLAG_USER_POLL;

	/* ioport */
	ioevent.io_addr	= vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY;
	ioevent.io_len	= sizeof(u16);
53
	ioevent.fd	= fd = eventfd(0, 0);
54
	r = ioeventfd__add_event(&ioevent, flags | IOEVENTFD_FLAG_PIO);
55
56
	if (r)
		return r;
57

58
59
	/* mmio */
	ioevent.io_addr	= vpci->mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY;
60
	ioevent.io_len	= sizeof(u16);
61
	ioevent.fd	= eventfd(0, 0);
62
63
64
	r = ioeventfd__add_event(&ioevent, flags);
	if (r)
		goto free_ioport_evt;
65

66
	if (vdev->ops->notify_vq_eventfd)
67
		vdev->ops->notify_vq_eventfd(kvm, vpci->dev, vq, fd);
68
	return 0;
69
70
71
72

free_ioport_evt:
	ioeventfd__del_event(vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq);
	return r;
73
74
}

75
76
77
78
79
80
81
82
83
84
static void virtio_pci_exit_vq(struct kvm *kvm, struct virtio_device *vdev,
			       int vq)
{
	struct virtio_pci *vpci = vdev->virtio;

	ioeventfd__del_event(vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq);
	ioeventfd__del_event(vpci->mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq);
	virtio_exit_vq(kvm, vdev, vpci->dev, vq);
}

Sasha Levin's avatar
Sasha Levin committed
85
86
static inline bool virtio_pci__msix_enabled(struct virtio_pci *vpci)
{
87
	return vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_ENABLE);
Sasha Levin's avatar
Sasha Levin committed
88
89
}

90
static bool virtio_pci__specific_io_in(struct kvm *kvm, struct virtio_device *vdev, u16 port,
91
92
93
					void *data, int size, int offset)
{
	u32 config_offset;
94
	struct virtio_pci *vpci = vdev->virtio;
Sasha Levin's avatar
Sasha Levin committed
95
96
	int type = virtio__get_dev_specific_field(offset - 20,
							virtio_pci__msix_enabled(vpci),
97
							&config_offset);
98
99
100
101
102
103
104
105
106
107
108
109
110
111
	if (type == VIRTIO_PCI_O_MSIX) {
		switch (offset) {
		case VIRTIO_MSI_CONFIG_VECTOR:
			ioport__write16(data, vpci->config_vector);
			break;
		case VIRTIO_MSI_QUEUE_VECTOR:
			ioport__write16(data, vpci->vq_vector[vpci->queue_selector]);
			break;
		};

		return true;
	} else if (type == VIRTIO_PCI_O_CONFIG) {
		u8 cfg;

112
		cfg = vdev->ops->get_config(kvm, vpci->dev)[config_offset];
113
114
115
116
117
118
119
		ioport__write8(data, cfg);
		return true;
	}

	return false;
}

120
static bool virtio_pci__io_in(struct ioport *ioport, struct kvm_cpu *vcpu, u16 port, void *data, int size)
121
122
123
{
	unsigned long offset;
	bool ret = true;
124
	struct virtio_device *vdev;
125
	struct virtio_pci *vpci;
126
	struct virt_queue *vq;
127
	struct kvm *kvm;
128
129
	u32 val;

130
	kvm = vcpu->kvm;
131
132
	vdev = ioport->priv;
	vpci = vdev->virtio;
133
	offset = port - vpci->port_addr;
134
135
136

	switch (offset) {
	case VIRTIO_PCI_HOST_FEATURES:
137
		val = vdev->ops->get_host_features(kvm, vpci->dev);
138
139
140
		ioport__write32(data, val);
		break;
	case VIRTIO_PCI_QUEUE_PFN:
141
142
		vq = vdev->ops->get_vq(kvm, vpci->dev, vpci->queue_selector);
		ioport__write32(data, vq->pfn);
143
144
		break;
	case VIRTIO_PCI_QUEUE_NUM:
145
		val = vdev->ops->get_size_vq(kvm, vpci->dev, vpci->queue_selector);
146
		ioport__write16(data, val);
147
148
149
150
151
152
		break;
	case VIRTIO_PCI_STATUS:
		ioport__write8(data, vpci->status);
		break;
	case VIRTIO_PCI_ISR:
		ioport__write8(data, vpci->isr);
153
		kvm__irq_line(kvm, vpci->legacy_irq_line, VIRTIO_IRQ_LOW);
154
155
156
		vpci->isr = VIRTIO_IRQ_LOW;
		break;
	default:
157
		ret = virtio_pci__specific_io_in(kvm, vdev, port, data, size, offset);
158
159
160
161
162
163
		break;
	};

	return ret;
}

164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
static void update_msix_map(struct virtio_pci *vpci,
			    struct msix_table *msix_entry, u32 vecnum)
{
	u32 gsi, i;

	/* Find the GSI number used for that vector */
	if (vecnum == vpci->config_vector) {
		gsi = vpci->config_gsi;
	} else {
		for (i = 0; i < VIRTIO_PCI_MAX_VQ; i++)
			if (vpci->vq_vector[i] == vecnum)
				break;
		if (i == VIRTIO_PCI_MAX_VQ)
			return;
		gsi = vpci->gsis[i];
	}

	if (gsi == 0)
		return;

	msix_entry = &msix_entry[vecnum];
	irq__update_msix_route(vpci->kvm, gsi, &msix_entry->msg);
}

188
static bool virtio_pci__specific_io_out(struct kvm *kvm, struct virtio_device *vdev, u16 port,
189
190
					void *data, int size, int offset)
{
191
	struct virtio_pci *vpci = vdev->virtio;
192
193
	u32 config_offset, vec;
	int gsi;
Sasha Levin's avatar
Sasha Levin committed
194
	int type = virtio__get_dev_specific_field(offset - 20, virtio_pci__msix_enabled(vpci),
195
							&config_offset);
196
197
198
199
	if (type == VIRTIO_PCI_O_MSIX) {
		switch (offset) {
		case VIRTIO_MSI_CONFIG_VECTOR:
			vec = vpci->config_vector = ioport__read16(data);
200
201
			if (vec == VIRTIO_MSI_NO_VECTOR)
				break;
202

203
			gsi = irq__add_msix_route(kvm,
204
205
						  &vpci->msix_table[vec].msg,
						  vpci->dev_hdr.dev_num << 3);
206
207
208
209
210
211
212
213
214
215
216
217
218
219
			/*
			 * We don't need IRQ routing if we can use
			 * MSI injection via the KVM_SIGNAL_MSI ioctl.
			 */
			if (gsi == -ENXIO &&
			    vpci->features & VIRTIO_PCI_F_SIGNAL_MSI)
				break;

			if (gsi < 0) {
				die("failed to configure MSIs");
				break;
			}

			vpci->config_gsi = gsi;
220
			break;
Sasha Levin's avatar
Sasha Levin committed
221
		case VIRTIO_MSI_QUEUE_VECTOR:
222
223
			vec = ioport__read16(data);
			vpci->vq_vector[vpci->queue_selector] = vec;
224

225
226
227
			if (vec == VIRTIO_MSI_NO_VECTOR)
				break;

228
			gsi = irq__add_msix_route(kvm,
229
230
						  &vpci->msix_table[vec].msg,
						  vpci->dev_hdr.dev_num << 3);
231
232
233
234
235
236
			/*
			 * We don't need IRQ routing if we can use
			 * MSI injection via the KVM_SIGNAL_MSI ioctl.
			 */
			if (gsi == -ENXIO &&
			    vpci->features & VIRTIO_PCI_F_SIGNAL_MSI)
237
				break;
238
239
240
241
242
243

			if (gsi < 0) {
				die("failed to configure MSIs");
				break;
			}

244
			vpci->gsis[vpci->queue_selector] = gsi;
245
246
			if (vdev->ops->notify_vq_gsi)
				vdev->ops->notify_vq_gsi(kvm, vpci->dev,
247
248
							 vpci->queue_selector,
							 gsi);
249
250
251
252
253
			break;
		};

		return true;
	} else if (type == VIRTIO_PCI_O_CONFIG) {
254
		vdev->ops->get_config(kvm, vpci->dev)[config_offset] = *(u8 *)data;
255
256
257
258
259
260
261

		return true;
	}

	return false;
}

262
static bool virtio_pci__io_out(struct ioport *ioport, struct kvm_cpu *vcpu, u16 port, void *data, int size)
263
264
265
{
	unsigned long offset;
	bool ret = true;
266
	struct virtio_device *vdev;
267
	struct virtio_pci *vpci;
268
	struct kvm *kvm;
269
270
	u32 val;

271
	kvm = vcpu->kvm;
272
273
	vdev = ioport->priv;
	vpci = vdev->virtio;
274
	offset = port - vpci->port_addr;
275
276
277
278

	switch (offset) {
	case VIRTIO_PCI_GUEST_FEATURES:
		val = ioport__read32(data);
279
		virtio_set_guest_features(kvm, vdev, vpci->dev, val);
280
281
282
		break;
	case VIRTIO_PCI_QUEUE_PFN:
		val = ioport__read32(data);
283
284
285
286
287
288
289
290
291
		if (val) {
			virtio_pci__init_ioeventfd(kvm, vdev,
						   vpci->queue_selector);
			vdev->ops->init_vq(kvm, vpci->dev, vpci->queue_selector,
					   1 << VIRTIO_PCI_QUEUE_ADDR_SHIFT,
					   VIRTIO_PCI_VRING_ALIGN, val);
		} else {
			virtio_pci_exit_vq(kvm, vdev, vpci->queue_selector);
		}
292
293
		break;
	case VIRTIO_PCI_QUEUE_SEL:
Sasha Levin's avatar
Sasha Levin committed
294
		vpci->queue_selector = ioport__read16(data);
295
296
		break;
	case VIRTIO_PCI_QUEUE_NOTIFY:
Sasha Levin's avatar
Sasha Levin committed
297
		val = ioport__read16(data);
298
		vdev->ops->notify_vq(kvm, vpci->dev, val);
299
300
		break;
	case VIRTIO_PCI_STATUS:
Sasha Levin's avatar
Sasha Levin committed
301
		vpci->status = ioport__read8(data);
302
303
		if (!vpci->status) /* Sample endianness on reset */
			vdev->endian = kvm_cpu__get_endianness(vcpu);
304
		virtio_notify_status(kvm, vdev, vpci->dev, vpci->status);
305
306
		break;
	default:
307
		ret = virtio_pci__specific_io_out(kvm, vdev, port, data, size, offset);
308
309
310
311
312
313
314
315
316
317
318
		break;
	};

	return ret;
}

static struct ioport_operations virtio_pci__io_ops = {
	.io_in	= virtio_pci__io_in,
	.io_out	= virtio_pci__io_out,
};

319
320
static void virtio_pci__msix_mmio_callback(struct kvm_cpu *vcpu,
					   u64 addr, u8 *data, u32 len,
321
					   u8 is_write, void *ptr)
322
323
{
	struct virtio_pci *vpci = ptr;
324
325
326
	struct msix_table *table;
	int vecnum;
	size_t offset;
327

328
	if (addr > vpci->msix_io_block + PCI_IO_SIZE) {
329
330
331
332
		if (is_write)
			return;
		table  = (struct msix_table *)&vpci->msix_pba;
		offset = addr - (vpci->msix_io_block + PCI_IO_SIZE);
333
	} else {
334
335
		table  = vpci->msix_table;
		offset = addr - vpci->msix_io_block;
336
	}
337
338
339
340
341
342
343
344
345
	vecnum = offset / sizeof(struct msix_table);
	offset = offset % sizeof(struct msix_table);

	if (!is_write) {
		memcpy(data, (void *)&table[vecnum] + offset, len);
		return;
	}

	memcpy((void *)&table[vecnum] + offset, data, len);
Sasha Levin's avatar
Sasha Levin committed
346

347
348
349
	/* Did we just update the address or payload? */
	if (offset < offsetof(struct msix_table, ctrl))
		update_msix_map(vpci, table, vecnum);
Sasha Levin's avatar
Sasha Levin committed
350
351
}

352
353
static void virtio_pci__signal_msi(struct kvm *kvm, struct virtio_pci *vpci,
				   int vec)
354
355
356
357
358
359
360
{
	struct kvm_msi msi = {
		.address_lo = vpci->msix_table[vec].msg.address_lo,
		.address_hi = vpci->msix_table[vec].msg.address_hi,
		.data = vpci->msix_table[vec].msg.data,
	};

361
362
363
364
365
	if (kvm->msix_needs_devid) {
		msi.flags = KVM_MSI_VALID_DEVID;
		msi.devid = vpci->dev_hdr.dev_num << 3;
	}

Jean-Philippe Brucker's avatar
Jean-Philippe Brucker committed
366
	irq__signal_msi(kvm, &msi);
367
368
}

369
int virtio_pci__signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq)
370
{
371
	struct virtio_pci *vpci = vdev->virtio;
Sasha Levin's avatar
Sasha Levin committed
372
	int tbl = vpci->vq_vector[vq];
373

374
	if (virtio_pci__msix_enabled(vpci) && tbl != VIRTIO_MSI_NO_VECTOR) {
375
376
		if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) ||
		    vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
Sasha Levin's avatar
Sasha Levin committed
377
378
379
380
381

			vpci->msix_pba |= 1 << tbl;
			return 0;
		}

382
383
384
385
		if (vpci->features & VIRTIO_PCI_F_SIGNAL_MSI)
			virtio_pci__signal_msi(kvm, vpci, vpci->vq_vector[vq]);
		else
			kvm__irq_trigger(kvm, vpci->gsis[vq]);
Sasha Levin's avatar
Sasha Levin committed
386
	} else {
Sasha Levin's avatar
Sasha Levin committed
387
		vpci->isr = VIRTIO_IRQ_HIGH;
388
		kvm__irq_trigger(kvm, vpci->legacy_irq_line);
Sasha Levin's avatar
Sasha Levin committed
389
	}
390
391
392
	return 0;
}

393
int virtio_pci__signal_config(struct kvm *kvm, struct virtio_device *vdev)
394
{
395
	struct virtio_pci *vpci = vdev->virtio;
Sasha Levin's avatar
Sasha Levin committed
396
397
	int tbl = vpci->config_vector;

398
	if (virtio_pci__msix_enabled(vpci) && tbl != VIRTIO_MSI_NO_VECTOR) {
399
400
		if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) ||
		    vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
Sasha Levin's avatar
Sasha Levin committed
401
402
403
404
405

			vpci->msix_pba |= 1 << tbl;
			return 0;
		}

406
		if (vpci->features & VIRTIO_PCI_F_SIGNAL_MSI)
407
			virtio_pci__signal_msi(kvm, vpci, tbl);
408
409
		else
			kvm__irq_trigger(kvm, vpci->config_gsi);
Sasha Levin's avatar
Sasha Levin committed
410
411
	} else {
		vpci->isr = VIRTIO_PCI_ISR_CONFIG;
412
		kvm__irq_trigger(kvm, vpci->legacy_irq_line);
Sasha Levin's avatar
Sasha Levin committed
413
	}
414
415
416
417

	return 0;
}

418
419
static void virtio_pci__io_mmio_callback(struct kvm_cpu *vcpu,
					 u64 addr, u8 *data, u32 len,
420
421
422
423
424
425
					 u8 is_write, void *ptr)
{
	struct virtio_pci *vpci = ptr;
	int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
	u16 port = vpci->port_addr + (addr & (IOPORT_SIZE - 1));

426
	kvm__emulate_io(vcpu, port, data, direction, len, 1);
427
428
}

429
430
int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
		     int device_id, int subsys_id, int class)
431
{
432
	struct virtio_pci *vpci = vdev->virtio;
433
	int r;
434

435
	vpci->kvm = kvm;
436
437
	vpci->dev = dev;

438
	r = ioport__register(kvm, IOPORT_EMPTY, &virtio_pci__io_ops, IOPORT_SIZE, vdev);
439
440
	if (r < 0)
		return r;
441
	vpci->port_addr = (u16)r;
442

443
444
445
	vpci->mmio_addr = pci_get_io_space_block(IOPORT_SIZE);
	r = kvm__register_mmio(kvm, vpci->mmio_addr, IOPORT_SIZE, false,
			       virtio_pci__io_mmio_callback, vpci);
446
447
	if (r < 0)
		goto free_ioport;
448

449
450
451
452
453
454
	vpci->msix_io_block = pci_get_io_space_block(PCI_IO_SIZE * 2);
	r = kvm__register_mmio(kvm, vpci->msix_io_block, PCI_IO_SIZE * 2, false,
			       virtio_pci__msix_mmio_callback, vpci);
	if (r < 0)
		goto free_mmio;

455
	vpci->pci_hdr = (struct pci_device_header) {
456
457
		.vendor_id		= cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET),
		.device_id		= cpu_to_le16(device_id),
458
		.command		= PCI_COMMAND_IO | PCI_COMMAND_MEMORY,
459
460
		.header_type		= PCI_HEADER_TYPE_NORMAL,
		.revision_id		= 0,
461
462
463
464
465
		.class[0]		= class & 0xff,
		.class[1]		= (class >> 8) & 0xff,
		.class[2]		= (class >> 16) & 0xff,
		.subsys_vendor_id	= cpu_to_le16(PCI_SUBSYSTEM_VENDOR_ID_REDHAT_QUMRANET),
		.subsys_id		= cpu_to_le16(subsys_id),
466
		.bar[0]			= cpu_to_le32(vpci->port_addr
467
							| PCI_BASE_ADDRESS_SPACE_IO),
468
469
		.bar[1]			= cpu_to_le32(vpci->mmio_addr
							| PCI_BASE_ADDRESS_SPACE_MEMORY),
470
		.bar[2]			= cpu_to_le32(vpci->msix_io_block
Sasha Levin's avatar
Sasha Levin committed
471
							| PCI_BASE_ADDRESS_SPACE_MEMORY),
472
		.status			= cpu_to_le16(PCI_STATUS_CAP_LIST),
473
		.capabilities		= (void *)&vpci->pci_hdr.msix - (void *)&vpci->pci_hdr,
474
475
476
		.bar_size[0]		= cpu_to_le32(IOPORT_SIZE),
		.bar_size[1]		= cpu_to_le32(IOPORT_SIZE),
		.bar_size[2]		= cpu_to_le32(PCI_IO_SIZE*2),
477
478
	};

479
480
481
482
483
	vpci->dev_hdr = (struct device_header) {
		.bus_type		= DEVICE_BUS_PCI,
		.data			= &vpci->pci_hdr,
	};

484
485
	vpci->pci_hdr.msix.cap = PCI_CAP_ID_MSIX;
	vpci->pci_hdr.msix.next = 0;
486
487
488
489
490
491
492
493
494
495
496
	/*
	 * We at most have VIRTIO_PCI_MAX_VQ entries for virt queue,
	 * VIRTIO_PCI_MAX_CONFIG entries for config.
	 *
	 * To quote the PCI spec:
	 *
	 * System software reads this field to determine the
	 * MSI-X Table Size N, which is encoded as N-1.
	 * For example, a returned value of "00000000011"
	 * indicates a table size of 4.
	 */
497
	vpci->pci_hdr.msix.ctrl = cpu_to_le16(VIRTIO_PCI_MAX_VQ + VIRTIO_PCI_MAX_CONFIG - 1);
Sasha Levin's avatar
Sasha Levin committed
498

499
500
501
	/* Both table and PBA are mapped to the same BAR (2) */
	vpci->pci_hdr.msix.table_offset = cpu_to_le32(2);
	vpci->pci_hdr.msix.pba_offset = cpu_to_le32(2 | PCI_IO_SIZE);
502
503
	vpci->config_vector = 0;

Jean-Philippe Brucker's avatar
Jean-Philippe Brucker committed
504
	if (irq__can_signal_msi(kvm))
505
506
		vpci->features |= VIRTIO_PCI_F_SIGNAL_MSI;

507
	r = device__register(&vpci->dev_hdr);
508
	if (r < 0)
509
		goto free_msix_mmio;
510

511
512
513
	/* save the IRQ that device__register() has allocated */
	vpci->legacy_irq_line = vpci->pci_hdr.irq_line;

514
	return 0;
515

516
free_msix_mmio:
517
	kvm__deregister_mmio(kvm, vpci->msix_io_block);
518
519
free_mmio:
	kvm__deregister_mmio(kvm, vpci->mmio_addr);
520
free_ioport:
521
	ioport__unregister(kvm, vpci->port_addr);
522
	return r;
523
}
524

525
int virtio_pci__exit(struct kvm *kvm, struct virtio_device *vdev)
526
{
527
	struct virtio_pci *vpci = vdev->virtio;
528
529
	int i;

530
	kvm__deregister_mmio(kvm, vpci->mmio_addr);
531
	kvm__deregister_mmio(kvm, vpci->msix_io_block);
532
	ioport__unregister(kvm, vpci->port_addr);
533

534
535
536
537
	for (i = 0; i < VIRTIO_PCI_MAX_VQ; i++) {
		ioeventfd__del_event(vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY, i);
		ioeventfd__del_event(vpci->mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY, i);
	}
538
539

	return 0;
540
}