smp.c 13.3 KB
Newer Older
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
 * Xen SMP support
 *
 * This file implements the Xen versions of smp_ops.  SMP under Xen is
 * very straightforward.  Bringing a CPU up is simply a matter of
 * loading its initial context and setting it running.
 *
 * IPIs are handled through the Xen event mechanism.
 *
 * Because virtual CPUs can be scheduled onto any real CPU, there's no
 * useful topology information for the kernel to make use of.  As a
 * result, all CPUs are treated as if they're single-core and
 * single-threaded.
 */
#include <linux/sched.h>
#include <linux/err.h>
17
#include <linux/slab.h>
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
18
19
20
21
22
23
24
25
26
27
28
29
30
#include <linux/smp.h>

#include <asm/paravirt.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
#include <asm/cpu.h>

#include <xen/interface/xen.h>
#include <xen/interface/vcpu.h>

#include <asm/xen/interface.h>
#include <asm/xen/hypercall.h>

31
#include <xen/xen.h>
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
32
33
34
#include <xen/page.h>
#include <xen/events.h>

35
#include <xen/hvc-console.h>
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
36
37
38
#include "xen-ops.h"
#include "mmu.h"

39
cpumask_var_t xen_cpu_initialized_map;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
40

41
42
43
44
static DEFINE_PER_CPU(int, xen_resched_irq);
static DEFINE_PER_CPU(int, xen_callfunc_irq);
static DEFINE_PER_CPU(int, xen_callfuncsingle_irq);
static DEFINE_PER_CPU(int, xen_debug_irq) = -1;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
45
46

static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
47
static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
48
49

/*
50
 * Reschedule call back.
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
51
52
53
 */
static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
{
54
	inc_irq_stat(irq_resched_count);
55
	scheduler_ipi();
56

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
57
58
59
	return IRQ_HANDLED;
}

60
static void __cpuinit cpu_bringup(void)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
61
{
62
	int cpu;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
63
64

	cpu_init();
Alex Nixon's avatar
Alex Nixon committed
65
	touch_softlockup_watchdog();
66
67
	preempt_disable();

68
	xen_enable_sysenter();
69
	xen_enable_syscall();
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
70

71
72
73
74
	cpu = smp_processor_id();
	smp_store_cpu_info(cpu);
	cpu_data(cpu).x86_max_cores = 1;
	set_cpu_sibling_map(cpu);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
75
76
77

	xen_setup_cpu_clockevents();

78
79
80
	notify_cpu_starting(cpu);

	ipi_call_lock();
81
	set_cpu_online(cpu, true);
82
83
	ipi_call_unlock();

84
	this_cpu_write(cpu_state, CPU_ONLINE);
85

86
87
	wmb();

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
88
89
90
91
	/* We can take interrupts now: we're officially "up". */
	local_irq_enable();

	wmb();			/* make sure everything is out */
Alex Nixon's avatar
Alex Nixon committed
92
93
}

94
static void __cpuinit cpu_bringup_and_idle(void)
Alex Nixon's avatar
Alex Nixon committed
95
96
{
	cpu_bringup();
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
97
98
99
100
101
102
	cpu_idle();
}

static int xen_smp_intr_init(unsigned int cpu)
{
	int rc;
103
	const char *resched_name, *callfunc_name, *debug_name;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
104
105
106
107
108
109
110
111
112
113

	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
				    cpu,
				    xen_reschedule_interrupt,
				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
				    resched_name,
				    NULL);
	if (rc < 0)
		goto fail;
114
	per_cpu(xen_resched_irq, cpu) = rc;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
115
116
117
118
119
120
121
122
123
124

	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
				    cpu,
				    xen_call_function_interrupt,
				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
				    callfunc_name,
				    NULL);
	if (rc < 0)
		goto fail;
125
	per_cpu(xen_callfunc_irq, cpu) = rc;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
126

127
128
129
130
131
132
	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
				     IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
				     debug_name, NULL);
	if (rc < 0)
		goto fail;
133
	per_cpu(xen_debug_irq, cpu) = rc;
134

135
136
137
138
139
140
141
142
143
	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
				    cpu,
				    xen_call_function_single_interrupt,
				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
				    callfunc_name,
				    NULL);
	if (rc < 0)
		goto fail;
144
	per_cpu(xen_callfuncsingle_irq, cpu) = rc;
145

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
146
147
148
	return 0;

 fail:
149
150
151
152
153
154
155
156
157
	if (per_cpu(xen_resched_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
	if (per_cpu(xen_callfunc_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
	if (per_cpu(xen_debug_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
	if (per_cpu(xen_callfuncsingle_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu),
				       NULL);
158

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
159
160
161
	return rc;
}

162
static void __init xen_fill_possible_map(void)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
163
164
165
{
	int i, rc;

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
	if (xen_initial_domain())
		return;

	for (i = 0; i < nr_cpu_ids; i++) {
		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
		if (rc >= 0) {
			num_processors++;
			set_cpu_possible(i, true);
		}
	}
}

static void __init xen_filter_cpu_maps(void)
{
	int i, rc;

	if (!xen_initial_domain())
		return;

185
186
	num_processors = 0;
	disabled_cpus = 0;
187
	for (i = 0; i < nr_cpu_ids; i++) {
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
188
		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
189
190
		if (rc >= 0) {
			num_processors++;
191
			set_cpu_possible(i, true);
192
193
194
		} else {
			set_cpu_possible(i, false);
			set_cpu_present(i, false);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
195
		}
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
196
197
198
	}
}

199
static void __init xen_smp_prepare_boot_cpu(void)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
200
201
202
203
204
205
{
	BUG_ON(smp_processor_id() != 0);
	native_smp_prepare_boot_cpu();

	/* We've switched to the "real" per-cpu gdt, so make sure the
	   old memory can be recycled */
206
	make_lowmem_page_readwrite(xen_initial_gdt);
207

208
	xen_filter_cpu_maps();
209
	xen_setup_vcpu_info_placement();
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
210
211
}

212
static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
213
214
{
	unsigned cpu;
Andrew Jones's avatar
Andrew Jones committed
215
	unsigned int i;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
216

217
218
219
220
221
222
223
224
225
	if (skip_ioapic_setup) {
		char *m = (max_cpus == 0) ?
			"The nosmp parameter is incompatible with Xen; " \
			"use Xen dom0_max_vcpus=1 parameter" :
			"The noapic parameter is incompatible with Xen";

		xen_raw_printk(m);
		panic(m);
	}
226
227
	xen_init_lock_cpu(0);

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
228
	smp_store_cpu_info(0);
229
	cpu_data(0).x86_max_cores = 1;
Andrew Jones's avatar
Andrew Jones committed
230
231
232
233
234
235

	for_each_possible_cpu(i) {
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
	}
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
236
237
238
239
240
	set_cpu_sibling_map(0);

	if (xen_smp_intr_init(0))
		BUG();

241
242
243
244
	if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
		panic("could not allocate xen_cpu_initialized_map\n");

	cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
245
246
247

	/* Restrict the possible_map according to max_cpus. */
	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
248
		for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
249
			continue;
250
		set_cpu_possible(cpu, false);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
251
252
253
254
255
256
257
258
259
260
261
262
	}

	for_each_possible_cpu (cpu) {
		struct task_struct *idle;

		if (cpu == 0)
			continue;

		idle = fork_idle(cpu);
		if (IS_ERR(idle))
			panic("failed fork for CPU %d", cpu);

263
		set_cpu_present(cpu, true);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
264
265
266
	}
}

267
static int __cpuinit
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
268
269
270
cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
{
	struct vcpu_guest_context *ctxt;
271
	struct desc_struct *gdt;
272
	unsigned long gdt_mfn;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
273

274
	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
275
276
277
278
279
280
		return 0;

	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
	if (ctxt == NULL)
		return -ENOMEM;

281
282
	gdt = get_cpu_gdt_table(cpu);

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
283
284
285
286
	ctxt->flags = VGCF_IN_KERNEL;
	ctxt->user_regs.ds = __USER_DS;
	ctxt->user_regs.es = __USER_DS;
	ctxt->user_regs.ss = __KERNEL_DS;
287
288
#ifdef CONFIG_X86_32
	ctxt->user_regs.fs = __KERNEL_PERCPU;
289
	ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
290
291
#else
	ctxt->gs_base_kernel = per_cpu_offset(cpu);
292
#endif
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
293
294
295
296
297
298
299
300
301
	ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
	ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */

	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));

	xen_copy_trap_info(ctxt->trap_ctxt);

	ctxt->ldt_ents = 0;

302
	BUG_ON((unsigned long)gdt & ~PAGE_MASK);
303
304

	gdt_mfn = arbitrary_virt_to_mfn(gdt);
305
	make_lowmem_page_readonly(gdt);
306
	make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
307

308
	ctxt->gdt_frames[0] = gdt_mfn;
309
	ctxt->gdt_ents      = GDT_ENTRIES;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
310
311

	ctxt->user_regs.cs = __KERNEL_CS;
312
	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
313
314

	ctxt->kernel_ss = __KERNEL_DS;
315
	ctxt->kernel_sp = idle->thread.sp0;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
316

317
#ifdef CONFIG_X86_32
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
318
319
	ctxt->event_callback_cs     = __KERNEL_CS;
	ctxt->failsafe_callback_cs  = __KERNEL_CS;
320
321
#endif
	ctxt->event_callback_eip    = (unsigned long)xen_hypervisor_callback;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
322
323
324
325
326
327
328
329
330
331
332
333
	ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;

	per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));

	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
		BUG();

	kfree(ctxt);
	return 0;
}

334
static int __cpuinit xen_cpu_up(unsigned int cpu)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
335
336
337
338
{
	struct task_struct *idle = idle_task(cpu);
	int rc;

339
	per_cpu(current_task, cpu) = idle;
340
#ifdef CONFIG_X86_32
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
341
	irq_ctx_init(cpu);
342
343
#else
	clear_tsk_thread_flag(idle, TIF_FORK);
344
345
346
	per_cpu(kernel_stack, cpu) =
		(unsigned long)task_stack_page(idle) -
		KERNEL_STACK_OFFSET + THREAD_SIZE;
347
#endif
348
	xen_setup_runstate_info(cpu);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
349
	xen_setup_timer(cpu);
350
	xen_init_lock_cpu(cpu);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
351

352
353
	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
	/* make sure interrupts start blocked */
	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;

	rc = cpu_initialize_context(cpu, idle);
	if (rc)
		return rc;

	if (num_online_cpus() == 1)
		alternatives_smp_switch(1);

	rc = xen_smp_intr_init(cpu);
	if (rc)
		return rc;

	rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
	BUG_ON(rc);

371
	while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
Hannes Eder's avatar
Hannes Eder committed
372
		HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
373
374
375
		barrier();
	}

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
376
377
378
	return 0;
}

379
static void xen_smp_cpus_done(unsigned int max_cpus)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
380
381
382
{
}

383
#ifdef CONFIG_HOTPLUG_CPU
384
static int xen_cpu_disable(void)
Alex Nixon's avatar
Alex Nixon committed
385
386
387
388
389
390
391
392
393
394
395
{
	unsigned int cpu = smp_processor_id();
	if (cpu == 0)
		return -EBUSY;

	cpu_disable_common();

	load_cr3(swapper_pg_dir);
	return 0;
}

396
static void xen_cpu_die(unsigned int cpu)
Alex Nixon's avatar
Alex Nixon committed
397
398
399
400
401
{
	while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
		current->state = TASK_UNINTERRUPTIBLE;
		schedule_timeout(HZ/10);
	}
402
403
404
405
	unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL);
Alex Nixon's avatar
Alex Nixon committed
406
407
408
409
410
411
412
	xen_uninit_lock_cpu(cpu);
	xen_teardown_timer(cpu);

	if (num_online_cpus() == 1)
		alternatives_smp_switch(0);
}

413
static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */
Alex Nixon's avatar
Alex Nixon committed
414
415
416
417
{
	play_dead_common();
	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
	cpu_bringup();
418
419
420
421
422
423
424
	/*
	 * Balance out the preempt calls - as we are running in cpu_idle
	 * loop which has been called at bootup from cpu_bringup_and_idle.
	 * The cpucpu_bringup_and_idle called cpu_bringup which made a
	 * preempt_disable() So this preempt_enable will balance it out.
	 */
	preempt_enable();
Alex Nixon's avatar
Alex Nixon committed
425
426
}

427
#else /* !CONFIG_HOTPLUG_CPU */
428
static int xen_cpu_disable(void)
429
430
431
432
{
	return -ENOSYS;
}

433
static void xen_cpu_die(unsigned int cpu)
434
435
436
437
{
	BUG();
}

438
static void xen_play_dead(void)
439
440
441
442
443
{
	BUG();
}

#endif
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
444
445
446
447
448
449
450
451
static void stop_self(void *v)
{
	int cpu = smp_processor_id();

	/* make sure we're not pinning something down */
	load_cr3(swapper_pg_dir);
	/* should set up a minimal gdt */

452
453
	set_cpu_online(cpu, false);

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
454
455
456
457
	HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
	BUG();
}

458
static void xen_stop_other_cpus(int wait)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
459
{
460
	smp_call_function(stop_self, NULL, wait);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
461
462
}

463
static void xen_smp_send_reschedule(int cpu)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
464
465
466
467
{
	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
}

468
469
static void xen_send_IPI_mask(const struct cpumask *mask,
			      enum ipi_vector vector)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
470
471
472
{
	unsigned cpu;

473
	for_each_cpu_and(cpu, mask, cpu_online_mask)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
474
475
476
		xen_send_IPI_one(cpu, vector);
}

477
static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
478
479
480
481
482
483
{
	int cpu;

	xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);

	/* Make sure other vcpus get a chance to run if they need to. */
484
	for_each_cpu(cpu, mask) {
485
		if (xen_vcpu_stolen(cpu)) {
Hannes Eder's avatar
Hannes Eder committed
486
			HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
487
488
489
490
491
			break;
		}
	}
}

492
static void xen_smp_send_call_function_single_ipi(int cpu)
493
{
494
	xen_send_IPI_mask(cpumask_of(cpu),
495
			  XEN_CALL_FUNCTION_SINGLE_VECTOR);
496
497
}

Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
498
499
500
static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
{
	irq_enter();
501
	generic_smp_call_function_interrupt();
502
	inc_irq_stat(irq_call_count);
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
503
504
505
506
507
	irq_exit();

	return IRQ_HANDLED;
}

508
static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
509
{
510
511
	irq_enter();
	generic_smp_call_function_single_interrupt();
512
	inc_irq_stat(irq_call_count);
513
	irq_exit();
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
514

515
	return IRQ_HANDLED;
Jeremy Fitzhardinge's avatar
Jeremy Fitzhardinge committed
516
}
517

518
static const struct smp_ops xen_smp_ops __initconst = {
519
520
521
522
	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
	.smp_prepare_cpus = xen_smp_prepare_cpus,
	.smp_cpus_done = xen_smp_cpus_done,

Alex Nixon's avatar
Alex Nixon committed
523
524
525
526
527
	.cpu_up = xen_cpu_up,
	.cpu_die = xen_cpu_die,
	.cpu_disable = xen_cpu_disable,
	.play_dead = xen_play_dead,

528
	.stop_other_cpus = xen_stop_other_cpus,
529
530
531
532
533
534
535
536
537
	.smp_send_reschedule = xen_smp_send_reschedule,

	.send_call_func_ipi = xen_smp_send_call_function_ipi,
	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
};

void __init xen_smp_init(void)
{
	smp_ops = xen_smp_ops;
538
	xen_fill_possible_map();
539
	xen_init_spinlocks();
540
}
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568

static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
{
	native_smp_prepare_cpus(max_cpus);
	WARN_ON(xen_smp_intr_init(0));

	xen_init_lock_cpu(0);
}

static int __cpuinit xen_hvm_cpu_up(unsigned int cpu)
{
	int rc;
	rc = native_cpu_up(cpu);
	WARN_ON (xen_smp_intr_init(cpu));
	return rc;
}

static void xen_hvm_cpu_die(unsigned int cpu)
{
	unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL);
	native_cpu_die(cpu);
}

void __init xen_hvm_smp_init(void)
{
569
570
	if (!xen_have_vector_callback)
		return;
571
572
573
574
575
576
577
	smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
	smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
	smp_ops.cpu_up = xen_hvm_cpu_up;
	smp_ops.cpu_die = xen_hvm_cpu_die;
	smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
	smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
}