irq-gic.c 41.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
Russell King's avatar
Russell King committed
2
3
4
5
6
7
8
9
10
11
/*
 *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
 *
 * Interrupt architecture for the GIC:
 *
 * o There is one Interrupt Distributor, which receives interrupts
 *   from system devices and sends them to the Interrupt Controllers.
 *
 * o There is one CPU Interface per CPU, which sends interrupts sent
 *   by the Distributor, and interrupts generated locally, to the
12
13
14
 *   associated CPU. The base address of the CPU interface is usually
 *   aliased so that the same address points to different chips depending
 *   on the CPU it is accessed from.
Russell King's avatar
Russell King committed
15
16
17
18
19
20
21
 *
 * Note that IRQs 0-31 are special - they are local to each CPU.
 * As such, the enable set/clear, pending set/clear and active bit
 * registers are banked per-cpu for these sources.
 */
#include <linux/init.h>
#include <linux/kernel.h>
22
#include <linux/err.h>
23
#include <linux/module.h>
Russell King's avatar
Russell King committed
24
25
#include <linux/list.h>
#include <linux/smp.h>
26
#include <linux/cpu.h>
27
#include <linux/cpu_pm.h>
28
#include <linux/cpumask.h>
29
#include <linux/io.h>
30
31
32
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
33
#include <linux/acpi.h>
Rob Herring's avatar
Rob Herring committed
34
#include <linux/irqdomain.h>
35
36
37
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/slab.h>
38
#include <linux/irqchip.h>
39
#include <linux/irqchip/chained_irq.h>
40
#include <linux/irqchip/arm-gic.h>
Russell King's avatar
Russell King committed
41

42
#include <asm/cputype.h>
Russell King's avatar
Russell King committed
43
#include <asm/irq.h>
44
#include <asm/exception.h>
45
#include <asm/smp_plat.h>
46
#include <asm/virt.h>
Russell King's avatar
Russell King committed
47

48
#include "irq-gic-common.h"
Russell King's avatar
Russell King committed
49

50
51
52
53
54
#ifdef CONFIG_ARM64
#include <asm/cpufeature.h>

static void gic_check_cpu_features(void)
{
55
	WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_SYSREG_GIC_CPUIF),
56
57
58
59
60
61
62
			TAINT_CPU_OUT_OF_SPEC,
			"GICv3 system registers enabled, broken firmware!\n");
}
#else
#define gic_check_cpu_features()	do { } while(0)
#endif

63
64
union gic_base {
	void __iomem *common_base;
65
	void __percpu * __iomem *percpu_base;
66
67
68
};

struct gic_chip_data {
69
	struct irq_chip chip;
70
71
	union gic_base dist_base;
	union gic_base cpu_base;
72
73
74
	void __iomem *raw_dist_base;
	void __iomem *raw_cpu_base;
	u32 percpu_offset;
75
#if defined(CONFIG_CPU_PM) || defined(CONFIG_ARM_GIC_PM)
76
	u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)];
77
	u32 saved_spi_active[DIV_ROUND_UP(1020, 32)];
78
79
80
	u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)];
	u32 saved_spi_target[DIV_ROUND_UP(1020, 4)];
	u32 __percpu *saved_ppi_enable;
81
	u32 __percpu *saved_ppi_active;
82
83
	u32 __percpu *saved_ppi_conf;
#endif
84
	struct irq_domain *domain;
85
86
87
88
89
90
	unsigned int gic_irqs;
#ifdef CONFIG_GIC_NON_BANKED
	void __iomem *(*get_base)(union gic_base *);
#endif
};

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#ifdef CONFIG_BL_SWITCHER

static DEFINE_RAW_SPINLOCK(cpu_map_lock);

#define gic_lock_irqsave(f)		\
	raw_spin_lock_irqsave(&cpu_map_lock, (f))
#define gic_unlock_irqrestore(f)	\
	raw_spin_unlock_irqrestore(&cpu_map_lock, (f))

#define gic_lock()			raw_spin_lock(&cpu_map_lock)
#define gic_unlock()			raw_spin_unlock(&cpu_map_lock)

#else

#define gic_lock_irqsave(f)		do { (void)(f); } while(0)
#define gic_unlock_irqrestore(f)	do { (void)(f); } while(0)

#define gic_lock()			do { } while(0)
#define gic_unlock()			do { } while(0)

#endif
Russell King's avatar
Russell King committed
112

113
114
115
116
117
118
119
120
/*
 * The GIC mapping of CPU interfaces does not necessarily match
 * the logical CPU numbering.  Let's use a mapping as returned
 * by the GIC itself.
 */
#define NR_GIC_CPU_IF 8
static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly;

121
static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
122

123
static struct gic_chip_data gic_data[CONFIG_ARM_GIC_MAX_NR] __read_mostly;
124

125
126
static struct gic_kvm_info gic_v2_kvm_info;

127
128
129
#ifdef CONFIG_GIC_NON_BANKED
static void __iomem *gic_get_percpu_base(union gic_base *base)
{
130
	return raw_cpu_read(*base->percpu_base);
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
}

static void __iomem *gic_get_common_base(union gic_base *base)
{
	return base->common_base;
}

static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data)
{
	return data->get_base(&data->dist_base);
}

static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data)
{
	return data->get_base(&data->cpu_base);
}

static inline void gic_set_base_accessor(struct gic_chip_data *data,
					 void __iomem *(*f)(union gic_base *))
{
	data->get_base = f;
}
#else
#define gic_data_dist_base(d)	((d)->dist_base.common_base)
#define gic_data_cpu_base(d)	((d)->cpu_base.common_base)
156
#define gic_set_base_accessor(d, f)
157
158
#endif

159
static inline void __iomem *gic_dist_base(struct irq_data *d)
160
{
161
	struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
162
	return gic_data_dist_base(gic_data);
163
164
}

165
static inline void __iomem *gic_cpu_base(struct irq_data *d)
166
{
167
	struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
168
	return gic_data_cpu_base(gic_data);
169
170
}

171
static inline unsigned int gic_irq(struct irq_data *d)
172
{
Rob Herring's avatar
Rob Herring committed
173
	return d->hwirq;
174
175
}

176
177
178
179
180
static inline bool cascading_gic_irq(struct irq_data *d)
{
	void *data = irq_data_get_irq_handler_data(d);

	/*
181
182
	 * If handler_data is set, this is a cascading interrupt, and
	 * it cannot possibly be forwarded.
183
	 */
184
	return data != NULL;
185
186
}

Russell King's avatar
Russell King committed
187
188
189
/*
 * Routines to acknowledge, disable and enable interrupts
 */
190
191
192
193
194
195
196
static void gic_poke_irq(struct irq_data *d, u32 offset)
{
	u32 mask = 1 << (gic_irq(d) % 32);
	writel_relaxed(mask, gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4);
}

static int gic_peek_irq(struct irq_data *d, u32 offset)
Russell King's avatar
Russell King committed
197
{
Rob Herring's avatar
Rob Herring committed
198
	u32 mask = 1 << (gic_irq(d) % 32);
199
200
201
202
203
204
	return !!(readl_relaxed(gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4) & mask);
}

static void gic_mask_irq(struct irq_data *d)
{
	gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR);
Russell King's avatar
Russell King committed
205
206
}

207
208
209
static void gic_eoimode1_mask_irq(struct irq_data *d)
{
	gic_mask_irq(d);
210
211
212
213
214
215
216
217
	/*
	 * When masking a forwarded interrupt, make sure it is
	 * deactivated as well.
	 *
	 * This ensures that an interrupt that is getting
	 * disabled/masked will not get "stuck", because there is
	 * noone to deactivate it (guest is being terminated).
	 */
218
	if (irqd_is_forwarded_to_vcpu(d))
219
		gic_poke_irq(d, GIC_DIST_ACTIVE_CLEAR);
220
221
}

222
static void gic_unmask_irq(struct irq_data *d)
Russell King's avatar
Russell King committed
223
{
224
	gic_poke_irq(d, GIC_DIST_ENABLE_SET);
Russell King's avatar
Russell King committed
225
226
}

227
228
static void gic_eoi_irq(struct irq_data *d)
{
229
	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
230
231
}

232
233
static void gic_eoimode1_eoi_irq(struct irq_data *d)
{
234
	/* Do not deactivate an IRQ forwarded to a vcpu. */
235
	if (irqd_is_forwarded_to_vcpu(d))
236
237
		return;

238
239
240
	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE);
}

241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
static int gic_irq_set_irqchip_state(struct irq_data *d,
				     enum irqchip_irq_state which, bool val)
{
	u32 reg;

	switch (which) {
	case IRQCHIP_STATE_PENDING:
		reg = val ? GIC_DIST_PENDING_SET : GIC_DIST_PENDING_CLEAR;
		break;

	case IRQCHIP_STATE_ACTIVE:
		reg = val ? GIC_DIST_ACTIVE_SET : GIC_DIST_ACTIVE_CLEAR;
		break;

	case IRQCHIP_STATE_MASKED:
		reg = val ? GIC_DIST_ENABLE_CLEAR : GIC_DIST_ENABLE_SET;
		break;

	default:
		return -EINVAL;
	}

	gic_poke_irq(d, reg);
	return 0;
}

static int gic_irq_get_irqchip_state(struct irq_data *d,
				      enum irqchip_irq_state which, bool *val)
{
	switch (which) {
	case IRQCHIP_STATE_PENDING:
		*val = gic_peek_irq(d, GIC_DIST_PENDING_SET);
		break;

	case IRQCHIP_STATE_ACTIVE:
		*val = gic_peek_irq(d, GIC_DIST_ACTIVE_SET);
		break;

	case IRQCHIP_STATE_MASKED:
		*val = !gic_peek_irq(d, GIC_DIST_ENABLE_SET);
		break;

	default:
		return -EINVAL;
	}

	return 0;
}

290
static int gic_set_type(struct irq_data *d, unsigned int type)
291
{
292
293
	void __iomem *base = gic_dist_base(d);
	unsigned int gicirq = gic_irq(d);
294
	int ret;
295
296
297
298
299

	/* Interrupt configuration for SGIs can't be changed */
	if (gicirq < 16)
		return -EINVAL;

300
301
302
	/* SPIs have restrictions on the supported types */
	if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
			    type != IRQ_TYPE_EDGE_RISING)
303
304
		return -EINVAL;

305
306
307
308
309
310
311
312
	ret = gic_configure_irq(gicirq, type, base + GIC_DIST_CONFIG, NULL);
	if (ret && gicirq < 32) {
		/* Misconfigured PPIs are usually not fatal */
		pr_warn("GIC: PPI%d is secure or misconfigured\n", gicirq - 16);
		ret = 0;
	}

	return ret;
313
314
}

315
316
317
318
319
320
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
	/* Only interrupts on the primary GIC can be forwarded to a vcpu. */
	if (cascading_gic_irq(d))
		return -EINVAL;

321
322
323
324
	if (vcpu)
		irqd_set_forwarded_to_vcpu(d);
	else
		irqd_clr_forwarded_to_vcpu(d);
325
326
327
	return 0;
}

328
#ifdef CONFIG_SMP
329
330
static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
			    bool force)
Russell King's avatar
Russell King committed
331
{
332
	void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3);
333
	unsigned int cpu, shift = (gic_irq(d) % 4) * 8;
334
	u32 val, mask, bit;
335
	unsigned long flags;
Russell King's avatar
Russell King committed
336

337
338
339
340
341
	if (!force)
		cpu = cpumask_any_and(mask_val, cpu_online_mask);
	else
		cpu = cpumask_first(mask_val);

342
	if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
343
		return -EINVAL;
344

345
	gic_lock_irqsave(flags);
346
	mask = 0xff << shift;
347
	bit = gic_cpu_map[cpu] << shift;
348
349
	val = readl_relaxed(reg) & ~mask;
	writel_relaxed(val | bit, reg);
350
	gic_unlock_irqrestore(flags);
351

352
353
	irq_data_update_effective_affinity(d, cpumask_of(cpu));

354
	return IRQ_SET_MASK_OK_DONE;
Russell King's avatar
Russell King committed
355
}
356
#endif
Russell King's avatar
Russell King committed
357

358
static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
359
360
361
362
363
364
365
{
	u32 irqstat, irqnr;
	struct gic_chip_data *gic = &gic_data[0];
	void __iomem *cpu_base = gic_data_cpu_base(gic);

	do {
		irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
366
		irqnr = irqstat & GICC_IAR_INT_ID_MASK;
367

368
		if (likely(irqnr > 15 && irqnr < 1020)) {
369
			if (static_branch_likely(&supports_deactivate_key))
370
				writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
371
			isb();
372
			handle_domain_irq(gic->domain, irqnr, regs);
373
374
375
376
			continue;
		}
		if (irqnr < 16) {
			writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
377
			if (static_branch_likely(&supports_deactivate_key))
378
				writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE);
379
#ifdef CONFIG_SMP
380
381
382
383
384
385
386
387
			/*
			 * Ensure any shared data written by the CPU sending
			 * the IPI is read after we've read the ACK register
			 * on the GIC.
			 *
			 * Pairs with the write barrier in gic_raise_softirq
			 */
			smp_rmb();
388
389
390
391
392
393
394
395
			handle_IPI(irqnr, regs);
#endif
			continue;
		}
		break;
	} while (1);
}

396
static void gic_handle_cascade_irq(struct irq_desc *desc)
397
{
398
399
	struct gic_chip_data *chip_data = irq_desc_get_handler_data(desc);
	struct irq_chip *chip = irq_desc_get_chip(desc);
400
	unsigned int cascade_irq, gic_irq;
401
402
	unsigned long status;

403
	chained_irq_enter(chip, desc);
404

405
	status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK);
406

407
408
	gic_irq = (status & GICC_IAR_INT_ID_MASK);
	if (gic_irq == GICC_INT_SPURIOUS)
409
410
		goto out;

411
	cascade_irq = irq_find_mapping(chip_data->domain, gic_irq);
412
	if (unlikely(gic_irq < 32 || gic_irq > 1020)) {
413
		handle_bad_irq(desc);
414
415
	} else {
		isb();
416
		generic_handle_irq(cascade_irq);
417
	}
418
419

 out:
420
	chained_irq_exit(chip, desc);
421
422
}

423
static const struct irq_chip gic_chip = {
424
425
	.irq_mask		= gic_mask_irq,
	.irq_unmask		= gic_unmask_irq,
426
	.irq_eoi		= gic_eoi_irq,
427
	.irq_set_type		= gic_set_type,
428
429
	.irq_get_irqchip_state	= gic_irq_get_irqchip_state,
	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
430
431
432
	.flags			= IRQCHIP_SET_TYPE_MASKED |
				  IRQCHIP_SKIP_SET_WAKE |
				  IRQCHIP_MASK_ON_SUSPEND,
Russell King's avatar
Russell King committed
433
434
};

435
436
void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
{
437
	BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
438
439
	irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq,
					 &gic_data[gic_nr]);
440
441
}

442
443
444
445
446
447
448
449
450
451
452
453
454
static u8 gic_get_cpumask(struct gic_chip_data *gic)
{
	void __iomem *base = gic_data_dist_base(gic);
	u32 mask, i;

	for (i = mask = 0; i < 32; i += 4) {
		mask = readl_relaxed(base + GIC_DIST_TARGET + i);
		mask |= mask >> 16;
		mask |= mask >> 8;
		if (mask)
			break;
	}

455
	if (!mask && num_possible_cpus() > 1)
456
457
458
459
460
		pr_crit("GIC CPU mask not found - kernel will fail to boot.\n");

	return mask;
}

461
462
463
464
465
466
static bool gic_check_gicv2(void __iomem *base)
{
	u32 val = readl_relaxed(base + GIC_CPU_IDENT);
	return (val & 0xff0fff) == 0x02043B;
}

467
static void gic_cpu_if_up(struct gic_chip_data *gic)
468
{
469
	void __iomem *cpu_base = gic_data_cpu_base(gic);
470
	u32 bypass = 0;
471
	u32 mode = 0;
472
	int i;
473

474
	if (gic == &gic_data[0] && static_branch_likely(&supports_deactivate_key))
475
		mode = GIC_CPU_CTRL_EOImodeNS;
476

477
478
479
480
	if (gic_check_gicv2(cpu_base))
		for (i = 0; i < 4; i++)
			writel_relaxed(0, cpu_base + GIC_CPU_ACTIVEPRIO + i * 4);

481
482
483
484
485
486
	/*
	* Preserve bypass disable bits to be written back later
	*/
	bypass = readl(cpu_base + GIC_CPU_CTRL);
	bypass &= GICC_DIS_BYPASS_MASK;

487
	writel_relaxed(bypass | mode | GICC_ENABLE, cpu_base + GIC_CPU_CTRL);
488
489
490
}


491
static void gic_dist_init(struct gic_chip_data *gic)
Russell King's avatar
Russell King committed
492
{
493
	unsigned int i;
494
	u32 cpumask;
Rob Herring's avatar
Rob Herring committed
495
	unsigned int gic_irqs = gic->gic_irqs;
496
	void __iomem *base = gic_data_dist_base(gic);
Russell King's avatar
Russell King committed
497

498
	writel_relaxed(GICD_DISABLE, base + GIC_DIST_CTRL);
Russell King's avatar
Russell King committed
499
500
501
502

	/*
	 * Set all global interrupts to this CPU only.
	 */
503
504
505
	cpumask = gic_get_cpumask(gic);
	cpumask |= cpumask << 8;
	cpumask |= cpumask << 16;
506
	for (i = 32; i < gic_irqs; i += 4)
507
		writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
Russell King's avatar
Russell King committed
508

509
	gic_dist_config(base, gic_irqs, NULL);
Russell King's avatar
Russell King committed
510

511
	writel_relaxed(GICD_ENABLE, base + GIC_DIST_CTRL);
Russell King's avatar
Russell King committed
512
513
}

514
static int gic_cpu_init(struct gic_chip_data *gic)
Russell King's avatar
Russell King committed
515
{
516
517
	void __iomem *dist_base = gic_data_dist_base(gic);
	void __iomem *base = gic_data_cpu_base(gic);
518
	unsigned int cpu_mask, cpu = smp_processor_id();
519
520
	int i;

521
	/*
522
523
524
	 * Setting up the CPU map is only relevant for the primary GIC
	 * because any nested/secondary GICs do not directly interface
	 * with the CPU(s).
525
	 */
526
527
528
529
	if (gic == &gic_data[0]) {
		/*
		 * Get what the GIC says our CPU mask is.
		 */
530
531
532
		if (WARN_ON(cpu >= NR_GIC_CPU_IF))
			return -EINVAL;

533
		gic_check_cpu_features();
534
535
		cpu_mask = gic_get_cpumask(gic);
		gic_cpu_map[cpu] = cpu_mask;
536

537
538
539
540
541
542
543
544
		/*
		 * Clear our mask from the other map entries in case they're
		 * still undefined.
		 */
		for (i = 0; i < NR_GIC_CPU_IF; i++)
			if (i != cpu)
				gic_cpu_map[i] &= ~cpu_mask;
	}
545

546
	gic_cpu_config(dist_base, 32, NULL);
547

548
	writel_relaxed(GICC_INT_PRI_THRESHOLD, base + GIC_CPU_PRIMASK);
549
	gic_cpu_if_up(gic);
550
551

	return 0;
Russell King's avatar
Russell King committed
552
553
}

554
int gic_cpu_if_down(unsigned int gic_nr)
555
{
556
	void __iomem *cpu_base;
557
558
	u32 val = 0;

559
	if (gic_nr >= CONFIG_ARM_GIC_MAX_NR)
560
561
562
		return -EINVAL;

	cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
563
564
565
	val = readl(cpu_base + GIC_CPU_CTRL);
	val &= ~GICC_ENABLE;
	writel_relaxed(val, cpu_base + GIC_CPU_CTRL);
566
567

	return 0;
568
569
}

570
#if defined(CONFIG_CPU_PM) || defined(CONFIG_ARM_GIC_PM)
571
572
573
574
575
576
/*
 * Saves the GIC distributor registers during suspend or idle.  Must be called
 * with interrupts disabled but before powering down the GIC.  After calling
 * this function, no interrupts will be delivered by the GIC, and another
 * platform-specific wakeup source must be enabled.
 */
577
void gic_dist_save(struct gic_chip_data *gic)
578
579
580
581
582
{
	unsigned int gic_irqs;
	void __iomem *dist_base;
	int i;

583
584
	if (WARN_ON(!gic))
		return;
585

586
587
	gic_irqs = gic->gic_irqs;
	dist_base = gic_data_dist_base(gic);
588
589
590
591
592

	if (!dist_base)
		return;

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
593
		gic->saved_spi_conf[i] =
594
595
596
			readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
597
		gic->saved_spi_target[i] =
598
599
600
			readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
601
		gic->saved_spi_enable[i] =
602
			readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);
603
604

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
605
		gic->saved_spi_active[i] =
606
			readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4);
607
608
609
610
611
}

/*
 * Restores the GIC distributor registers during resume or when coming out of
 * idle.  Must be called before enabling interrupts.  If a level interrupt
612
613
 * that occurred while the GIC was suspended is still present, it will be
 * handled normally, but any edge interrupts that occurred will not be seen by
614
615
 * the GIC and need to be handled by the platform-specific wakeup source.
 */
616
void gic_dist_restore(struct gic_chip_data *gic)
617
618
619
620
621
{
	unsigned int gic_irqs;
	unsigned int i;
	void __iomem *dist_base;

622
623
	if (WARN_ON(!gic))
		return;
624

625
626
	gic_irqs = gic->gic_irqs;
	dist_base = gic_data_dist_base(gic);
627
628
629
630

	if (!dist_base)
		return;

631
	writel_relaxed(GICD_DISABLE, dist_base + GIC_DIST_CTRL);
632
633

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
634
		writel_relaxed(gic->saved_spi_conf[i],
635
636
637
			dist_base + GIC_DIST_CONFIG + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
638
		writel_relaxed(GICD_INT_DEF_PRI_X4,
639
640
641
			dist_base + GIC_DIST_PRI + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
642
		writel_relaxed(gic->saved_spi_target[i],
643
644
			dist_base + GIC_DIST_TARGET + i * 4);

645
646
647
	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			dist_base + GIC_DIST_ENABLE_CLEAR + i * 4);
648
		writel_relaxed(gic->saved_spi_enable[i],
649
			dist_base + GIC_DIST_ENABLE_SET + i * 4);
650
	}
651

652
653
654
	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4);
655
		writel_relaxed(gic->saved_spi_active[i],
656
657
658
			dist_base + GIC_DIST_ACTIVE_SET + i * 4);
	}

659
	writel_relaxed(GICD_ENABLE, dist_base + GIC_DIST_CTRL);
660
661
}

662
void gic_cpu_save(struct gic_chip_data *gic)
663
664
665
666
667
668
{
	int i;
	u32 *ptr;
	void __iomem *dist_base;
	void __iomem *cpu_base;

669
670
	if (WARN_ON(!gic))
		return;
671

672
673
	dist_base = gic_data_dist_base(gic);
	cpu_base = gic_data_cpu_base(gic);
674
675
676
677

	if (!dist_base || !cpu_base)
		return;

678
	ptr = raw_cpu_ptr(gic->saved_ppi_enable);
679
680
681
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
		ptr[i] = readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);

682
	ptr = raw_cpu_ptr(gic->saved_ppi_active);
683
684
685
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
		ptr[i] = readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4);

686
	ptr = raw_cpu_ptr(gic->saved_ppi_conf);
687
688
689
690
691
	for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
		ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);

}

692
void gic_cpu_restore(struct gic_chip_data *gic)
693
694
695
696
697
698
{
	int i;
	u32 *ptr;
	void __iomem *dist_base;
	void __iomem *cpu_base;

699
700
	if (WARN_ON(!gic))
		return;
701

702
703
	dist_base = gic_data_dist_base(gic);
	cpu_base = gic_data_cpu_base(gic);
704
705
706
707

	if (!dist_base || !cpu_base)
		return;

708
	ptr = raw_cpu_ptr(gic->saved_ppi_enable);
709
710
711
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			       dist_base + GIC_DIST_ENABLE_CLEAR + i * 4);
712
		writel_relaxed(ptr[i], dist_base + GIC_DIST_ENABLE_SET + i * 4);
713
	}
714

715
	ptr = raw_cpu_ptr(gic->saved_ppi_active);
716
717
718
719
720
721
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			       dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4);
		writel_relaxed(ptr[i], dist_base + GIC_DIST_ACTIVE_SET + i * 4);
	}

722
	ptr = raw_cpu_ptr(gic->saved_ppi_conf);
723
724
725
726
	for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
		writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4);

	for (i = 0; i < DIV_ROUND_UP(32, 4); i++)
727
728
		writel_relaxed(GICD_INT_DEF_PRI_X4,
					dist_base + GIC_DIST_PRI + i * 4);
729

730
	writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK);
731
	gic_cpu_if_up(gic);
732
733
734
735
736
737
}

static int gic_notifier(struct notifier_block *self, unsigned long cmd,	void *v)
{
	int i;

738
	for (i = 0; i < CONFIG_ARM_GIC_MAX_NR; i++) {
739
740
741
742
743
#ifdef CONFIG_GIC_NON_BANKED
		/* Skip over unused GICs */
		if (!gic_data[i].get_base)
			continue;
#endif
744
745
		switch (cmd) {
		case CPU_PM_ENTER:
746
			gic_cpu_save(&gic_data[i]);
747
748
749
			break;
		case CPU_PM_ENTER_FAILED:
		case CPU_PM_EXIT:
750
			gic_cpu_restore(&gic_data[i]);
751
752
			break;
		case CPU_CLUSTER_PM_ENTER:
753
			gic_dist_save(&gic_data[i]);
754
755
756
			break;
		case CPU_CLUSTER_PM_ENTER_FAILED:
		case CPU_CLUSTER_PM_EXIT:
757
			gic_dist_restore(&gic_data[i]);
758
759
760
761
762
763
764
765
766
767
768
			break;
		}
	}

	return NOTIFY_OK;
}

static struct notifier_block gic_notifier_block = {
	.notifier_call = gic_notifier,
};

769
static int gic_pm_init(struct gic_chip_data *gic)
770
771
772
{
	gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
		sizeof(u32));
773
774
	if (WARN_ON(!gic->saved_ppi_enable))
		return -ENOMEM;
775

776
777
	gic->saved_ppi_active = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
		sizeof(u32));
778
779
	if (WARN_ON(!gic->saved_ppi_active))
		goto free_ppi_enable;
780

781
782
	gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4,
		sizeof(u32));
783
784
	if (WARN_ON(!gic->saved_ppi_conf))
		goto free_ppi_active;
785

786
787
	if (gic == &gic_data[0])
		cpu_pm_register_notifier(&gic_notifier_block);
788
789
790
791
792
793
794
795
796

	return 0;

free_ppi_active:
	free_percpu(gic->saved_ppi_active);
free_ppi_enable:
	free_percpu(gic->saved_ppi_enable);

	return -ENOMEM;
797
798
}
#else
799
static int gic_pm_init(struct gic_chip_data *gic)
800
{
801
	return 0;
802
803
804
}
#endif

805
#ifdef CONFIG_SMP
806
static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
807
808
{
	int cpu;
809
810
	unsigned long flags, map = 0;

811
812
813
814
815
816
817
	if (unlikely(nr_cpu_ids == 1)) {
		/* Only one CPU? let's do a self-IPI... */
		writel_relaxed(2 << 24 | irq,
			       gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
		return;
	}

818
	gic_lock_irqsave(flags);
819
820
821

	/* Convert our logical CPU mask into a physical one. */
	for_each_cpu(cpu, mask)
822
		map |= gic_cpu_map[cpu];
823
824
825

	/*
	 * Ensure that stores to Normal memory are visible to the
826
	 * other CPUs before they observe us issuing the IPI.
827
	 */
828
	dmb(ishst);
829
830
831

	/* this always happens on GIC0 */
	writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
832

833
	gic_unlock_irqrestore(flags);
834
835
836
837
}
#endif

#ifdef CONFIG_BL_SWITCHER
838
839
840
841
842
843
844
845
846
847
848
849
850
851
/*
 * gic_send_sgi - send a SGI directly to given CPU interface number
 *
 * cpu_id: the ID for the destination CPU interface
 * irq: the IPI number to send a SGI for
 */
void gic_send_sgi(unsigned int cpu_id, unsigned int irq)
{
	BUG_ON(cpu_id >= NR_GIC_CPU_IF);
	cpu_id = 1 << cpu_id;
	/* this always happens on GIC0 */
	writel_relaxed((cpu_id << 16) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
}

852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
/*
 * gic_get_cpu_id - get the CPU interface ID for the specified CPU
 *
 * @cpu: the logical CPU number to get the GIC ID for.
 *
 * Return the CPU interface ID for the given logical CPU number,
 * or -1 if the CPU number is too large or the interface ID is
 * unknown (more than one bit set).
 */
int gic_get_cpu_id(unsigned int cpu)
{
	unsigned int cpu_bit;

	if (cpu >= NR_GIC_CPU_IF)
		return -1;
	cpu_bit = gic_cpu_map[cpu];
	if (cpu_bit & (cpu_bit - 1))
		return -1;
	return __ffs(cpu_bit);
}

873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
/*
 * gic_migrate_target - migrate IRQs to another CPU interface
 *
 * @new_cpu_id: the CPU target ID to migrate IRQs to
 *
 * Migrate all peripheral interrupts with a target matching the current CPU
 * to the interface corresponding to @new_cpu_id.  The CPU interface mapping
 * is also updated.  Targets to other CPU interfaces are unchanged.
 * This must be called with IRQs locally disabled.
 */
void gic_migrate_target(unsigned int new_cpu_id)
{
	unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
	void __iomem *dist_base;
	int i, ror_val, cpu = smp_processor_id();
	u32 val, cur_target_mask, active_mask;

890
	BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
891
892
893
894
895
896
897
898
899
900

	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
	if (!dist_base)
		return;
	gic_irqs = gic_data[gic_nr].gic_irqs;

	cur_cpu_id = __ffs(gic_cpu_map[cpu]);
	cur_target_mask = 0x01010101 << cur_cpu_id;
	ror_val = (cur_cpu_id - new_cpu_id) & 31;

901
	gic_lock();
902
903
904
905
906

	/* Update the target interface for this logical CPU */
	gic_cpu_map[cpu] = 1 << new_cpu_id;

	/*
907
	 * Find all the peripheral interrupts targeting the current
908
909
910
911
912
913
914
915
916
917
918
919
920
	 * CPU interface and migrate them to the new CPU interface.
	 * We skip DIST_TARGET 0 to 7 as they are read-only.
	 */
	for (i = 8; i < DIV_ROUND_UP(gic_irqs, 4); i++) {
		val = readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
		active_mask = val & cur_target_mask;
		if (active_mask) {
			val &= ~active_mask;
			val |= ror32(active_mask, ror_val);
			writel_relaxed(val, dist_base + GIC_DIST_TARGET + i*4);
		}
	}

921
	gic_unlock();
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945

	/*
	 * Now let's migrate and clear any potential SGIs that might be
	 * pending for us (cur_cpu_id).  Since GIC_DIST_SGI_PENDING_SET
	 * is a banked register, we can only forward the SGI using
	 * GIC_DIST_SOFTINT.  The original SGI source is lost but Linux
	 * doesn't use that information anyway.
	 *
	 * For the same reason we do not adjust SGI source information
	 * for previously sent SGIs by us to other CPUs either.
	 */
	for (i = 0; i < 16; i += 4) {
		int j;
		val = readl_relaxed(dist_base + GIC_DIST_SGI_PENDING_SET + i);
		if (!val)
			continue;
		writel_relaxed(val, dist_base + GIC_DIST_SGI_PENDING_CLEAR + i);
		for (j = i; j < i + 4; j++) {
			if (val & 0xff)
				writel_relaxed((1 << (new_cpu_id + 16)) | j,
						dist_base + GIC_DIST_SOFTINT);
			val >>= 8;
		}
	}
946
}
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962

/*
 * gic_get_sgir_physaddr - get the physical address for the SGI register
 *
 * REturn the physical address of the SGI register to be used
 * by some early assembly code when the kernel is not yet available.
 */
static unsigned long gic_dist_physaddr;

unsigned long gic_get_sgir_physaddr(void)
{
	if (!gic_dist_physaddr)
		return 0;
	return gic_dist_physaddr + GIC_DIST_SOFTINT;
}

963
static void __init gic_init_physaddr(struct device_node *node)
964
965
966
967
968
969
970
971
972
973
{
	struct resource res;
	if (of_address_to_resource(node, 0, &res) == 0) {
		gic_dist_physaddr = res.start;
		pr_info("GIC physical location is %#lx\n", gic_dist_physaddr);
	}
}

#else
#define gic_init_physaddr(node)  do { } while (0)
974
975
#endif

976
977
978
static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
				irq_hw_number_t hw)
{
979
	struct gic_chip_data *gic = d->host_data;
980

981
982
	if (hw < 32) {
		irq_set_percpu_devid(irq);
983
		irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
984
				    handle_percpu_devid_irq, NULL, NULL);
985
		irq_set_status_flags(irq, IRQ_NOAUTOEN);
986
	} else {
987
		irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
988
				    handle_fasteoi_irq, NULL, NULL);
989
		irq_set_probe(irq);
990
		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
991
992
993
994
	}
	return 0;
}

995
996
997
998
static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
{
}

999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
static int gic_irq_domain_translate(struct irq_domain *d,
				    struct irq_fwspec *fwspec,
				    unsigned long *hwirq,
				    unsigned int *type)
{
	if (is_of_node(fwspec->fwnode)) {
		if (fwspec->param_count < 3)
			return -EINVAL;

		/* Get the interrupt number and add 16 to skip over SGIs */
		*hwirq = fwspec->param[1] + 16;

		/*
		 * For SPIs, we need to add 16 more to get the GIC irq
		 * ID number
		 */
		if (!fwspec->param[0])
			*hwirq += 16;

		*type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1019
1020
1021

		/* Make it clear that broken DTs are... broken */
		WARN_ON(*type == IRQ_TYPE_NONE);
1022
1023
1024
		return 0;
	}

1025
	if (is_fwnode_irqchip(fwspec->fwnode)) {
1026
1027
1028
1029
1030
		if(fwspec->param_count != 2)
			return -EINVAL;

		*hwirq = fwspec->param[0];
		*type = fwspec->param[1];
1031
1032

		WARN_ON(*type == IRQ_TYPE_NONE);
1033
1034
1035
		return 0;
	}

1036
1037
1038
	return -EINVAL;
}

1039
static int gic_starting_cpu(unsigned int cpu)
1040
{
1041
1042
	gic_cpu_init(&gic_data[0]);
	return 0;
1043
1044
}

1045
1046
1047
1048
1049
1050
static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
				unsigned int nr_irqs, void *arg)
{
	int i, ret;
	irq_hw_number_t hwirq;
	unsigned int type = IRQ_TYPE_NONE;
1051
	struct irq_fwspec *fwspec = arg;
1052

1053
	ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1054
1055
1056
	if (ret)
		return ret;

1057
1058
1059
1060
1061
	for (i = 0; i < nr_irqs; i++) {
		ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
		if (ret)
			return ret;
	}
1062
1063
1064
1065
1066

	return 0;
}

static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = {
1067
	.translate = gic_irq_domain_translate,
1068
1069
1070
1071
	.alloc = gic_irq_domain_alloc,
	.free = irq_domain_free_irqs_top,
};

1072
static const struct irq_domain_ops gic_irq_domain_ops = {
1073
	.map = gic_irq_domain_map,
1074
	.unmap = gic_irq_domain_unmap,
Rob Herring's avatar
Rob Herring committed
1075
1076
};

1077
1078
static void gic_init_chip(struct gic_chip_data *gic, struct device *dev,
			  const char *name, bool use_eoimode1)
1079
{
1080
	/* Initialize irq_chip */
1081
	gic->chip = gic_chip;
1082
1083
	gic->chip.name = name;
	gic->chip.parent_device = dev;
1084

1085
	if (use_eoimode1) {
1086
1087
1088
		gic->chip.irq_mask = gic_eoimode1_mask_irq;
		gic->chip.irq_eoi = gic_eoimode1_eoi_irq;
		gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity;
1089
1090
	}

1091
#ifdef CONFIG_SMP