bpf_jit_comp.c 35.2 KB
Newer Older
1
2
3
/*
 * BPF Jit compiler for s390.
 *
4
5
6
7
8
9
10
11
12
 * Minimum build requirements:
 *
 *  - HAVE_MARCH_Z196_FEATURES: laal, laalg
 *  - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj
 *  - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf
 *  - PACK_STACK
 *  - 64BIT
 *
 * Copyright IBM Corp. 2012,2015
13
14
 *
 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
15
 *	      Michael Holzheu <holzheu@linux.vnet.ibm.com>
16
 */
17
18
19
20

#define KMSG_COMPONENT "bpf_jit"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

21
22
#include <linux/netdevice.h>
#include <linux/filter.h>
23
#include <linux/init.h>
24
#include <linux/bpf.h>
25
#include <asm/cacheflush.h>
26
#include <asm/dis.h>
27
#include "bpf_jit.h"
28
29
30

int bpf_jit_enable __read_mostly;

31
32
33
34
35
36
37
38
39
40
41
42
43
struct bpf_jit {
	u32 seen;		/* Flags to remember seen eBPF instructions */
	u32 seen_reg[16];	/* Array to remember which registers are used */
	u32 *addrs;		/* Array with relative instruction addresses */
	u8 *prg_buf;		/* Start of program */
	int size;		/* Size of program and literal pool */
	int size_prg;		/* Size of program */
	int prg;		/* Current position in program */
	int lit_start;		/* Start of literal pool */
	int lit;		/* Current position in literal pool */
	int base_ip;		/* Base address for literal pool */
	int ret0_ip;		/* Address of return 0 */
	int exit_ip;		/* Address of exit */
44
45
	int tail_call_start;	/* Tail call start offset */
	int labels[1];		/* Labels for local jumps */
46
47
48
49
50
51
52
53
54
};

#define BPF_SIZE_MAX	4096	/* Max size for program */

#define SEEN_SKB	1	/* skb access */
#define SEEN_MEM	2	/* use mem[] for temporary storage */
#define SEEN_RET0	4	/* ret0_ip points to a valid return 0 */
#define SEEN_LITERAL	8	/* code uses literals */
#define SEEN_FUNC	16	/* calls C functions */
55
#define SEEN_TAIL_CALL	32	/* code uses tail calls */
56
57
#define SEEN_STACK	(SEEN_FUNC | SEEN_MEM | SEEN_SKB)

58
/*
59
 * s390 registers
60
 */
61
62
63
64
65
66
#define REG_W0		(__MAX_BPF_REG+0)	/* Work register 1 (even) */
#define REG_W1		(__MAX_BPF_REG+1)	/* Work register 2 (odd) */
#define REG_SKB_DATA	(__MAX_BPF_REG+2)	/* SKB data register */
#define REG_L		(__MAX_BPF_REG+3)	/* Literal pool register */
#define REG_15		(__MAX_BPF_REG+4)	/* Register 15 */
#define REG_0		REG_W0			/* Register 0 */
67
#define REG_1		REG_W1			/* Register 1 */
68
69
#define REG_2		BPF_REG_1		/* Register 2 */
#define REG_14		BPF_REG_0		/* Register 14 */
70

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
/*
 * Mapping of BPF registers to s390 registers
 */
static const int reg2hex[] = {
	/* Return code */
	[BPF_REG_0]	= 14,
	/* Function parameters */
	[BPF_REG_1]	= 2,
	[BPF_REG_2]	= 3,
	[BPF_REG_3]	= 4,
	[BPF_REG_4]	= 5,
	[BPF_REG_5]	= 6,
	/* Call saved registers */
	[BPF_REG_6]	= 7,
	[BPF_REG_7]	= 8,
	[BPF_REG_8]	= 9,
	[BPF_REG_9]	= 10,
	/* BPF stack pointer */
	[BPF_REG_FP]	= 13,
	/* SKB data pointer */
	[REG_SKB_DATA]	= 12,
	/* Work registers for s390x backend */
	[REG_W0]	= 0,
	[REG_W1]	= 1,
	[REG_L]		= 11,
	[REG_15]	= 15,
97
98
};

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
static inline u32 reg(u32 dst_reg, u32 src_reg)
{
	return reg2hex[dst_reg] << 4 | reg2hex[src_reg];
}

static inline u32 reg_high(u32 reg)
{
	return reg2hex[reg] << 4;
}

static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
{
	u32 r1 = reg2hex[b1];

	if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)
		jit->seen_reg[r1] = 1;
}

#define REG_SET_SEEN(b1)					\
({								\
	reg_set_seen(jit, b1);					\
})

#define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]

/*
 * EMIT macros for code generation
 */

#define _EMIT2(op)						\
({								\
	if (jit->prg_buf)					\
		*(u16 *) (jit->prg_buf + jit->prg) = op;	\
	jit->prg += 2;						\
})
134

135
136
137
138
139
#define EMIT2(op, b1, b2)					\
({								\
	_EMIT2(op | reg(b1, b2));				\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
140
141
})

142
143
144
145
146
#define _EMIT4(op)						\
({								\
	if (jit->prg_buf)					\
		*(u32 *) (jit->prg_buf + jit->prg) = op;	\
	jit->prg += 4;						\
147
148
})

149
150
151
152
153
#define EMIT4(op, b1, b2)					\
({								\
	_EMIT4(op | reg(b1, b2));				\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
154
155
})

156
157
158
159
160
161
#define EMIT4_RRF(op, b1, b2, b3)				\
({								\
	_EMIT4(op | reg_high(b3) << 8 | reg(b1, b2));		\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
	REG_SET_SEEN(b3);					\
162
163
})

164
165
166
167
#define _EMIT4_DISP(op, disp)					\
({								\
	unsigned int __disp = (disp) & 0xfff;			\
	_EMIT4(op | __disp);					\
168
169
})

170
171
172
173
174
175
#define EMIT4_DISP(op, b1, b2, disp)				\
({								\
	_EMIT4_DISP(op | reg_high(b1) << 16 |			\
		    reg_high(b2) << 8, disp);			\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
176
177
})

178
179
180
181
182
#define EMIT4_IMM(op, b1, imm)					\
({								\
	unsigned int __imm = (imm) & 0xffff;			\
	_EMIT4(op | reg_high(b1) << 16 | __imm);		\
	REG_SET_SEEN(b1);					\
183
184
})

185
186
187
188
#define EMIT4_PCREL(op, pcrel)					\
({								\
	long __pcrel = ((pcrel) >> 1) & 0xffff;			\
	_EMIT4(op | __pcrel);					\
189
190
})

191
192
193
194
195
196
197
#define _EMIT6(op1, op2)					\
({								\
	if (jit->prg_buf) {					\
		*(u32 *) (jit->prg_buf + jit->prg) = op1;	\
		*(u16 *) (jit->prg_buf + jit->prg + 4) = op2;	\
	}							\
	jit->prg += 6;						\
198
199
})

200
201
202
203
#define _EMIT6_DISP(op1, op2, disp)				\
({								\
	unsigned int __disp = (disp) & 0xfff;			\
	_EMIT6(op1 | __disp, op2);				\
204
205
})

206
207
208
209
210
211
212
213
214
215
216
#define EMIT6_DISP(op1, op2, b1, b2, b3, disp)			\
({								\
	_EMIT6_DISP(op1 | reg(b1, b2) << 16 |			\
		    reg_high(b3) << 8, op2, disp);		\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
	REG_SET_SEEN(b3);					\
})

#define _EMIT6_DISP_LH(op1, op2, disp)				\
({								\
217
218
219
	u32 _disp = (u32) disp;					\
	unsigned int __disp_h = _disp & 0xff000;		\
	unsigned int __disp_l = _disp & 0x00fff;		\
220
221
222
223
224
225
226
227
228
229
230
231
	_EMIT6(op1 | __disp_l, op2 | __disp_h >> 4);		\
})

#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp)		\
({								\
	_EMIT6_DISP_LH(op1 | reg(b1, b2) << 16 |		\
		       reg_high(b3) << 8, op2, disp);		\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
	REG_SET_SEEN(b3);					\
})

232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
#define EMIT6_PCREL_LABEL(op1, op2, b1, b2, label, mask)	\
({								\
	int rel = (jit->labels[label] - jit->prg) >> 1;		\
	_EMIT6(op1 | reg(b1, b2) << 16 | (rel & 0xffff),	\
	       op2 | mask << 12);				\
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
})

#define EMIT6_PCREL_IMM_LABEL(op1, op2, b1, imm, label, mask)	\
({								\
	int rel = (jit->labels[label] - jit->prg) >> 1;		\
	_EMIT6(op1 | (reg_high(b1) | mask) << 16 |		\
		(rel & 0xffff), op2 | (imm & 0xff) << 8);	\
	REG_SET_SEEN(b1);					\
	BUILD_BUG_ON(((unsigned long) imm) > 0xff);		\
})

250
251
252
253
#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask)		\
({								\
	/* Branch instruction needs 6 bytes */			\
	int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\
Michael Holzheu's avatar
Michael Holzheu committed
254
	_EMIT6(op1 | reg(b1, b2) << 16 | (rel & 0xffff), op2 | mask);	\
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
	REG_SET_SEEN(b1);					\
	REG_SET_SEEN(b2);					\
})

#define _EMIT6_IMM(op, imm)					\
({								\
	unsigned int __imm = (imm);				\
	_EMIT6(op | (__imm >> 16), __imm & 0xffff);		\
})

#define EMIT6_IMM(op, b1, imm)					\
({								\
	_EMIT6_IMM(op | reg_high(b1) << 16, imm);		\
	REG_SET_SEEN(b1);					\
})

#define EMIT_CONST_U32(val)					\
({								\
	unsigned int ret;					\
	ret = jit->lit - jit->base_ip;				\
	jit->seen |= SEEN_LITERAL;				\
	if (jit->prg_buf)					\
		*(u32 *) (jit->prg_buf + jit->lit) = (u32) val;	\
	jit->lit += 4;						\
	ret;							\
})

#define EMIT_CONST_U64(val)					\
({								\
	unsigned int ret;					\
	ret = jit->lit - jit->base_ip;				\
	jit->seen |= SEEN_LITERAL;				\
	if (jit->prg_buf)					\
		*(u64 *) (jit->prg_buf + jit->lit) = (u64) val;	\
	jit->lit += 8;						\
	ret;							\
})

#define EMIT_ZERO(b1)						\
({								\
	/* llgfr %dst,%dst (zero extend to 64 bit) */		\
	EMIT4(0xb9160000, b1, b1);				\
	REG_SET_SEEN(b1);					\
})

/*
 * Fill whole space with illegal instructions
 */
static void jit_fill_hole(void *area, unsigned int size)
304
305
306
307
{
	memset(area, 0, size);
}

308
309
310
311
312
/*
 * Save registers from "rs" (register start) to "re" (register end) on stack
 */
static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
{
313
	u32 off = STK_OFF_R6 + (rs - 6) * 8;
314
315
316
317
318
319
320
321
322
323
324
325
326

	if (rs == re)
		/* stg %rs,off(%r15) */
		_EMIT6(0xe300f000 | rs << 20 | off, 0x0024);
	else
		/* stmg %rs,%re,off(%r15) */
		_EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off);
}

/*
 * Restore registers from "rs" (register start) to "re" (register end) on stack
 */
static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)
327
{
328
	u32 off = STK_OFF_R6 + (rs - 6) * 8;
329
330
331
332
333
334
335
336
337
338
339

	if (jit->seen & SEEN_STACK)
		off += STK_OFF;

	if (rs == re)
		/* lg %rs,off(%r15) */
		_EMIT6(0xe300f000 | rs << 20 | off, 0x0004);
	else
		/* lmg %rs,%re,off(%r15) */
		_EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off);
}
340

341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
/*
 * Return first seen register (from start)
 */
static int get_start(struct bpf_jit *jit, int start)
{
	int i;

	for (i = start; i <= 15; i++) {
		if (jit->seen_reg[i])
			return i;
	}
	return 0;
}

/*
 * Return last seen register (from start) (gap >= 2)
 */
static int get_end(struct bpf_jit *jit, int start)
{
	int i;

	for (i = start; i < 15; i++) {
		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
			return i - 1;
	}
	return jit->seen_reg[15] ? 15 : 14;
}

#define REGS_SAVE	1
#define REGS_RESTORE	0
/*
 * Save and restore clobbered registers (6-15) on stack.
 * We save/restore registers in chunks with gap >= 2 registers.
 */
static void save_restore_regs(struct bpf_jit *jit, int op)
{

	int re = 6, rs;

	do {
		rs = get_start(jit, re);
		if (!rs)
			break;
		re = get_end(jit, rs + 1);
		if (op == REGS_SAVE)
			save_regs(jit, rs, re);
		else
			restore_regs(jit, rs, re);
		re++;
	} while (re <= 15);
}

/*
 * Emit function prologue
 *
 * Save registers and create stack frame if necessary.
 * See stack frame layout desription in "bpf_jit.h"!
 */
static void bpf_jit_prologue(struct bpf_jit *jit)
{
401
402
403
404
405
406
407
408
409
410
	if (jit->seen & SEEN_TAIL_CALL) {
		/* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */
		_EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);
	} else {
		/* j tail_call_start: NOP if no tail calls are used */
		EMIT4_PCREL(0xa7f40000, 6);
		_EMIT2(0);
	}
	/* Tail calls have to skip above initialization */
	jit->tail_call_start = jit->prg;
411
412
	/* Save registers */
	save_restore_regs(jit, REGS_SAVE);
413
414
415
	/* Setup literal pool */
	if (jit->seen & SEEN_LITERAL) {
		/* basr %r13,0 */
416
		EMIT2(0x0d00, REG_L, REG_0);
417
418
		jit->base_ip = jit->prg;
	}
419
420
	/* Setup stack and backchain */
	if (jit->seen & SEEN_STACK) {
421
422
423
424
425
		if (jit->seen & SEEN_FUNC)
			/* lgr %w1,%r15 (backchain) */
			EMIT4(0xb9040000, REG_W1, REG_15);
		/* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
		EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
426
427
428
		/* aghi %r15,-STK_OFF */
		EMIT4_IMM(0xa70b0000, REG_15, -STK_OFF);
		if (jit->seen & SEEN_FUNC)
429
430
			/* stg %w1,152(%r15) (backchain) */
			EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
				      REG_15, 152);
	}
	/*
	 * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
	 * we store the SKB header length on the stack and the SKB data
	 * pointer in REG_SKB_DATA.
	 */
	if (jit->seen & SEEN_SKB) {
		/* Header length: llgf %w1,<len>(%b1) */
		EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
			      offsetof(struct sk_buff, len));
		/* s %w1,<data_len>(%b1) */
		EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
			   offsetof(struct sk_buff, data_len));
		/* stg %w1,ST_OFF_HLEN(%r0,%r15) */
		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15,
			      STK_OFF_HLEN);
		/* lg %skb_data,data_off(%b1) */
		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
			      BPF_REG_1, offsetof(struct sk_buff, data));
451
	}
452
453
454
455
456
457
458
	/* BPF compatibility: clear A (%b0) and X (%b7) registers */
	if (REG_SEEN(BPF_REG_A))
		/* lghi %ba,0 */
		EMIT4_IMM(0xa7090000, BPF_REG_A, 0);
	if (REG_SEEN(BPF_REG_X))
		/* lghi %bx,0 */
		EMIT4_IMM(0xa7090000, BPF_REG_X, 0);
459
460
}

461
462
463
/*
 * Function epilogue
 */
464
465
466
467
468
static void bpf_jit_epilogue(struct bpf_jit *jit)
{
	/* Return 0 */
	if (jit->seen & SEEN_RET0) {
		jit->ret0_ip = jit->prg;
469
470
		/* lghi %b0,0 */
		EMIT4_IMM(0xa7090000, BPF_REG_0, 0);
471
472
	}
	jit->exit_ip = jit->prg;
473
474
	/* Load exit code: lgr %r2,%b0 */
	EMIT4(0xb9040000, REG_2, BPF_REG_0);
475
	/* Restore registers */
476
	save_restore_regs(jit, REGS_RESTORE);
477
	/* br %r14 */
478
	_EMIT2(0x07fe);
479
480
481
}

/*
482
 * Compile one eBPF instruction into s390x code
483
484
485
 *
 * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of
 * stack space for the large switch statement.
486
 */
487
static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)
488
{
489
490
491
492
493
494
495
496
	struct bpf_insn *insn = &fp->insnsi[i];
	int jmp_off, last, insn_count = 1;
	unsigned int func_addr, mask;
	u32 dst_reg = insn->dst_reg;
	u32 src_reg = insn->src_reg;
	u32 *addrs = jit->addrs;
	s32 imm = insn->imm;
	s16 off = insn->off;
497

498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
	switch (insn->code) {
	/*
	 * BPF_MOV
	 */
	case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */
		/* llgfr %dst,%src */
		EMIT4(0xb9160000, dst_reg, src_reg);
		break;
	case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
		/* lgr %dst,%src */
		EMIT4(0xb9040000, dst_reg, src_reg);
		break;
	case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */
		/* llilf %dst,imm */
		EMIT6_IMM(0xc00f0000, dst_reg, imm);
		break;
	case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */
		/* lgfi %dst,imm */
		EMIT6_IMM(0xc0010000, dst_reg, imm);
		break;
	/*
	 * BPF_LD 64
	 */
	case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
	{
		/* 16 byte instruction that uses two 'struct bpf_insn' */
		u64 imm64;

		imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32;
		/* lg %dst,<d(imm)>(%l) */
		EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L,
			      EMIT_CONST_U64(imm64));
		insn_count = 2;
		break;
	}
	/*
	 * BPF_ADD
	 */
	case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */
		/* ar %dst,%src */
		EMIT2(0x1a00, dst_reg, src_reg);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */
		/* agr %dst,%src */
		EMIT4(0xb9080000, dst_reg, src_reg);
		break;
	case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */
		if (!imm)
547
			break;
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
		/* alfi %dst,imm */
		EMIT6_IMM(0xc20b0000, dst_reg, imm);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */
		if (!imm)
			break;
		/* agfi %dst,imm */
		EMIT6_IMM(0xc2080000, dst_reg, imm);
		break;
	/*
	 * BPF_SUB
	 */
	case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */
		/* sr %dst,%src */
		EMIT2(0x1b00, dst_reg, src_reg);
		EMIT_ZERO(dst_reg);
565
		break;
566
567
568
	case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */
		/* sgr %dst,%src */
		EMIT4(0xb9090000, dst_reg, src_reg);
569
		break;
570
571
	case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */
		if (!imm)
572
			break;
573
574
575
		/* alfi %dst,-imm */
		EMIT6_IMM(0xc20b0000, dst_reg, -imm);
		EMIT_ZERO(dst_reg);
576
		break;
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
	case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */
		if (!imm)
			break;
		/* agfi %dst,-imm */
		EMIT6_IMM(0xc2080000, dst_reg, -imm);
		break;
	/*
	 * BPF_MUL
	 */
	case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */
		/* msr %dst,%src */
		EMIT4(0xb2520000, dst_reg, src_reg);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */
		/* msgr %dst,%src */
		EMIT4(0xb90c0000, dst_reg, src_reg);
		break;
	case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */
		if (imm == 1)
			break;
		/* msfi %r5,imm */
		EMIT6_IMM(0xc2010000, dst_reg, imm);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */
		if (imm == 1)
604
			break;
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
		/* msgfi %dst,imm */
		EMIT6_IMM(0xc2000000, dst_reg, imm);
		break;
	/*
	 * BPF_DIV / BPF_MOD
	 */
	case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */
	case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */
	{
		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;

		jit->seen |= SEEN_RET0;
		/* ltr %src,%src (if src == 0 goto fail) */
		EMIT2(0x1200, src_reg, src_reg);
		/* jz <ret0> */
		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
		/* lhi %w0,0 */
		EMIT4_IMM(0xa7080000, REG_W0, 0);
		/* lr %w1,%dst */
		EMIT2(0x1800, REG_W1, dst_reg);
		/* dlr %w0,%src */
		EMIT4(0xb9970000, REG_W0, src_reg);
		/* llgfr %dst,%rc */
		EMIT4(0xb9160000, dst_reg, rc_reg);
		break;
	}
631
632
	case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / src */
	case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % src */
633
634
635
636
637
638
	{
		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;

		jit->seen |= SEEN_RET0;
		/* ltgr %src,%src (if src == 0 goto fail) */
		EMIT4(0xb9020000, src_reg, src_reg);
639
		/* jz <ret0> */
640
641
642
643
644
645
		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
		/* lghi %w0,0 */
		EMIT4_IMM(0xa7090000, REG_W0, 0);
		/* lgr %w1,%dst */
		EMIT4(0xb9040000, REG_W1, dst_reg);
		/* dlgr %w0,%dst */
646
		EMIT4(0xb9870000, REG_W0, src_reg);
647
648
649
650
651
652
653
654
655
656
657
658
659
		/* lgr %dst,%rc */
		EMIT4(0xb9040000, dst_reg, rc_reg);
		break;
	}
	case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */
	case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */
	{
		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;

		if (imm == 1) {
			if (BPF_OP(insn->code) == BPF_MOD)
				/* lhgi %dst,0 */
				EMIT4_IMM(0xa7090000, dst_reg, 0);
660
661
			break;
		}
662
663
664
665
666
667
668
669
670
671
672
		/* lhi %w0,0 */
		EMIT4_IMM(0xa7080000, REG_W0, 0);
		/* lr %w1,%dst */
		EMIT2(0x1800, REG_W1, dst_reg);
		/* dl %w0,<d(imm)>(%l) */
		EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
			      EMIT_CONST_U32(imm));
		/* llgfr %dst,%rc */
		EMIT4(0xb9160000, dst_reg, rc_reg);
		break;
	}
673
674
	case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / imm */
	case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % imm */
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
	{
		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;

		if (imm == 1) {
			if (BPF_OP(insn->code) == BPF_MOD)
				/* lhgi %dst,0 */
				EMIT4_IMM(0xa7090000, dst_reg, 0);
			break;
		}
		/* lghi %w0,0 */
		EMIT4_IMM(0xa7090000, REG_W0, 0);
		/* lgr %w1,%dst */
		EMIT4(0xb9040000, REG_W1, dst_reg);
		/* dlg %w0,<d(imm)>(%l) */
		EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
690
			      EMIT_CONST_U64(imm));
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
		/* lgr %dst,%rc */
		EMIT4(0xb9040000, dst_reg, rc_reg);
		break;
	}
	/*
	 * BPF_AND
	 */
	case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */
		/* nr %dst,%src */
		EMIT2(0x1400, dst_reg, src_reg);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
		/* ngr %dst,%src */
		EMIT4(0xb9800000, dst_reg, src_reg);
		break;
	case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */
		/* nilf %dst,imm */
		EMIT6_IMM(0xc00b0000, dst_reg, imm);
		EMIT_ZERO(dst_reg);
711
		break;
712
713
714
715
	case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
		/* ng %dst,<d(imm)>(%l) */
		EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L,
			      EMIT_CONST_U64(imm));
716
		break;
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
	/*
	 * BPF_OR
	 */
	case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
		/* or %dst,%src */
		EMIT2(0x1600, dst_reg, src_reg);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
		/* ogr %dst,%src */
		EMIT4(0xb9810000, dst_reg, src_reg);
		break;
	case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */
		/* oilf %dst,imm */
		EMIT6_IMM(0xc00d0000, dst_reg, imm);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */
		/* og %dst,<d(imm)>(%l) */
		EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L,
			      EMIT_CONST_U64(imm));
		break;
	/*
	 * BPF_XOR
	 */
	case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */
		/* xr %dst,%src */
		EMIT2(0x1700, dst_reg, src_reg);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */
		/* xgr %dst,%src */
		EMIT4(0xb9820000, dst_reg, src_reg);
		break;
	case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */
		if (!imm)
753
			break;
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
		/* xilf %dst,imm */
		EMIT6_IMM(0xc0070000, dst_reg, imm);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */
		/* xg %dst,<d(imm)>(%l) */
		EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L,
			      EMIT_CONST_U64(imm));
		break;
	/*
	 * BPF_LSH
	 */
	case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */
		/* sll %dst,0(%src) */
		EMIT4_DISP(0x89000000, dst_reg, src_reg, 0);
		EMIT_ZERO(dst_reg);
770
		break;
771
772
773
	case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */
		/* sllg %dst,%dst,0(%src) */
		EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0);
774
		break;
775
776
	case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */
		if (imm == 0)
777
			break;
778
779
780
		/* sll %dst,imm(%r0) */
		EMIT4_DISP(0x89000000, dst_reg, REG_0, imm);
		EMIT_ZERO(dst_reg);
781
		break;
782
783
784
785
786
787
788
789
790
791
792
793
794
	case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */
		if (imm == 0)
			break;
		/* sllg %dst,%dst,imm(%r0) */
		EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm);
		break;
	/*
	 * BPF_RSH
	 */
	case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */
		/* srl %dst,0(%src) */
		EMIT4_DISP(0x88000000, dst_reg, src_reg, 0);
		EMIT_ZERO(dst_reg);
795
		break;
796
797
798
799
800
801
	case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */
		/* srlg %dst,%dst,0(%src) */
		EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0);
		break;
	case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */
		if (imm == 0)
802
			break;
803
804
805
806
807
808
		/* srl %dst,imm(%r0) */
		EMIT4_DISP(0x88000000, dst_reg, REG_0, imm);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */
		if (imm == 0)
809
			break;
810
811
		/* srlg %dst,%dst,imm(%r0) */
		EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm);
812
		break;
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
	/*
	 * BPF_ARSH
	 */
	case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
		/* srag %dst,%dst,0(%src) */
		EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
		break;
	case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
		if (imm == 0)
			break;
		/* srag %dst,%dst,imm(%r0) */
		EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm);
		break;
	/*
	 * BPF_NEG
	 */
	case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */
		/* lcr %dst,%dst */
		EMIT2(0x1300, dst_reg, dst_reg);
		EMIT_ZERO(dst_reg);
		break;
	case BPF_ALU64 | BPF_NEG: /* dst = -dst */
		/* lcgr %dst,%dst */
		EMIT4(0xb9130000, dst_reg, dst_reg);
		break;
	/*
	 * BPF_FROM_BE/LE
	 */
	case BPF_ALU | BPF_END | BPF_FROM_BE:
		/* s390 is big endian, therefore only clear high order bytes */
		switch (imm) {
		case 16: /* dst = (u16) cpu_to_be16(dst) */
			/* llghr %dst,%dst */
			EMIT4(0xb9850000, dst_reg, dst_reg);
			break;
		case 32: /* dst = (u32) cpu_to_be32(dst) */
			/* llgfr %dst,%dst */
			EMIT4(0xb9160000, dst_reg, dst_reg);
			break;
		case 64: /* dst = (u64) cpu_to_be64(dst) */
			break;
854
855
		}
		break;
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
	case BPF_ALU | BPF_END | BPF_FROM_LE:
		switch (imm) {
		case 16: /* dst = (u16) cpu_to_le16(dst) */
			/* lrvr %dst,%dst */
			EMIT4(0xb91f0000, dst_reg, dst_reg);
			/* srl %dst,16(%r0) */
			EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);
			/* llghr %dst,%dst */
			EMIT4(0xb9850000, dst_reg, dst_reg);
			break;
		case 32: /* dst = (u32) cpu_to_le32(dst) */
			/* lrvr %dst,%dst */
			EMIT4(0xb91f0000, dst_reg, dst_reg);
			/* llgfr %dst,%dst */
			EMIT4(0xb9160000, dst_reg, dst_reg);
			break;
		case 64: /* dst = (u64) cpu_to_le64(dst) */
			/* lrvgr %dst,%dst */
			EMIT4(0xb90f0000, dst_reg, dst_reg);
875
876
877
			break;
		}
		break;
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
	/*
	 * BPF_ST(X)
	 */
	case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */
		/* stcy %src,off(%dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
		/* sthy %src,off(%dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
		/* sty %src,off(%dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
		/* stg %src,off(%dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
		/* lhi %w0,imm */
		EMIT4_IMM(0xa7080000, REG_W0, (u8) imm);
		/* stcy %w0,off(dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
		/* lhi %w0,imm */
		EMIT4_IMM(0xa7080000, REG_W0, (u16) imm);
		/* sthy %w0,off(dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, REG_0, off);
913
914
		jit->seen |= SEEN_MEM;
		break;
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
	case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
		/* llilf %w0,imm  */
		EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm);
		/* sty %w0,off(%dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
		/* lgfi %w0,imm */
		EMIT6_IMM(0xc0010000, REG_W0, imm);
		/* stg %w0,off(%dst) */
		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	/*
	 * BPF_STX XADD (atomic_add)
	 */
	case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */
		/* laal %w0,%src,off(%dst) */
		EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W0, src_reg,
			      dst_reg, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */
		/* laalg %w0,%src,off(%dst) */
		EMIT6_DISP_LH(0xeb000000, 0x00ea, REG_W0, src_reg,
			      dst_reg, off);
		jit->seen |= SEEN_MEM;
		break;
	/*
	 * BPF_LDX
	 */
	case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
		/* llgc %dst,0(off,%src) */
		EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
		/* llgh %dst,0(off,%src) */
		EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
		jit->seen |= SEEN_MEM;
		break;
	case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
		/* llgf %dst,off(%src) */
		jit->seen |= SEEN_MEM;
		EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
		break;
	case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
		/* lg %dst,0(off,%src) */
		jit->seen |= SEEN_MEM;
		EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
		break;
	/*
	 * BPF_JMP / CALL
	 */
	case BPF_JMP | BPF_CALL:
	{
		/*
		 * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5)
		 */
		const u64 func = (u64)__bpf_call_base + imm;

977
978
979
980
		if (bpf_helper_changes_skb_data((void *)func))
			/* TODO reload skb->data, hlen */
			return -1;

981
982
983
984
985
986
987
988
989
990
991
		REG_SET_SEEN(BPF_REG_5);
		jit->seen |= SEEN_FUNC;
		/* lg %w1,<d(imm)>(%l) */
		EMIT6_DISP(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
			   EMIT_CONST_U64(func));
		/* basr %r14,%w1 */
		EMIT2(0x0d00, REG_14, REG_W1);
		/* lgr %b0,%r2: load return value into %b0 */
		EMIT4(0xb9040000, BPF_REG_0, REG_2);
		break;
	}
992
993
994
995
996
997
998
999
1000
	case BPF_JMP | BPF_CALL | BPF_X:
		/*
		 * Implicit input:
		 *  B1: pointer to ctx
		 *  B2: pointer to bpf_array
		 *  B3: index in bpf_array
		 */
		jit->seen |= SEEN_TAIL_CALL;

For faster browsing, not all history is shown. View entire blame