cls_flower.c 61.5 KB
Newer Older
Jiri Pirko's avatar
Jiri Pirko committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 * net/sched/cls_flower.c		Flower classifier
 *
 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/rhashtable.h>
16
#include <linux/workqueue.h>
Jiri Pirko's avatar
Jiri Pirko committed
17
18
19
20

#include <linux/if_ether.h>
#include <linux/in6.h>
#include <linux/ip.h>
21
#include <linux/mpls.h>
Jiri Pirko's avatar
Jiri Pirko committed
22
23
24
25
26

#include <net/sch_generic.h>
#include <net/pkt_cls.h>
#include <net/ip.h>
#include <net/flow_dissector.h>
27
#include <net/geneve.h>
Jiri Pirko's avatar
Jiri Pirko committed
28

29
30
31
#include <net/dst.h>
#include <net/dst_metadata.h>

Jiri Pirko's avatar
Jiri Pirko committed
32
33
struct fl_flow_key {
	int	indev_ifindex;
34
	struct flow_dissector_key_control control;
35
	struct flow_dissector_key_control enc_control;
Jiri Pirko's avatar
Jiri Pirko committed
36
37
	struct flow_dissector_key_basic basic;
	struct flow_dissector_key_eth_addrs eth;
38
	struct flow_dissector_key_vlan vlan;
39
	struct flow_dissector_key_vlan cvlan;
Jiri Pirko's avatar
Jiri Pirko committed
40
	union {
41
		struct flow_dissector_key_ipv4_addrs ipv4;
Jiri Pirko's avatar
Jiri Pirko committed
42
43
44
		struct flow_dissector_key_ipv6_addrs ipv6;
	};
	struct flow_dissector_key_ports tp;
45
	struct flow_dissector_key_icmp icmp;
46
	struct flow_dissector_key_arp arp;
47
48
49
50
51
	struct flow_dissector_key_keyid enc_key_id;
	union {
		struct flow_dissector_key_ipv4_addrs enc_ipv4;
		struct flow_dissector_key_ipv6_addrs enc_ipv6;
	};
52
	struct flow_dissector_key_ports enc_tp;
53
	struct flow_dissector_key_mpls mpls;
54
	struct flow_dissector_key_tcp tcp;
55
	struct flow_dissector_key_ip ip;
56
	struct flow_dissector_key_ip enc_ip;
57
	struct flow_dissector_key_enc_opts enc_opts;
58
59
	struct flow_dissector_key_ports tp_min;
	struct flow_dissector_key_ports tp_max;
Jiri Pirko's avatar
Jiri Pirko committed
60
61
62
63
64
65
66
67
68
69
} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */

struct fl_flow_mask_range {
	unsigned short int start;
	unsigned short int end;
};

struct fl_flow_mask {
	struct fl_flow_key key;
	struct fl_flow_mask_range range;
70
	u32 flags;
71
72
73
74
75
	struct rhash_head ht_node;
	struct rhashtable ht;
	struct rhashtable_params filter_ht_params;
	struct flow_dissector dissector;
	struct list_head filters;
76
	struct rcu_work rwork;
77
	struct list_head list;
Jiri Pirko's avatar
Jiri Pirko committed
78
79
};

80
81
82
83
84
85
86
struct fl_flow_tmplt {
	struct fl_flow_key dummy_key;
	struct fl_flow_key mask;
	struct flow_dissector dissector;
	struct tcf_chain *chain;
};

Jiri Pirko's avatar
Jiri Pirko committed
87
88
struct cls_fl_head {
	struct rhashtable ht;
89
	struct list_head masks;
Cong Wang's avatar
Cong Wang committed
90
	struct rcu_work rwork;
91
	struct idr handle_idr;
Jiri Pirko's avatar
Jiri Pirko committed
92
93
94
};

struct cls_fl_filter {
95
	struct fl_flow_mask *mask;
Jiri Pirko's avatar
Jiri Pirko committed
96
97
98
99
100
101
102
	struct rhash_head ht_node;
	struct fl_flow_key mkey;
	struct tcf_exts exts;
	struct tcf_result res;
	struct fl_flow_key key;
	struct list_head list;
	u32 handle;
103
	u32 flags;
104
	u32 in_hw_count;
Cong Wang's avatar
Cong Wang committed
105
	struct rcu_work rwork;
106
	struct net_device *hw_dev;
Jiri Pirko's avatar
Jiri Pirko committed
107
108
};

109
110
111
112
113
114
115
static const struct rhashtable_params mask_ht_params = {
	.key_offset = offsetof(struct fl_flow_mask, key),
	.key_len = sizeof(struct fl_flow_key),
	.head_offset = offsetof(struct fl_flow_mask, ht_node),
	.automatic_shrinking = true,
};

Jiri Pirko's avatar
Jiri Pirko committed
116
117
118
119
120
121
122
123
124
static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
{
	return mask->range.end - mask->range.start;
}

static void fl_mask_update_range(struct fl_flow_mask *mask)
{
	const u8 *bytes = (const u8 *) &mask->key;
	size_t size = sizeof(mask->key);
125
	size_t i, first = 0, last;
Jiri Pirko's avatar
Jiri Pirko committed
126

127
128
129
130
131
132
133
134
	for (i = 0; i < size; i++) {
		if (bytes[i]) {
			first = i;
			break;
		}
	}
	last = first;
	for (i = size - 1; i != first; i--) {
Jiri Pirko's avatar
Jiri Pirko committed
135
136
		if (bytes[i]) {
			last = i;
137
			break;
Jiri Pirko's avatar
Jiri Pirko committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
		}
	}
	mask->range.start = rounddown(first, sizeof(long));
	mask->range.end = roundup(last + 1, sizeof(long));
}

static void *fl_key_get_start(struct fl_flow_key *key,
			      const struct fl_flow_mask *mask)
{
	return (u8 *) key + mask->range.start;
}

static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
			      struct fl_flow_mask *mask)
{
	const long *lkey = fl_key_get_start(key, mask);
	const long *lmask = fl_key_get_start(&mask->key, mask);
	long *lmkey = fl_key_get_start(mkey, mask);
	int i;

	for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
		*lmkey++ = *lkey++ & *lmask++;
}

162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt,
			       struct fl_flow_mask *mask)
{
	const long *lmask = fl_key_get_start(&mask->key, mask);
	const long *ltmplt;
	int i;

	if (!tmplt)
		return true;
	ltmplt = fl_key_get_start(&tmplt->mask, mask);
	for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) {
		if (~*ltmplt++ & *lmask++)
			return false;
	}
	return true;
}

Jiri Pirko's avatar
Jiri Pirko committed
179
180
181
182
183
184
static void fl_clear_masked_range(struct fl_flow_key *key,
				  struct fl_flow_mask *mask)
{
	memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
}

185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter,
				  struct fl_flow_key *key,
				  struct fl_flow_key *mkey)
{
	__be16 min_mask, max_mask, min_val, max_val;

	min_mask = htons(filter->mask->key.tp_min.dst);
	max_mask = htons(filter->mask->key.tp_max.dst);
	min_val = htons(filter->key.tp_min.dst);
	max_val = htons(filter->key.tp_max.dst);

	if (min_mask && max_mask) {
		if (htons(key->tp.dst) < min_val ||
		    htons(key->tp.dst) > max_val)
			return false;

		/* skb does not have min and max values */
		mkey->tp_min.dst = filter->mkey.tp_min.dst;
		mkey->tp_max.dst = filter->mkey.tp_max.dst;
	}
	return true;
}

static bool fl_range_port_src_cmp(struct cls_fl_filter *filter,
				  struct fl_flow_key *key,
				  struct fl_flow_key *mkey)
{
	__be16 min_mask, max_mask, min_val, max_val;

	min_mask = htons(filter->mask->key.tp_min.src);
	max_mask = htons(filter->mask->key.tp_max.src);
	min_val = htons(filter->key.tp_min.src);
	max_val = htons(filter->key.tp_max.src);

	if (min_mask && max_mask) {
		if (htons(key->tp.src) < min_val ||
		    htons(key->tp.src) > max_val)
			return false;

		/* skb does not have min and max values */
		mkey->tp_min.src = filter->mkey.tp_min.src;
		mkey->tp_max.src = filter->mkey.tp_max.src;
	}
	return true;
}

static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask,
					 struct fl_flow_key *mkey)
233
{
234
235
	return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask),
				      mask->filter_ht_params);
236
237
}

238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask,
					     struct fl_flow_key *mkey,
					     struct fl_flow_key *key)
{
	struct cls_fl_filter *filter, *f;

	list_for_each_entry_rcu(filter, &mask->filters, list) {
		if (!fl_range_port_dst_cmp(filter, key, mkey))
			continue;

		if (!fl_range_port_src_cmp(filter, key, mkey))
			continue;

		f = __fl_lookup(mask, mkey);
		if (f)
			return f;
	}
	return NULL;
}

static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask,
				       struct fl_flow_key *mkey,
				       struct fl_flow_key *key)
{
	if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE))
		return fl_lookup_range(mask, mkey, key);

	return __fl_lookup(mask, mkey);
}

Jiri Pirko's avatar
Jiri Pirko committed
268
269
270
271
272
static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
		       struct tcf_result *res)
{
	struct cls_fl_head *head = rcu_dereference_bh(tp->root);
	struct cls_fl_filter *f;
273
	struct fl_flow_mask *mask;
Jiri Pirko's avatar
Jiri Pirko committed
274
275
276
	struct fl_flow_key skb_key;
	struct fl_flow_key skb_mkey;

277
278
	list_for_each_entry_rcu(mask, &head->masks, list) {
		fl_clear_masked_range(&skb_key, mask);
279

280
281
282
283
284
285
286
		skb_key.indev_ifindex = skb->skb_iif;
		/* skb_flow_dissect() does not set n_proto in case an unknown
		 * protocol, so do it rather here.
		 */
		skb_key.basic.n_proto = skb->protocol;
		skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key);
		skb_flow_dissect(skb, &mask->dissector, &skb_key, 0);
Jiri Pirko's avatar
Jiri Pirko committed
287

288
		fl_set_masked_key(&skb_mkey, &skb_key, mask);
Jiri Pirko's avatar
Jiri Pirko committed
289

290
		f = fl_lookup(mask, &skb_mkey, &skb_key);
291
292
293
294
		if (f && !tc_skip_sw(f->flags)) {
			*res = f->res;
			return tcf_exts_exec(skb, &f->exts, res);
		}
Jiri Pirko's avatar
Jiri Pirko committed
295
296
297
298
299
300
301
302
303
304
305
306
	}
	return -1;
}

static int fl_init(struct tcf_proto *tp)
{
	struct cls_fl_head *head;

	head = kzalloc(sizeof(*head), GFP_KERNEL);
	if (!head)
		return -ENOBUFS;

307
	INIT_LIST_HEAD_RCU(&head->masks);
Jiri Pirko's avatar
Jiri Pirko committed
308
	rcu_assign_pointer(tp->root, head);
309
	idr_init(&head->handle_idr);
Jiri Pirko's avatar
Jiri Pirko committed
310

311
312
313
	return rhashtable_init(&head->ht, &mask_ht_params);
}

314
315
316
317
318
319
320
321
322
323
324
325
326
327
static void fl_mask_free(struct fl_flow_mask *mask)
{
	rhashtable_destroy(&mask->ht);
	kfree(mask);
}

static void fl_mask_free_work(struct work_struct *work)
{
	struct fl_flow_mask *mask = container_of(to_rcu_work(work),
						 struct fl_flow_mask, rwork);

	fl_mask_free(mask);
}

328
329
330
331
332
333
334
335
336
static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask,
			bool async)
{
	if (!list_empty(&mask->filters))
		return false;

	rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params);
	list_del_rcu(&mask->list);
	if (async)
337
		tcf_queue_work(&mask->rwork, fl_mask_free_work);
338
	else
339
		fl_mask_free(mask);
340
341

	return true;
Jiri Pirko's avatar
Jiri Pirko committed
342
343
}

344
345
346
347
348
349
350
static void __fl_destroy_filter(struct cls_fl_filter *f)
{
	tcf_exts_destroy(&f->exts);
	tcf_exts_put_net(&f->exts);
	kfree(f);
}

351
static void fl_destroy_filter_work(struct work_struct *work)
Jiri Pirko's avatar
Jiri Pirko committed
352
{
Cong Wang's avatar
Cong Wang committed
353
354
	struct cls_fl_filter *f = container_of(to_rcu_work(work),
					struct cls_fl_filter, rwork);
Jiri Pirko's avatar
Jiri Pirko committed
355

356
	rtnl_lock();
357
	__fl_destroy_filter(f);
358
359
360
	rtnl_unlock();
}

361
362
static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f,
				 struct netlink_ext_ack *extack)
363
{
364
	struct tc_cls_flower_offload cls_flower = {};
365
	struct tcf_block *block = tp->chain->block;
366

367
	tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
368
369
	cls_flower.command = TC_CLSFLOWER_DESTROY;
	cls_flower.cookie = (unsigned long) f;
370

371
	tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
372
	tcf_block_offload_dec(block, &f->flags);
373
374
}

375
static int fl_hw_replace_filter(struct tcf_proto *tp,
376
377
				struct cls_fl_filter *f,
				struct netlink_ext_ack *extack)
378
{
379
	struct tc_cls_flower_offload cls_flower = {};
380
	struct tcf_block *block = tp->chain->block;
381
	bool skip_sw = tc_skip_sw(f->flags);
382
	int err;
383

384
	tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
385
386
	cls_flower.command = TC_CLSFLOWER_REPLACE;
	cls_flower.cookie = (unsigned long) f;
387
388
	cls_flower.dissector = &f->mask->dissector;
	cls_flower.mask = &f->mask->key;
389
390
	cls_flower.key = &f->mkey;
	cls_flower.exts = &f->exts;
391
	cls_flower.classid = f->res.classid;
392

393
	err = tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, skip_sw);
394
	if (err < 0) {
395
		fl_hw_destroy_filter(tp, f, NULL);
396
		return err;
397
	} else if (err > 0) {
398
		f->in_hw_count = err;
399
		tcf_block_offload_inc(block, &f->flags);
400
401
402
403
404
	}

	if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW))
		return -EINVAL;

405
	return 0;
406
407
}

408
409
static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
{
410
	struct tc_cls_flower_offload cls_flower = {};
411
	struct tcf_block *block = tp->chain->block;
412

413
	tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL);
414
415
416
	cls_flower.command = TC_CLSFLOWER_STATS;
	cls_flower.cookie = (unsigned long) f;
	cls_flower.exts = &f->exts;
417
	cls_flower.classid = f->res.classid;
418

419
	tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
420
421
}

422
static bool __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f,
423
			struct netlink_ext_ack *extack)
424
{
425
	struct cls_fl_head *head = rtnl_dereference(tp->root);
426
427
	bool async = tcf_exts_get_net(&f->exts);
	bool last;
428

429
	idr_remove(&head->handle_idr, f->handle);
430
	list_del_rcu(&f->list);
431
	last = fl_mask_put(head, f->mask, async);
432
	if (!tc_skip_hw(f->flags))
433
		fl_hw_destroy_filter(tp, f, extack);
434
	tcf_unbind_filter(tp, &f->res);
435
	if (async)
Cong Wang's avatar
Cong Wang committed
436
		tcf_queue_work(&f->rwork, fl_destroy_filter_work);
437
438
	else
		__fl_destroy_filter(f);
439
440

	return last;
441
442
}

443
444
static void fl_destroy_sleepable(struct work_struct *work)
{
Cong Wang's avatar
Cong Wang committed
445
446
447
	struct cls_fl_head *head = container_of(to_rcu_work(work),
						struct cls_fl_head,
						rwork);
448
449

	rhashtable_destroy(&head->ht);
450
451
452
453
	kfree(head);
	module_put(THIS_MODULE);
}

454
static void fl_destroy(struct tcf_proto *tp, struct netlink_ext_ack *extack)
Jiri Pirko's avatar
Jiri Pirko committed
455
456
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);
457
	struct fl_flow_mask *mask, *next_mask;
Jiri Pirko's avatar
Jiri Pirko committed
458
459
	struct cls_fl_filter *f, *next;

460
461
462
463
464
465
	list_for_each_entry_safe(mask, next_mask, &head->masks, list) {
		list_for_each_entry_safe(f, next, &mask->filters, list) {
			if (__fl_delete(tp, f, extack))
				break;
		}
	}
466
	idr_destroy(&head->handle_idr);
467
468

	__module_get(THIS_MODULE);
Cong Wang's avatar
Cong Wang committed
469
	tcf_queue_work(&head->rwork, fl_destroy_sleepable);
Jiri Pirko's avatar
Jiri Pirko committed
470
471
}

472
static void *fl_get(struct tcf_proto *tp, u32 handle)
Jiri Pirko's avatar
Jiri Pirko committed
473
474
475
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);

476
	return idr_find(&head->handle_idr, handle);
Jiri Pirko's avatar
Jiri Pirko committed
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
}

static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
	[TCA_FLOWER_UNSPEC]		= { .type = NLA_UNSPEC },
	[TCA_FLOWER_CLASSID]		= { .type = NLA_U32 },
	[TCA_FLOWER_INDEV]		= { .type = NLA_STRING,
					    .len = IFNAMSIZ },
	[TCA_FLOWER_KEY_ETH_DST]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_DST_MASK]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_SRC]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_SRC_MASK]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_TYPE]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_IP_PROTO]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IPV4_SRC]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV4_SRC_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV4_DST]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV4_DST_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV6_SRC]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_IPV6_SRC_MASK]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_IPV6_DST]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_IPV6_DST_MASK]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_TCP_SRC]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_TCP_DST]	= { .type = NLA_U16 },
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
500
501
	[TCA_FLOWER_KEY_UDP_SRC]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_DST]	= { .type = NLA_U16 },
502
503
504
	[TCA_FLOWER_KEY_VLAN_ID]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_VLAN_PRIO]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_VLAN_ETH_TYPE]	= { .type = NLA_U16 },
505
506
507
508
509
510
511
512
513
	[TCA_FLOWER_KEY_ENC_KEY_ID]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_SRC]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_DST]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV6_SRC]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_ENC_IPV6_DST]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
514
515
516
517
	[TCA_FLOWER_KEY_TCP_SRC_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_TCP_DST_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_SRC_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_DST_MASK]	= { .type = NLA_U16 },
518
519
520
521
	[TCA_FLOWER_KEY_SCTP_SRC_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_SCTP_DST_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_SCTP_SRC]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_SCTP_DST]	= { .type = NLA_U16 },
522
523
524
525
	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK]	= { .type = NLA_U16 },
526
527
	[TCA_FLOWER_KEY_FLAGS]		= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_FLAGS_MASK]	= { .type = NLA_U32 },
528
529
530
531
532
533
534
535
	[TCA_FLOWER_KEY_ICMPV4_TYPE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV4_CODE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_TYPE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_CODE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
536
537
538
539
540
541
542
543
544
545
	[TCA_FLOWER_KEY_ARP_SIP]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_SIP_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_TIP]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_TIP_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_OP]		= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ARP_OP_MASK]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ARP_SHA]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ARP_SHA_MASK]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ARP_THA]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ARP_THA_MASK]	= { .len = ETH_ALEN },
546
547
548
549
	[TCA_FLOWER_KEY_MPLS_TTL]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_MPLS_BOS]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_MPLS_TC]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_MPLS_LABEL]	= { .type = NLA_U32 },
550
551
	[TCA_FLOWER_KEY_TCP_FLAGS]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_TCP_FLAGS_MASK]	= { .type = NLA_U16 },
552
553
554
555
	[TCA_FLOWER_KEY_IP_TOS]		= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IP_TOS_MASK]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IP_TTL]		= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IP_TTL_MASK]	= { .type = NLA_U8 },
556
557
558
	[TCA_FLOWER_KEY_CVLAN_ID]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_CVLAN_PRIO]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_CVLAN_ETH_TYPE]	= { .type = NLA_U16 },
559
560
561
562
	[TCA_FLOWER_KEY_ENC_IP_TOS]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ENC_IP_TTL]	 = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 },
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
	[TCA_FLOWER_KEY_ENC_OPTS]	= { .type = NLA_NESTED },
	[TCA_FLOWER_KEY_ENC_OPTS_MASK]	= { .type = NLA_NESTED },
};

static const struct nla_policy
enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = {
	[TCA_FLOWER_KEY_ENC_OPTS_GENEVE]        = { .type = NLA_NESTED },
};

static const struct nla_policy
geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = {
	[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]      = { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]       = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]       = { .type = NLA_BINARY,
						       .len = 128 },
Jiri Pirko's avatar
Jiri Pirko committed
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
};

static void fl_set_key_val(struct nlattr **tb,
			   void *val, int val_type,
			   void *mask, int mask_type, int len)
{
	if (!tb[val_type])
		return;
	memcpy(val, nla_data(tb[val_type]), len);
	if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
		memset(mask, 0xff, len);
	else
		memcpy(mask, nla_data(tb[mask_type]), len);
}

593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key,
				 struct fl_flow_key *mask)
{
	fl_set_key_val(tb, &key->tp_min.dst,
		       TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_min.dst,
		       TCA_FLOWER_UNSPEC, sizeof(key->tp_min.dst));
	fl_set_key_val(tb, &key->tp_max.dst,
		       TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_max.dst,
		       TCA_FLOWER_UNSPEC, sizeof(key->tp_max.dst));
	fl_set_key_val(tb, &key->tp_min.src,
		       TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_min.src,
		       TCA_FLOWER_UNSPEC, sizeof(key->tp_min.src));
	fl_set_key_val(tb, &key->tp_max.src,
		       TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_max.src,
		       TCA_FLOWER_UNSPEC, sizeof(key->tp_max.src));

	if ((mask->tp_min.dst && mask->tp_max.dst &&
	     htons(key->tp_max.dst) <= htons(key->tp_min.dst)) ||
	     (mask->tp_min.src && mask->tp_max.src &&
	      htons(key->tp_max.src) <= htons(key->tp_min.src)))
		return -EINVAL;

	return 0;
}

618
619
620
static int fl_set_key_mpls(struct nlattr **tb,
			   struct flow_dissector_key_mpls *key_val,
			   struct flow_dissector_key_mpls *key_mask)
621
622
623
624
625
626
{
	if (tb[TCA_FLOWER_KEY_MPLS_TTL]) {
		key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]);
		key_mask->mpls_ttl = MPLS_TTL_MASK;
	}
	if (tb[TCA_FLOWER_KEY_MPLS_BOS]) {
627
628
629
630
631
		u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]);

		if (bos & ~MPLS_BOS_MASK)
			return -EINVAL;
		key_val->mpls_bos = bos;
632
633
634
		key_mask->mpls_bos = MPLS_BOS_MASK;
	}
	if (tb[TCA_FLOWER_KEY_MPLS_TC]) {
635
636
637
638
639
		u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]);

		if (tc & ~MPLS_TC_MASK)
			return -EINVAL;
		key_val->mpls_tc = tc;
640
641
642
		key_mask->mpls_tc = MPLS_TC_MASK;
	}
	if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
643
644
645
646
647
		u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]);

		if (label & ~MPLS_LABEL_MASK)
			return -EINVAL;
		key_val->mpls_label = label;
648
649
		key_mask->mpls_label = MPLS_LABEL_MASK;
	}
650
	return 0;
651
652
}

653
static void fl_set_key_vlan(struct nlattr **tb,
654
			    __be16 ethertype,
655
			    int vlan_id_key, int vlan_prio_key,
656
657
658
659
660
			    struct flow_dissector_key_vlan *key_val,
			    struct flow_dissector_key_vlan *key_mask)
{
#define VLAN_PRIORITY_MASK	0x7

661
	if (tb[vlan_id_key]) {
662
		key_val->vlan_id =
663
			nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK;
664
665
		key_mask->vlan_id = VLAN_VID_MASK;
	}
666
	if (tb[vlan_prio_key]) {
667
		key_val->vlan_priority =
668
			nla_get_u8(tb[vlan_prio_key]) &
669
670
671
			VLAN_PRIORITY_MASK;
		key_mask->vlan_priority = VLAN_PRIORITY_MASK;
	}
672
673
	key_val->vlan_tpid = ethertype;
	key_mask->vlan_tpid = cpu_to_be16(~0);
674
675
}

676
677
678
679
680
681
682
683
684
685
686
static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
			    u32 *dissector_key, u32 *dissector_mask,
			    u32 flower_flag_bit, u32 dissector_flag_bit)
{
	if (flower_mask & flower_flag_bit) {
		*dissector_mask |= dissector_flag_bit;
		if (flower_key & flower_flag_bit)
			*dissector_key |= dissector_flag_bit;
	}
}

687
688
static int fl_set_key_flags(struct nlattr **tb,
			    u32 *flags_key, u32 *flags_mask)
689
690
691
{
	u32 key, mask;

692
693
694
	/* mask is mandatory for flags */
	if (!tb[TCA_FLOWER_KEY_FLAGS_MASK])
		return -EINVAL;
695
696

	key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
697
	mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
698
699
700
701
702
703

	*flags_key  = 0;
	*flags_mask = 0;

	fl_set_key_flag(key, mask, flags_key, flags_mask,
			TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
704
705
706
	fl_set_key_flag(key, mask, flags_key, flags_mask,
			TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
			FLOW_DIS_FIRST_FRAG);
707
708

	return 0;
709
710
}

711
static void fl_set_key_ip(struct nlattr **tb, bool encap,
712
713
714
			  struct flow_dissector_key_ip *key,
			  struct flow_dissector_key_ip *mask)
{
715
716
717
718
	int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
	int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
	int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
	int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
719

720
721
	fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos));
	fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl));
722
723
}

724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key,
			     int depth, int option_len,
			     struct netlink_ext_ack *extack)
{
	struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1];
	struct nlattr *class = NULL, *type = NULL, *data = NULL;
	struct geneve_opt *opt;
	int err, data_len = 0;

	if (option_len > sizeof(struct geneve_opt))
		data_len = option_len - sizeof(struct geneve_opt);

	opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len];
	memset(opt, 0xff, option_len);
	opt->length = data_len / 4;
	opt->r1 = 0;
	opt->r2 = 0;
	opt->r3 = 0;

	/* If no mask has been prodived we assume an exact match. */
	if (!depth)
		return sizeof(struct geneve_opt) + data_len;

	if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) {
		NL_SET_ERR_MSG(extack, "Non-geneve option type for mask");
		return -EINVAL;
	}

	err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX,
			       nla, geneve_opt_policy, extack);
	if (err < 0)
		return err;

	/* We are not allowed to omit any of CLASS, TYPE or DATA
	 * fields from the key.
	 */
	if (!option_len &&
	    (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] ||
	     !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] ||
	     !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) {
		NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data");
		return -EINVAL;
	}

	/* Omitting any of CLASS, TYPE or DATA fields is allowed
	 * for the mask.
	 */
	if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) {
		int new_len = key->enc_opts.len;

		data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA];
		data_len = nla_len(data);
		if (data_len < 4) {
			NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long");
			return -ERANGE;
		}
		if (data_len % 4) {
			NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long");
			return -ERANGE;
		}

		new_len += sizeof(struct geneve_opt) + data_len;
		BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX);
		if (new_len > FLOW_DIS_TUN_OPTS_MAX) {
			NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size");
			return -ERANGE;
		}
		opt->length = data_len / 4;
		memcpy(opt->opt_data, nla_data(data), data_len);
	}

	if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) {
		class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS];
		opt->opt_class = nla_get_be16(class);
	}

	if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) {
		type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE];
		opt->type = nla_get_u8(type);
	}

	return sizeof(struct geneve_opt) + data_len;
}

static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
			  struct fl_flow_key *mask,
			  struct netlink_ext_ack *extack)
{
	const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL;
813
814
815
816
817
818
819
	int err, option_len, key_depth, msk_depth = 0;

	err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS],
				  TCA_FLOWER_KEY_ENC_OPTS_MAX,
				  enc_opts_policy, extack);
	if (err)
		return err;
820
821
822
823

	nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]);

	if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) {
824
825
826
827
828
829
		err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK],
					  TCA_FLOWER_KEY_ENC_OPTS_MAX,
					  enc_opts_policy, extack);
		if (err)
			return err;

830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
		nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
		msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
	}

	nla_for_each_attr(nla_opt_key, nla_enc_key,
			  nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) {
		switch (nla_type(nla_opt_key)) {
		case TCA_FLOWER_KEY_ENC_OPTS_GENEVE:
			option_len = 0;
			key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
			option_len = fl_set_geneve_opt(nla_opt_key, key,
						       key_depth, option_len,
						       extack);
			if (option_len < 0)
				return option_len;

			key->enc_opts.len += option_len;
			/* At the same time we need to parse through the mask
			 * in order to verify exact and mask attribute lengths.
			 */
			mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
			option_len = fl_set_geneve_opt(nla_opt_msk, mask,
						       msk_depth, option_len,
						       extack);
			if (option_len < 0)
				return option_len;

			mask->enc_opts.len += option_len;
			if (key->enc_opts.len != mask->enc_opts.len) {
				NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
				return -EINVAL;
			}

			if (msk_depth)
				nla_opt_msk = nla_next(nla_opt_msk, &msk_depth);
			break;
		default:
			NL_SET_ERR_MSG(extack, "Unknown tunnel option type");
			return -EINVAL;
		}
	}

	return 0;
}

Jiri Pirko's avatar
Jiri Pirko committed
875
static int fl_set_key(struct net *net, struct nlattr **tb,
876
877
		      struct fl_flow_key *key, struct fl_flow_key *mask,
		      struct netlink_ext_ack *extack)
Jiri Pirko's avatar
Jiri Pirko committed
878
{
879
	__be16 ethertype;
880
	int ret = 0;
Brian Haley's avatar
Brian Haley committed
881
#ifdef CONFIG_NET_CLS_IND
Jiri Pirko's avatar
Jiri Pirko committed
882
	if (tb[TCA_FLOWER_INDEV]) {
883
		int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack);
Jiri Pirko's avatar
Jiri Pirko committed
884
885
886
887
888
		if (err < 0)
			return err;
		key->indev_ifindex = err;
		mask->indev_ifindex = 0xffffffff;
	}
Brian Haley's avatar
Brian Haley committed
889
#endif
Jiri Pirko's avatar
Jiri Pirko committed
890
891
892
893
894
895
896

	fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
		       mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
		       sizeof(key->eth.dst));
	fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
		       mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
		       sizeof(key->eth.src));
897

898
	if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
899
900
		ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);

901
		if (eth_type_vlan(ethertype)) {
902
903
904
905
			fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID,
					TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan,
					&mask->vlan);

906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
			if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) {
				ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]);
				if (eth_type_vlan(ethertype)) {
					fl_set_key_vlan(tb, ethertype,
							TCA_FLOWER_KEY_CVLAN_ID,
							TCA_FLOWER_KEY_CVLAN_PRIO,
							&key->cvlan, &mask->cvlan);
					fl_set_key_val(tb, &key->basic.n_proto,
						       TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
						       &mask->basic.n_proto,
						       TCA_FLOWER_UNSPEC,
						       sizeof(key->basic.n_proto));
				} else {
					key->basic.n_proto = ethertype;
					mask->basic.n_proto = cpu_to_be16(~0);
				}
922
			}
923
924
925
926
		} else {
			key->basic.n_proto = ethertype;
			mask->basic.n_proto = cpu_to_be16(~0);
		}
927
	}
928

Jiri Pirko's avatar
Jiri Pirko committed
929
930
931
932
933
	if (key->basic.n_proto == htons(ETH_P_IP) ||
	    key->basic.n_proto == htons(ETH_P_IPV6)) {
		fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
			       &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
			       sizeof(key->basic.ip_proto));
934
		fl_set_key_ip(tb, false, &key->ip, &mask->ip);
Jiri Pirko's avatar
Jiri Pirko committed
935
	}
936
937
938

	if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
939
		mask->control.addr_type = ~0;
Jiri Pirko's avatar
Jiri Pirko committed
940
941
942
943
944
945
		fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
			       &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
			       sizeof(key->ipv4.src));
		fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
			       &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
			       sizeof(key->ipv4.dst));
946
947
	} else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
948
		mask->control.addr_type = ~0;
Jiri Pirko's avatar
Jiri Pirko committed
949
950
951
952
953
954
955
		fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
			       &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
			       sizeof(key->ipv6.src));
		fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
			       &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
			       sizeof(key->ipv6.dst));
	}
956

Jiri Pirko's avatar
Jiri Pirko committed
957
958
	if (key->basic.ip_proto == IPPROTO_TCP) {
		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
959
			       &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
Jiri Pirko's avatar
Jiri Pirko committed
960
961
			       sizeof(key->tp.src));
		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
962
			       &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
Jiri Pirko's avatar
Jiri Pirko committed
963
			       sizeof(key->tp.dst));
964
965
966
		fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
			       &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
			       sizeof(key->tcp.flags));
Jiri Pirko's avatar
Jiri Pirko committed
967
968
	} else if (key->basic.ip_proto == IPPROTO_UDP) {
		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
969
			       &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
Jiri Pirko's avatar
Jiri Pirko committed
970
971
			       sizeof(key->tp.src));
		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
972
			       &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
Jiri Pirko's avatar
Jiri Pirko committed
973
			       sizeof(key->tp.dst));
974
975
976
977
978
979
980
	} else if (key->basic.ip_proto == IPPROTO_SCTP) {
		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
			       &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
			       sizeof(key->tp.src));
		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
			       &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
			       sizeof(key->tp.dst));
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
	} else if (key->basic.n_proto == htons(ETH_P_IP) &&
		   key->basic.ip_proto == IPPROTO_ICMP) {
		fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE,
			       &mask->icmp.type,
			       TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
			       sizeof(key->icmp.type));
		fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
			       &mask->icmp.code,
			       TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
			       sizeof(key->icmp.code));
	} else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
		   key->basic.ip_proto == IPPROTO_ICMPV6) {
		fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE,
			       &mask->icmp.type,
			       TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
			       sizeof(key->icmp.type));
997
		fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE,
998
			       &mask->icmp.code,
999
			       TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
1000
			       sizeof(key->icmp.code));
1001
1002
	} else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) ||
		   key->basic.n_proto == htons(ETH_P_MPLS_MC)) {
1003
1004
1005
		ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls);
		if (ret)
			return ret;
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
	} else if (key->basic.n_proto == htons(ETH_P_ARP) ||
		   key->basic.n_proto == htons(ETH_P_RARP)) {
		fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP,
			       &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK,
			       sizeof(key->arp.sip));
		fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP,
			       &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK,
			       sizeof(key->arp.tip));
		fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP,
			       &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK,
			       sizeof(key->arp.op));
		fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
			       mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
			       sizeof(key->arp.sha));
		fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
			       mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
			       sizeof(key->arp.tha));
Jiri Pirko's avatar
Jiri Pirko committed
1023
1024
	}

1025
1026
1027
1028
1029
1030
1031
1032
	if (key->basic.ip_proto == IPPROTO_TCP ||
	    key->basic.ip_proto == IPPROTO_UDP ||
	    key->basic.ip_proto == IPPROTO_SCTP) {
		ret = fl_set_key_port_range(tb, key, mask);
		if (ret)
			return ret;
	}

1033
1034
1035
	if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
	    tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1036
		mask->enc_control.addr_type = ~0;
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
		fl_set_key_val(tb, &key->enc_ipv4.src,
			       TCA_FLOWER_KEY_ENC_IPV4_SRC,
			       &mask->enc_ipv4.src,
			       TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
			       sizeof(key->enc_ipv4.src));
		fl_set_key_val(tb, &key->enc_ipv4.dst,
			       TCA_FLOWER_KEY_ENC_IPV4_DST,
			       &mask->enc_ipv4.dst,
			       TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
			       sizeof(key->enc_ipv4.dst));
	}

	if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
	    tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1052
		mask->enc_control.addr_type = ~0;
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
		fl_set_key_val(tb, &key->enc_ipv6.src,
			       TCA_FLOWER_KEY_ENC_IPV6_SRC,
			       &mask->enc_ipv6.src,
			       TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
			       sizeof(key->enc_ipv6.src));
		fl_set_key_val(tb, &key->enc_ipv6.dst,
			       TCA_FLOWER_KEY_ENC_IPV6_DST,
			       &mask->enc_ipv6.dst,
			       TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
			       sizeof(key->enc_ipv6.dst));
	}

	fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
1066
		       &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
1067
1068
		       sizeof(key->enc_key_id.keyid));

1069
1070
1071
1072
1073
1074
1075
1076
	fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
		       &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
		       sizeof(key->enc_tp.src));

	fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
		       &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
		       sizeof(key->enc_tp.dst));

1077
1078
	fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip);

1079
1080
1081
1082
1083
1084
	if (tb[TCA_FLOWER_KEY_ENC_OPTS]) {
		ret = fl_set_enc_opt(tb, key, mask, extack);
		if (ret)
			return ret;
	}

1085
1086
	if (tb[TCA_FLOWER_KEY_FLAGS])
		ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags);
1087

1088
	return ret;
Jiri Pirko's avatar
Jiri Pirko committed
1089
1090
}

1091
1092
static void fl_mask_copy(struct fl_flow_mask *dst,
			 struct fl_flow_mask *src)
Jiri Pirko's avatar
Jiri Pirko committed
1093
{
1094
1095
	const void *psrc = fl_key_get_start(&src->key, src);
	void *pdst = fl_key_get_start(&dst->key, src);
Jiri Pirko's avatar
Jiri Pirko committed
1096

1097
1098
	memcpy(pdst, psrc, fl_mask_range(src));
	dst->range = src->range;
Jiri Pirko's avatar
Jiri Pirko committed
1099
1100
1101
1102
1103
1104
1105
1106
}

static const struct rhashtable_params fl_ht_params = {
	.key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
	.head_offset = offsetof(struct cls_fl_filter, ht_node),
	.automatic_shrinking = true,
};

1107
static int fl_init_mask_hashtable(struct fl_flow_mask *mask)
Jiri Pirko's avatar
Jiri Pirko committed
1108
{
1109
1110
1111
	mask->filter_ht_params = fl_ht_params;
	mask->filter_ht_params.key_len = fl_mask_range(mask);
	mask->filter_ht_params.key_offset += mask->range.start;
Jiri Pirko's avatar
Jiri Pirko committed
1112

1113
	return rhashtable_init(&mask->ht, &mask->filter_ht_params);
Jiri Pirko's avatar
Jiri Pirko committed
1114
1115
1116
}

#define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
1117
#define FL_KEY_MEMBER_SIZE(member) FIELD_SIZEOF(struct fl_flow_key, member)
Jiri Pirko's avatar
Jiri Pirko committed
1118

1119
1120
1121
#define FL_KEY_IS_MASKED(mask, member)						\
	memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member),		\
		   0, FL_KEY_MEMBER_SIZE(member))				\
Jiri Pirko's avatar
Jiri Pirko committed
1122
1123
1124
1125
1126
1127
1128
1129

#define FL_KEY_SET(keys, cnt, id, member)					\
	do {									\
		keys[cnt].key_id = id;						\
		keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member);		\
		cnt++;								\
	} while(0);

1130
#define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member)			\
Jiri Pirko's avatar
Jiri Pirko committed
1131
	do {									\
1132
		if (FL_KEY_IS_MASKED(mask, member))				\
Jiri Pirko's avatar
Jiri Pirko committed
1133
1134
1135
			FL_KEY_SET(keys, cnt, id, member);			\
	} while(0);

1136
1137
static void fl_init_dissector(struct flow_dissector *dissector,
			      struct fl_flow_key *mask)
Jiri Pirko's avatar
Jiri Pirko committed
1138
1139
1140
1141
{
	struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
	size_t cnt = 0;

1142
	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
Jiri Pirko's avatar
Jiri Pirko committed
1143
	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
1144
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1145
			     FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
1146
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1147
			     FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
1148
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1149
			     FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
1150
1151
1152
	if (FL_KEY_IS_MASKED(mask, tp) ||
	    FL_KEY_IS_MASKED(mask, tp_min) || FL_KEY_IS_MASKED(mask, tp_max))
		FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_PORTS, tp);
1153
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1154
			     FLOW_DISSECTOR_KEY_IP, ip);
1155
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1156
			     FLOW_DISSECTOR_KEY_TCP, tcp);
1157
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1158
			     FLOW_DISSECTOR_KEY_ICMP, icmp);
1159
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1160
			     FLOW_DISSECTOR_KEY_ARP, arp);
1161
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1162
			     FLOW_DISSECTOR_KEY_MPLS, mpls);
1163
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1164
			     FLOW_DISSECTOR_KEY_VLAN, vlan);
1165
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1166
			     FLOW_DISSECTOR_KEY_CVLAN, cvlan);
1167
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1168
			     FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
1169
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1170
			     FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
1171
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1172
			     FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
1173
1174
	if (FL_KEY_IS_MASKED(mask, enc_ipv4) ||
	    FL_KEY_IS_MASKED(mask, enc_ipv6))
1175
1176
		FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
			   enc_control);
1177
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1178
			     FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
1179
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1180
			     FLOW_DISSECTOR_KEY_ENC_IP, enc_ip);
1181
1182
	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
			     FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts);
Jiri Pirko's avatar
Jiri Pirko committed
1183

1184
	skb_flow_dissector_init(dissector, keys, cnt);
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
}

static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head,
					       struct fl_flow_mask *mask)
{
	struct fl_flow_mask *newmask;
	int err;

	newmask = kzalloc(sizeof(*newmask), GFP_KERNEL);
	if (!newmask)
		return ERR_PTR(-ENOMEM);

	fl_mask_copy(newmask, mask);

1199
1200
1201
1202
	if ((newmask->key.tp_min.dst && newmask->key.tp_max.dst) ||