fs_core.c 80.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
/*
 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/mutex.h>
#include <linux/mlx5/driver.h>
35
#include <linux/mlx5/vport.h>
36
#include <linux/mlx5/eswitch.h>
37
38
39

#include "mlx5_core.h"
#include "fs_core.h"
40
#include "fs_cmd.h"
Matan Barak's avatar
Matan Barak committed
41
#include "diag/fs_tracepoint.h"
42
#include "accel/ipsec.h"
43
#include "fpga/ipsec.h"
44
#include "eswitch.h"
45

46
47
48
#define INIT_TREE_NODE_ARRAY_SIZE(...)	(sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
					 sizeof(struct init_tree_node))

49
#define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
50
		 ...) {.type = FS_TYPE_PRIO,\
51
	.min_ft_level = min_level_val,\
52
	.num_levels = num_levels_val,\
53
	.num_leaf_prios = num_prios_val,\
54
	.caps = caps_val,\
55
56
57
58
	.children = (struct init_tree_node[]) {__VA_ARGS__},\
	.ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
}

59
60
#define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
	ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
61
		 __VA_ARGS__)\
62

63
64
#define ADD_NS(def_miss_act, ...) {.type = FS_TYPE_NAMESPACE,	\
	.def_miss_action = def_miss_act,\
65
66
67
68
	.children = (struct init_tree_node[]) {__VA_ARGS__},\
	.ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
}

69
70
71
72
73
74
75
76
#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
				   sizeof(long))

#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))

#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
			       .caps = (long[]) {__VA_ARGS__} }

77
78
79
80
81
#define FS_CHAINING_CAPS  FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
					   FS_CAP(flow_table_properties_nic_receive.modify_root), \
					   FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
					   FS_CAP(flow_table_properties_nic_receive.flow_table_modify))

82
83
84
85
86
87
88
89
#define FS_CHAINING_CAPS_EGRESS                                                \
	FS_REQUIRED_CAPS(                                                      \
		FS_CAP(flow_table_properties_nic_transmit.flow_modify_en),     \
		FS_CAP(flow_table_properties_nic_transmit.modify_root),        \
		FS_CAP(flow_table_properties_nic_transmit                      \
			       .identified_miss_table_mode),                   \
		FS_CAP(flow_table_properties_nic_transmit.flow_table_modify))

90
#define LEFTOVERS_NUM_LEVELS 1
91
92
#define LEFTOVERS_NUM_PRIOS 1

93
#define BY_PASS_PRIO_NUM_LEVELS 1
94
#define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
95
96
			   LEFTOVERS_NUM_PRIOS)

97
#define ETHTOOL_PRIO_NUM_LEVELS 1
98
#define ETHTOOL_NUM_PRIOS 11
99
#define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
100
101
/* Vlan, mac, ttc, inner ttc, aRFS */
#define KERNEL_NIC_PRIO_NUM_LEVELS 5
102
103
104
#define KERNEL_NIC_NUM_PRIOS 1
/* One more level for tc */
#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
105

106
107
108
#define KERNEL_NIC_TC_NUM_PRIOS  1
#define KERNEL_NIC_TC_NUM_LEVELS 2

109
#define ANCHOR_NUM_LEVELS 1
110
111
#define ANCHOR_NUM_PRIOS 1
#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
112
113
114
115
116

#define OFFLOADS_MAX_FT 1
#define OFFLOADS_NUM_PRIOS 1
#define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + 1)

117
118
119
120
#define LAG_PRIO_NUM_LEVELS 1
#define LAG_NUM_PRIOS 1
#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)

121
122
123
124
struct node_caps {
	size_t	arr_sz;
	long	*caps;
};
125

126
127
128
129
static struct init_tree_node {
	enum fs_node_type	type;
	struct init_tree_node *children;
	int ar_size;
130
	struct node_caps caps;
131
	int min_ft_level;
132
	int num_leaf_prios;
133
	int prio;
134
	int num_levels;
135
	enum mlx5_flow_table_miss_action def_miss_action;
136
137
} root_fs = {
	.type = FS_TYPE_NAMESPACE,
138
	.ar_size = 7,
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
	  .children = (struct init_tree_node[]){
		  ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
						    BY_PASS_PRIO_NUM_LEVELS))),
		  ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS,
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
						    LAG_PRIO_NUM_LEVELS))),
		  ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, {},
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS,
						    OFFLOADS_MAX_FT))),
		  ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0, FS_CHAINING_CAPS,
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
						    ETHTOOL_PRIO_NUM_LEVELS))),
		  ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS,
						    KERNEL_NIC_TC_NUM_LEVELS),
				  ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
						    KERNEL_NIC_PRIO_NUM_LEVELS))),
		  ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS,
						    LEFTOVERS_NUM_LEVELS))),
		  ADD_PRIO(0, ANCHOR_MIN_LEVEL, 0, {},
			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				  ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS,
						    ANCHOR_NUM_LEVELS))),
170
171
172
	}
};

173
174
175
176
177
178
static struct init_tree_node egress_root_fs = {
	.type = FS_TYPE_NAMESPACE,
	.ar_size = 1,
	.children = (struct init_tree_node[]) {
		ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0,
			 FS_CHAINING_CAPS_EGRESS,
179
180
			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
181
182
183
184
						  BY_PASS_PRIO_NUM_LEVELS))),
	}
};

185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
#define RDMA_RX_BYPASS_PRIO 0
#define RDMA_RX_KERNEL_PRIO 1
static struct init_tree_node rdma_rx_root_fs = {
	.type = FS_TYPE_NAMESPACE,
	.ar_size = 2,
	.children = (struct init_tree_node[]) {
		[RDMA_RX_BYPASS_PRIO] =
		ADD_PRIO(0, MLX5_BY_PASS_NUM_REGULAR_PRIOS, 0,
			 FS_CHAINING_CAPS,
			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
				ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS,
						  BY_PASS_PRIO_NUM_LEVELS))),
		[RDMA_RX_KERNEL_PRIO] =
		ADD_PRIO(0, MLX5_BY_PASS_NUM_REGULAR_PRIOS + 1, 0,
			 FS_CHAINING_CAPS,
			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
				ADD_MULTIPLE_PRIO(1, 1))),
	}
};

205
206
207
208
enum fs_i_lock_class {
	FS_LOCK_GRANDPARENT,
	FS_LOCK_PARENT,
	FS_LOCK_CHILD
209
210
};

211
static const struct rhashtable_params rhash_fte = {
212
	.key_len = sizeof_field(struct fs_fte, val),
213
214
215
216
217
218
	.key_offset = offsetof(struct fs_fte, val),
	.head_offset = offsetof(struct fs_fte, hash),
	.automatic_shrinking = true,
	.min_size = 1,
};

219
static const struct rhashtable_params rhash_fg = {
220
	.key_len = sizeof_field(struct mlx5_flow_group, mask),
221
222
223
224
225
226
227
	.key_offset = offsetof(struct mlx5_flow_group, mask),
	.head_offset = offsetof(struct mlx5_flow_group, hash),
	.automatic_shrinking = true,
	.min_size = 1,

};

228
229
230
231
232
233
static void del_hw_flow_table(struct fs_node *node);
static void del_hw_flow_group(struct fs_node *node);
static void del_hw_fte(struct fs_node *node);
static void del_sw_flow_table(struct fs_node *node);
static void del_sw_flow_group(struct fs_node *node);
static void del_sw_fte(struct fs_node *node);
234
235
static void del_sw_prio(struct fs_node *node);
static void del_sw_ns(struct fs_node *node);
236
237
238
239
/* Delete rule (destination) is special case that 
 * requires to lock the FTE for all the deletion process.
 */
static void del_sw_hw_rule(struct fs_node *node);
Mark Bloch's avatar
Mark Bloch committed
240
241
static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
				struct mlx5_flow_destination *d2);
242
static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns);
Mark Bloch's avatar
Mark Bloch committed
243
244
245
static struct mlx5_flow_rule *
find_flow_rule(struct fs_fte *fte,
	       struct mlx5_flow_destination *dest);
246
247

static void tree_init_node(struct fs_node *node,
248
249
			   void (*del_hw_func)(struct fs_node *),
			   void (*del_sw_func)(struct fs_node *))
250
{
251
	refcount_set(&node->refcount, 1);
252
253
	INIT_LIST_HEAD(&node->list);
	INIT_LIST_HEAD(&node->children);
254
	init_rwsem(&node->lock);
255
256
	node->del_hw_func = del_hw_func;
	node->del_sw_func = del_sw_func;
257
	node->active = false;
258
259
260
261
262
}

static void tree_add_node(struct fs_node *node, struct fs_node *parent)
{
	if (parent)
263
		refcount_inc(&parent->refcount);
264
265
266
267
268
269
270
271
272
	node->parent = parent;

	/* Parent is the root */
	if (!parent)
		node->root = node;
	else
		node->root = parent->root;
}

273
static int tree_get_node(struct fs_node *node)
274
{
275
	return refcount_inc_not_zero(&node->refcount);
276
277
}

278
279
static void nested_down_read_ref_node(struct fs_node *node,
				      enum fs_i_lock_class class)
280
281
{
	if (node) {
282
		down_read_nested(&node->lock, class);
283
		refcount_inc(&node->refcount);
284
285
286
	}
}

287
288
static void nested_down_write_ref_node(struct fs_node *node,
				       enum fs_i_lock_class class)
289
290
{
	if (node) {
291
		down_write_nested(&node->lock, class);
292
		refcount_inc(&node->refcount);
293
294
295
	}
}

296
static void down_write_ref_node(struct fs_node *node, bool locked)
297
298
{
	if (node) {
299
300
		if (!locked)
			down_write(&node->lock);
301
		refcount_inc(&node->refcount);
302
303
304
	}
}

305
306
static void up_read_ref_node(struct fs_node *node)
{
307
	refcount_dec(&node->refcount);
308
309
310
	up_read(&node->lock);
}

311
static void up_write_ref_node(struct fs_node *node, bool locked)
312
{
313
	refcount_dec(&node->refcount);
314
315
	if (!locked)
		up_write(&node->lock);
316
317
}

318
static void tree_put_node(struct fs_node *node, bool locked)
319
320
321
{
	struct fs_node *parent_node = node->parent;

322
	if (refcount_dec_and_test(&node->refcount)) {
323
324
325
326
327
328
		if (node->del_hw_func)
			node->del_hw_func(node);
		if (parent_node) {
			/* Only root namespace doesn't have parent and we just
			 * need to free its node.
			 */
329
			down_write_ref_node(parent_node, locked);
330
			list_del_init(&node->list);
331
332
			if (node->del_sw_func)
				node->del_sw_func(node);
333
			up_write_ref_node(parent_node, locked);
334
335
		} else {
			kfree(node);
336
		}
337
338
339
		node = NULL;
	}
	if (!node && parent_node)
340
		tree_put_node(parent_node, locked);
341
342
}

343
static int tree_remove_node(struct fs_node *node, bool locked)
344
{
345
346
	if (refcount_read(&node->refcount) > 1) {
		refcount_dec(&node->refcount);
347
348
		return -EEXIST;
	}
349
	tree_put_node(node, locked);
350
351
	return 0;
}
352
353
354
355
356
357
358
359
360
361
362
363
364
365

static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
				 unsigned int prio)
{
	struct fs_prio *iter_prio;

	fs_for_each_prio(iter_prio, ns) {
		if (iter_prio->prio == prio)
			return iter_prio;
	}

	return NULL;
}

366
static bool check_valid_spec(const struct mlx5_flow_spec *spec)
367
{
368
369
370
371
372
373
374
375
	int i;

	for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
		if (spec->match_value[i] & ~spec->match_criteria[i]) {
			pr_warn("mlx5_core: match_value differs from match_criteria\n");
			return false;
		}

376
	return true;
377
}
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394

static struct mlx5_flow_root_namespace *find_root(struct fs_node *node)
{
	struct fs_node *root;
	struct mlx5_flow_namespace *ns;

	root = node->root;

	if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) {
		pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
		return NULL;
	}

	ns = container_of(root, struct mlx5_flow_namespace, node);
	return container_of(ns, struct mlx5_flow_root_namespace, ns);
}

395
396
397
398
399
400
401
402
403
static inline struct mlx5_flow_steering *get_steering(struct fs_node *node)
{
	struct mlx5_flow_root_namespace *root = find_root(node);

	if (root)
		return root->dev->priv.steering;
	return NULL;
}

404
405
406
407
408
409
410
411
412
static inline struct mlx5_core_dev *get_dev(struct fs_node *node)
{
	struct mlx5_flow_root_namespace *root = find_root(node);

	if (root)
		return root->dev;
	return NULL;
}

413
414
415
416
417
418
419
420
421
422
static void del_sw_ns(struct fs_node *node)
{
	kfree(node);
}

static void del_sw_prio(struct fs_node *node)
{
	kfree(node);
}

423
static void del_hw_flow_table(struct fs_node *node)
424
{
425
	struct mlx5_flow_root_namespace *root;
426
427
428
429
430
431
	struct mlx5_flow_table *ft;
	struct mlx5_core_dev *dev;
	int err;

	fs_get_obj(ft, node);
	dev = get_dev(&ft->node);
432
	root = find_root(&ft->node);
433
	trace_mlx5_fs_del_ft(ft);
434

435
	if (node->active) {
436
		err = root->cmds->destroy_flow_table(root, ft);
437
438
439
		if (err)
			mlx5_core_warn(dev, "flow steering can't destroy ft\n");
	}
440
441
442
443
444
445
446
447
448
}

static void del_sw_flow_table(struct fs_node *node)
{
	struct mlx5_flow_table *ft;
	struct fs_prio *prio;

	fs_get_obj(ft, node);

449
	rhltable_destroy(&ft->fgs_hash);
450
451
	fs_get_obj(prio, ft->node.parent);
	prio->num_ft--;
452
	kfree(ft);
453
454
}

455
static void modify_fte(struct fs_fte *fte)
456
{
457
	struct mlx5_flow_root_namespace *root;
458
459
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *fg;
460
	struct mlx5_core_dev *dev;
461
462
463
464
	int err;

	fs_get_obj(fg, fte->node.parent);
	fs_get_obj(ft, fg->node.parent);
465
466
467
	dev = get_dev(&fte->node);

	root = find_root(&ft->node);
468
	err = root->cmds->update_fte(root, ft, fg, fte->modify_mask, fte);
469
470
471
472
473
474
475
476
477
478
479
480
481
482
	if (err)
		mlx5_core_warn(dev,
			       "%s can't del rule fg id=%d fte_index=%d\n",
			       __func__, fg->id, fte->index);
	fte->modify_mask = 0;
}

static void del_sw_hw_rule(struct fs_node *node)
{
	struct mlx5_flow_rule *rule;
	struct fs_fte *fte;

	fs_get_obj(rule, node);
	fs_get_obj(fte, rule->node.parent);
Matan Barak's avatar
Matan Barak committed
483
	trace_mlx5_fs_del_rule(rule);
484
485
486
487
488
	if (rule->sw_action == MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO) {
		mutex_lock(&rule->dest_attr.ft->lock);
		list_del(&rule->next_ft);
		mutex_unlock(&rule->dest_attr.ft->lock);
	}
489
490
491

	if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER  &&
	    --fte->dests_size) {
492
493
494
		fte->modify_mask |=
			BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) |
			BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
495
		fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT;
496
497
498
		goto out;
	}

499
	if ((fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
500
	    --fte->dests_size) {
501
502
		fte->modify_mask |=
			BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
503
504
	}
out:
505
	kfree(rule);
506
507
}

508
static void del_hw_fte(struct fs_node *node)
509
{
510
	struct mlx5_flow_root_namespace *root;
511
512
513
514
515
516
517
518
519
520
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *fg;
	struct mlx5_core_dev *dev;
	struct fs_fte *fte;
	int err;

	fs_get_obj(fte, node);
	fs_get_obj(fg, fte->node.parent);
	fs_get_obj(ft, fg->node.parent);

521
	trace_mlx5_fs_del_fte(fte);
522
	dev = get_dev(&ft->node);
523
	root = find_root(&ft->node);
524
	if (node->active) {
525
		err = root->cmds->delete_fte(root, ft, fte);
526
527
528
529
		if (err)
			mlx5_core_warn(dev,
				       "flow steering can't delete fte in index %d of flow group id %d\n",
				       fte->index, fg->id);
530
		node->active = 0;
531
	}
532
533
534
535
}

static void del_sw_fte(struct fs_node *node)
{
536
	struct mlx5_flow_steering *steering = get_steering(node);
537
538
539
540
541
542
	struct mlx5_flow_group *fg;
	struct fs_fte *fte;
	int err;

	fs_get_obj(fte, node);
	fs_get_obj(fg, fte->node.parent);
543

544
545
546
547
548
	err = rhashtable_remove_fast(&fg->ftes_hash,
				     &fte->hash,
				     rhash_fte);
	WARN_ON(err);
	ida_simple_remove(&fg->fte_allocator, fte->index - fg->start_index);
549
	kmem_cache_free(steering->ftes_cache, fte);
550
551
}

552
static void del_hw_flow_group(struct fs_node *node)
553
{
554
	struct mlx5_flow_root_namespace *root;
555
556
557
558
559
560
561
	struct mlx5_flow_group *fg;
	struct mlx5_flow_table *ft;
	struct mlx5_core_dev *dev;

	fs_get_obj(fg, node);
	fs_get_obj(ft, fg->node.parent);
	dev = get_dev(&ft->node);
Matan Barak's avatar
Matan Barak committed
562
	trace_mlx5_fs_del_fg(fg);
563

564
	root = find_root(&ft->node);
565
	if (fg->node.active && root->cmds->destroy_flow_group(root, ft, fg))
566
567
568
569
570
571
		mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
			       fg->id, ft->id);
}

static void del_sw_flow_group(struct fs_node *node)
{
572
	struct mlx5_flow_steering *steering = get_steering(node);
573
574
575
576
577
578
	struct mlx5_flow_group *fg;
	struct mlx5_flow_table *ft;
	int err;

	fs_get_obj(fg, node);
	fs_get_obj(ft, fg->node.parent);
579

580
	rhashtable_destroy(&fg->ftes_hash);
581
	ida_destroy(&fg->fte_allocator);
582
583
584
	if (ft->autogroup.active &&
	    fg->max_ftes == ft->autogroup.group_size &&
	    fg->start_index < ft->autogroup.max_fte)
585
		ft->autogroup.num_groups--;
586
587
588
589
	err = rhltable_remove(&ft->fgs_hash,
			      &fg->hash,
			      rhash_fg);
	WARN_ON(err);
590
	kmem_cache_free(steering->fgs_cache, fg);
591
592
}

593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte)
{
	int index;
	int ret;

	index = ida_simple_get(&fg->fte_allocator, 0, fg->max_ftes, GFP_KERNEL);
	if (index < 0)
		return index;

	fte->index = index + fg->start_index;
	ret = rhashtable_insert_fast(&fg->ftes_hash,
				     &fte->hash,
				     rhash_fte);
	if (ret)
		goto err_ida_remove;

	tree_add_node(&fte->node, &fg->node);
	list_add_tail(&fte->node.list, &fg->node.children);
	return 0;

err_ida_remove:
	ida_simple_remove(&fg->fte_allocator, index);
	return ret;
}

618
static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft,
619
				const struct mlx5_flow_spec *spec,
620
				struct mlx5_flow_act *flow_act)
621
{
622
	struct mlx5_flow_steering *steering = get_steering(&ft->node);
623
624
	struct fs_fte *fte;

625
	fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL);
626
627
628
	if (!fte)
		return ERR_PTR(-ENOMEM);

629
	memcpy(fte->val, &spec->match_value, sizeof(fte->val));
630
	fte->node.type =  FS_TYPE_FLOW_ENTRY;
631
	fte->action = *flow_act;
632
	fte->flow_context = spec->flow_context;
633

634
	tree_init_node(&fte->node, NULL, del_sw_fte);
635
636
637
638

	return fte;
}

639
640
static void dealloc_flow_group(struct mlx5_flow_steering *steering,
			       struct mlx5_flow_group *fg)
641
642
{
	rhashtable_destroy(&fg->ftes_hash);
643
	kmem_cache_free(steering->fgs_cache, fg);
644
645
}

646
647
static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering,
						u8 match_criteria_enable,
648
						const void *match_criteria,
649
650
						int start_index,
						int end_index)
651
652
{
	struct mlx5_flow_group *fg;
653
654
	int ret;

655
	fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);
656
657
658
	if (!fg)
		return ERR_PTR(-ENOMEM);

659
660
	ret = rhashtable_init(&fg->ftes_hash, &rhash_fte);
	if (ret) {
661
		kmem_cache_free(steering->fgs_cache, fg);
662
		return ERR_PTR(ret);
663
664
	}

665
	ida_init(&fg->fte_allocator);
666
667
668
669
	fg->mask.match_criteria_enable = match_criteria_enable;
	memcpy(&fg->mask.match_criteria, match_criteria,
	       sizeof(fg->mask.match_criteria));
	fg->node.type =  FS_TYPE_FLOW_GROUP;
670
671
672
673
674
675
676
677
	fg->start_index = start_index;
	fg->max_ftes = end_index - start_index + 1;

	return fg;
}

static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *ft,
						       u8 match_criteria_enable,
678
						       const void *match_criteria,
679
680
681
682
						       int start_index,
						       int end_index,
						       struct list_head *prev)
{
683
	struct mlx5_flow_steering *steering = get_steering(&ft->node);
684
685
686
	struct mlx5_flow_group *fg;
	int ret;

687
	fg = alloc_flow_group(steering, match_criteria_enable, match_criteria,
688
689
690
691
692
693
694
695
696
			      start_index, end_index);
	if (IS_ERR(fg))
		return fg;

	/* initialize refcnt, add to parent list */
	ret = rhltable_insert(&ft->fgs_hash,
			      &fg->hash,
			      rhash_fg);
	if (ret) {
697
		dealloc_flow_group(steering, fg);
698
699
700
		return ERR_PTR(ret);
	}

701
	tree_init_node(&fg->node, del_hw_flow_group, del_sw_flow_group);
702
703
704
	tree_add_node(&fg->node, &ft->node);
	/* Add node to group list */
	list_add(&fg->node.list, prev);
705
	atomic_inc(&ft->node.version);
706

707
708
709
	return fg;
}

710
static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_fte,
711
						enum fs_flow_table_type table_type,
712
713
						enum fs_flow_table_op_mod op_mod,
						u32 flags)
714
715
{
	struct mlx5_flow_table *ft;
716
	int ret;
717
718
719

	ft  = kzalloc(sizeof(*ft), GFP_KERNEL);
	if (!ft)
720
721
722
723
724
725
726
		return ERR_PTR(-ENOMEM);

	ret = rhltable_init(&ft->fgs_hash, &rhash_fg);
	if (ret) {
		kfree(ft);
		return ERR_PTR(ret);
	}
727
728
729

	ft->level = level;
	ft->node.type = FS_TYPE_FLOW_TABLE;
730
	ft->op_mod = op_mod;
731
	ft->type = table_type;
732
	ft->vport = vport;
733
	ft->max_fte = max_fte;
734
	ft->flags = flags;
735
736
	INIT_LIST_HEAD(&ft->fwd_rules);
	mutex_init(&ft->lock);
737
738
739
740

	return ft;
}

741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
/* If reverse is false, then we search for the first flow table in the
 * root sub-tree from start(closest from right), else we search for the
 * last flow table in the root sub-tree till start(closest from left).
 */
static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node  *root,
							 struct list_head *start,
							 bool reverse)
{
#define list_advance_entry(pos, reverse)		\
	((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))

#define list_for_each_advance_continue(pos, head, reverse)	\
	for (pos = list_advance_entry(pos, reverse);		\
	     &pos->list != (head);				\
	     pos = list_advance_entry(pos, reverse))

	struct fs_node *iter = list_entry(start, struct fs_node, list);
	struct mlx5_flow_table *ft = NULL;

760
	if (!root || root->type == FS_TYPE_PRIO_CHAINS)
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
		return NULL;

	list_for_each_advance_continue(iter, &root->children, reverse) {
		if (iter->type == FS_TYPE_FLOW_TABLE) {
			fs_get_obj(ft, iter);
			return ft;
		}
		ft = find_closest_ft_recursive(iter, &iter->children, reverse);
		if (ft)
			return ft;
	}

	return ft;
}

/* If reverse if false then return the first flow table in next priority of
 * prio in the tree, else return the last flow table in the previous priority
 * of prio in the tree.
 */
static struct mlx5_flow_table *find_closest_ft(struct fs_prio *prio, bool reverse)
{
	struct mlx5_flow_table *ft = NULL;
	struct fs_node *curr_node;
	struct fs_node *parent;

	parent = prio->node.parent;
	curr_node = &prio->node;
	while (!ft && parent) {
		ft = find_closest_ft_recursive(parent, &curr_node->list, reverse);
		curr_node = parent;
		parent = curr_node->parent;
	}
	return ft;
}

/* Assuming all the tree is locked by mutex chain lock */
static struct mlx5_flow_table *find_next_chained_ft(struct fs_prio *prio)
{
	return find_closest_ft(prio, false);
}

/* Assuming all the tree is locked by mutex chain lock */
static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio)
{
	return find_closest_ft(prio, true);
}

808
809
810
811
static int connect_fts_in_prio(struct mlx5_core_dev *dev,
			       struct fs_prio *prio,
			       struct mlx5_flow_table *ft)
{
812
	struct mlx5_flow_root_namespace *root = find_root(&prio->node);
813
814
815
816
817
818
	struct mlx5_flow_table *iter;
	int i = 0;
	int err;

	fs_for_each_ft(iter, prio) {
		i++;
819
		err = root->cmds->modify_flow_table(root, iter, ft);
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
		if (err) {
			mlx5_core_warn(dev, "Failed to modify flow table %d\n",
				       iter->id);
			/* The driver is out of sync with the FW */
			if (i > 1)
				WARN_ON(true);
			return err;
		}
	}
	return 0;
}

/* Connect flow tables from previous priority of prio to ft */
static int connect_prev_fts(struct mlx5_core_dev *dev,
			    struct mlx5_flow_table *ft,
			    struct fs_prio *prio)
{
	struct mlx5_flow_table *prev_ft;

	prev_ft = find_prev_chained_ft(prio);
	if (prev_ft) {
		struct fs_prio *prev_prio;

		fs_get_obj(prev_prio, prev_ft->node.parent);
		return connect_fts_in_prio(dev, prev_prio, ft);
	}
	return 0;
}

849
850
851
852
static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio
				 *prio)
{
	struct mlx5_flow_root_namespace *root = find_root(&prio->node);
853
	struct mlx5_ft_underlay_qp *uqp;
854
	int min_level = INT_MAX;
855
	int err = 0;
856
	u32 qpn;
857
858
859
860
861
862
863

	if (root->root_ft)
		min_level = root->root_ft->level;

	if (ft->level >= min_level)
		return 0;

864
865
866
	if (list_empty(&root->underlay_qpns)) {
		/* Don't set any QPN (zero) in case QPN list is empty */
		qpn = 0;
867
		err = root->cmds->update_root_ft(root, ft, qpn, false);
868
869
870
	} else {
		list_for_each_entry(uqp, &root->underlay_qpns, list) {
			qpn = uqp->qpn;
871
			err = root->cmds->update_root_ft(root, ft,
872
							 qpn, false);
873
874
875
876
877
			if (err)
				break;
		}
	}

878
	if (err)
879
880
881
		mlx5_core_warn(root->dev,
			       "Update root flow table of id(%u) qpn(%d) failed\n",
			       ft->id, qpn);
882
883
884
885
886
887
	else
		root->root_ft = ft;

	return err;
}

Mark Bloch's avatar
Mark Bloch committed
888
889
static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule,
					 struct mlx5_flow_destination *dest)
890
{
891
	struct mlx5_flow_root_namespace *root;
892
893
894
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *fg;
	struct fs_fte *fte;
895
	int modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
896
897
898
	int err = 0;

	fs_get_obj(fte, rule->node.parent);
899
	if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
900
		return -EINVAL;
901
	down_write_ref_node(&fte->node, false);
902
903
904
905
	fs_get_obj(fg, fte->node.parent);
	fs_get_obj(ft, fg->node.parent);

	memcpy(&rule->dest_attr, dest, sizeof(*dest));
906
	root = find_root(&ft->node);
907
	err = root->cmds->update_fte(root, ft, fg,
908
				     modify_mask, fte);
909
	up_write_ref_node(&fte->node, false);
910
911
912
913

	return err;
}

Mark Bloch's avatar
Mark Bloch committed
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle,
				 struct mlx5_flow_destination *new_dest,
				 struct mlx5_flow_destination *old_dest)
{
	int i;

	if (!old_dest) {
		if (handle->num_rules != 1)
			return -EINVAL;
		return _mlx5_modify_rule_destination(handle->rule[0],
						     new_dest);
	}

	for (i = 0; i < handle->num_rules; i++) {
		if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr))
			return _mlx5_modify_rule_destination(handle->rule[i],
							     new_dest);
	}

	return -EINVAL;
}

936
937
938
939
940
/* Modify/set FWD rules that point on old_next_ft to point on new_next_ft  */
static int connect_fwd_rules(struct mlx5_core_dev *dev,
			     struct mlx5_flow_table *new_next_ft,
			     struct mlx5_flow_table *old_next_ft)
{
941
	struct mlx5_flow_destination dest = {};
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
	struct mlx5_flow_rule *iter;
	int err = 0;

	/* new_next_ft and old_next_ft could be NULL only
	 * when we create/destroy the anchor flow table.
	 */
	if (!new_next_ft || !old_next_ft)
		return 0;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	dest.ft = new_next_ft;

	mutex_lock(&old_next_ft->lock);
	list_splice_init(&old_next_ft->fwd_rules, &new_next_ft->fwd_rules);
	mutex_unlock(&old_next_ft->lock);
	list_for_each_entry(iter, &new_next_ft->fwd_rules, next_ft) {
Mark Bloch's avatar
Mark Bloch committed
958
		err = _mlx5_modify_rule_destination(iter, &dest);
959
960
961
962
963
964
965
		if (err)
			pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
			       new_next_ft->id);
	}
	return 0;
}

966
967
968
static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
			      struct fs_prio *prio)
{
969
	struct mlx5_flow_table *next_ft;
970
971
972
973
974
975
976
977
	int err = 0;

	/* Connect_prev_fts and update_root_ft_create are mutually exclusive */

	if (list_empty(&prio->node.children)) {
		err = connect_prev_fts(dev, ft, prio);
		if (err)
			return err;
978
979
980
981
982

		next_ft = find_next_chained_ft(prio);
		err = connect_fwd_rules(dev, ft, next_ft);
		if (err)
			return err;
983
984
985
986
987
988
989
990
	}

	if (MLX5_CAP_FLOWTABLE(dev,
			       flow_table_properties_nic_receive.modify_root))
		err = update_root_ft_create(ft, prio);
	return err;
}

991
992
993
994
995
996
997
998
999
1000
static void list_add_flow_table(struct mlx5_flow_table *ft,
				struct fs_prio *prio)
{
	struct list_head *prev = &prio->node.children;
	struct mlx5_flow_table *iter;

	fs_for_each_ft(iter, prio) {
		if (iter->level > ft->level)
			break;
		prev = &iter->node.list;