fs_cmd.c 27.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/*
 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/mlx5/driver.h>
#include <linux/mlx5/device.h>
#include <linux/mlx5/mlx5_ifc.h>

#include "fs_core.h"
#include "fs_cmd.h"
#include "mlx5_core.h"
40
#include "eswitch.h"
41

42
static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns,
43
44
45
46
47
48
49
					struct mlx5_flow_table *ft,
					u32 underlay_qpn,
					bool disconnect)
{
	return 0;
}

50
51
static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns,
					   struct mlx5_flow_table *ft,
52
					   unsigned int log_size,
53
					   struct mlx5_flow_table *next_ft)
54
55
56
57
{
	return 0;
}

58
static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
59
60
61
62
63
					    struct mlx5_flow_table *ft)
{
	return 0;
}

64
static int mlx5_cmd_stub_modify_flow_table(struct mlx5_flow_root_namespace *ns,
65
66
67
68
69
70
					   struct mlx5_flow_table *ft,
					   struct mlx5_flow_table *next_ft)
{
	return 0;
}

71
static int mlx5_cmd_stub_create_flow_group(struct mlx5_flow_root_namespace *ns,
72
73
					   struct mlx5_flow_table *ft,
					   u32 *in,
74
					   struct mlx5_flow_group *fg)
75
76
77
78
{
	return 0;
}

79
static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
80
					    struct mlx5_flow_table *ft,
81
					    struct mlx5_flow_group *fg)
82
83
84
85
{
	return 0;
}

86
static int mlx5_cmd_stub_create_fte(struct mlx5_flow_root_namespace *ns,
87
88
89
90
91
92
93
				    struct mlx5_flow_table *ft,
				    struct mlx5_flow_group *group,
				    struct fs_fte *fte)
{
	return 0;
}

94
static int mlx5_cmd_stub_update_fte(struct mlx5_flow_root_namespace *ns,
95
				    struct mlx5_flow_table *ft,
96
				    struct mlx5_flow_group *group,
97
98
99
100
101
102
				    int modify_mask,
				    struct fs_fte *fte)
{
	return -EOPNOTSUPP;
}

103
static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns,
104
				    struct mlx5_flow_table *ft,
105
				    struct fs_fte *fte)
106
107
108
109
{
	return 0;
}

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
static int mlx5_cmd_stub_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
					       int reformat_type,
					       size_t size,
					       void *reformat_data,
					       enum mlx5_flow_namespace_type namespace,
					       struct mlx5_pkt_reformat *pkt_reformat)
{
	return 0;
}

static void mlx5_cmd_stub_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
						  struct mlx5_pkt_reformat *pkt_reformat)
{
}

static int mlx5_cmd_stub_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
					     u8 namespace, u8 num_actions,
					     void *modify_actions,
					     struct mlx5_modify_hdr *modify_hdr)
{
	return 0;
}

static void mlx5_cmd_stub_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
						struct mlx5_modify_hdr *modify_hdr)
{
}

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
static int mlx5_cmd_stub_set_peer(struct mlx5_flow_root_namespace *ns,
				  struct mlx5_flow_root_namespace *peer_ns)
{
	return 0;
}

static int mlx5_cmd_stub_create_ns(struct mlx5_flow_root_namespace *ns)
{
	return 0;
}

static int mlx5_cmd_stub_destroy_ns(struct mlx5_flow_root_namespace *ns)
{
	return 0;
}

154
static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns,
155
156
				   struct mlx5_flow_table *ft, u32 underlay_qpn,
				   bool disconnect)
157
{
158
159
	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)]   = {0};
	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0};
160
	struct mlx5_core_dev *dev = ns->dev;
161

162
	if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
163
	    underlay_qpn == 0)
164
165
		return 0;

166
167
168
	MLX5_SET(set_flow_table_root_in, in, opcode,
		 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
	MLX5_SET(set_flow_table_root_in, in, table_type, ft->type);
169
170
171
172
173
174
175
176
177

	if (disconnect) {
		MLX5_SET(set_flow_table_root_in, in, op_mod, 1);
		MLX5_SET(set_flow_table_root_in, in, table_id, 0);
	} else {
		MLX5_SET(set_flow_table_root_in, in, op_mod, 0);
		MLX5_SET(set_flow_table_root_in, in, table_id, ft->id);
	}

178
	MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn);
179
180
181
182
	if (ft->vport) {
		MLX5_SET(set_flow_table_root_in, in, vport_number, ft->vport);
		MLX5_SET(set_flow_table_root_in, in, other_vport, 1);
	}
183

184
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
185
186
}

187
188
static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
				      struct mlx5_flow_table *ft,
189
				      unsigned int log_size,
190
				      struct mlx5_flow_table *next_ft)
191
{
192
193
	int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT);
	int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
194
	int term = !!(ft->flags & MLX5_FLOW_TABLE_TERMINATION);
195
196
	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0};
	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)]   = {0};
197
	struct mlx5_core_dev *dev = ns->dev;
198
199
200
201
202
	int err;

	MLX5_SET(create_flow_table_in, in, opcode,
		 MLX5_CMD_OP_CREATE_FLOW_TABLE);

203
204
	MLX5_SET(create_flow_table_in, in, table_type, ft->type);
	MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level);
205
	MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, log_size);
206
207
	if (ft->vport) {
		MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
208
209
		MLX5_SET(create_flow_table_in, in, other_vport, 1);
	}
210

211
	MLX5_SET(create_flow_table_in, in, flow_table_context.decap_en,
212
		 en_decap);
213
	MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en,
214
		 en_encap);
215
216
	MLX5_SET(create_flow_table_in, in, flow_table_context.termination_table,
		 term);
217

218
	switch (ft->op_mod) {
219
220
	case FS_FT_OP_MOD_NORMAL:
		if (next_ft) {
221
			MLX5_SET(create_flow_table_in, in,
222
223
				 flow_table_context.table_miss_action,
				 MLX5_FLOW_TABLE_MISS_ACTION_FWD);
224
225
			MLX5_SET(create_flow_table_in, in,
				 flow_table_context.table_miss_id, next_ft->id);
226
227
228
		} else {
			MLX5_SET(create_flow_table_in, in,
				 flow_table_context.table_miss_action,
229
				 ft->def_miss_action);
230
231
232
233
234
235
		}
		break;

	case FS_FT_OP_MOD_LAG_DEMUX:
		MLX5_SET(create_flow_table_in, in, op_mod, 0x1);
		if (next_ft)
236
237
			MLX5_SET(create_flow_table_in, in,
				 flow_table_context.lag_master_next_table_id,
238
239
240
241
				 next_ft->id);
		break;
	}

242
	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
243
	if (!err)
244
245
		ft->id = MLX5_GET(create_flow_table_out, out,
				  table_id);
246
247
248
	return err;
}

249
static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
250
				       struct mlx5_flow_table *ft)
251
{
252
253
	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)]   = {0};
	u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0};
254
	struct mlx5_core_dev *dev = ns->dev;
255
256
257
258
259

	MLX5_SET(destroy_flow_table_in, in, opcode,
		 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
	MLX5_SET(destroy_flow_table_in, in, table_type, ft->type);
	MLX5_SET(destroy_flow_table_in, in, table_id, ft->id);
260
261
262
263
	if (ft->vport) {
		MLX5_SET(destroy_flow_table_in, in, vport_number, ft->vport);
		MLX5_SET(destroy_flow_table_in, in, other_vport, 1);
	}
264

265
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
266
267
}

268
static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns,
269
270
				      struct mlx5_flow_table *ft,
				      struct mlx5_flow_table *next_ft)
271
{
272
273
	u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)]   = {0};
	u32 out[MLX5_ST_SZ_DW(modify_flow_table_out)] = {0};
274
	struct mlx5_core_dev *dev = ns->dev;
275
276
277
278
279

	MLX5_SET(modify_flow_table_in, in, opcode,
		 MLX5_CMD_OP_MODIFY_FLOW_TABLE);
	MLX5_SET(modify_flow_table_in, in, table_type, ft->type);
	MLX5_SET(modify_flow_table_in, in, table_id, ft->id);
280
281
282
283
284
285

	if (ft->op_mod == FS_FT_OP_MOD_LAG_DEMUX) {
		MLX5_SET(modify_flow_table_in, in, modify_field_select,
			 MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID);
		if (next_ft) {
			MLX5_SET(modify_flow_table_in, in,
286
				 flow_table_context.lag_master_next_table_id, next_ft->id);
287
288
		} else {
			MLX5_SET(modify_flow_table_in, in,
289
				 flow_table_context.lag_master_next_table_id, 0);
290
		}
291
	} else {
292
293
294
295
296
297
298
299
		if (ft->vport) {
			MLX5_SET(modify_flow_table_in, in, vport_number,
				 ft->vport);
			MLX5_SET(modify_flow_table_in, in, other_vport, 1);
		}
		MLX5_SET(modify_flow_table_in, in, modify_field_select,
			 MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID);
		if (next_ft) {
300
			MLX5_SET(modify_flow_table_in, in,
301
302
				 flow_table_context.table_miss_action,
				 MLX5_FLOW_TABLE_MISS_ACTION_FWD);
303
304
			MLX5_SET(modify_flow_table_in, in,
				 flow_table_context.table_miss_id,
305
306
				 next_ft->id);
		} else {
307
			MLX5_SET(modify_flow_table_in, in,
308
				 flow_table_context.table_miss_action,
309
				 ft->def_miss_action);
310
		}
311
312
	}

313
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
314
315
}

316
static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns,
317
318
				      struct mlx5_flow_table *ft,
				      u32 *in,
319
				      struct mlx5_flow_group *fg)
320
{
321
	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0};
322
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
323
	struct mlx5_core_dev *dev = ns->dev;
324
325
326
327
328
329
	int err;

	MLX5_SET(create_flow_group_in, in, opcode,
		 MLX5_CMD_OP_CREATE_FLOW_GROUP);
	MLX5_SET(create_flow_group_in, in, table_type, ft->type);
	MLX5_SET(create_flow_group_in, in, table_id, ft->id);
330
331
332
333
	if (ft->vport) {
		MLX5_SET(create_flow_group_in, in, vport_number, ft->vport);
		MLX5_SET(create_flow_group_in, in, other_vport, 1);
	}
334

335
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
336
	if (!err)
337
338
		fg->id = MLX5_GET(create_flow_group_out, out,
				  group_id);
339
340
341
	return err;
}

342
static int mlx5_cmd_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
343
				       struct mlx5_flow_table *ft,
344
				       struct mlx5_flow_group *fg)
345
{
346
347
	u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0};
	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)]   = {0};
348
	struct mlx5_core_dev *dev = ns->dev;
349
350
351
352
353

	MLX5_SET(destroy_flow_group_in, in, opcode,
		 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
	MLX5_SET(destroy_flow_group_in, in, table_type, ft->type);
	MLX5_SET(destroy_flow_group_in, in, table_id, ft->id);
354
	MLX5_SET(destroy_flow_group_in, in, group_id, fg->id);
355
356
357
358
	if (ft->vport) {
		MLX5_SET(destroy_flow_group_in, in, vport_number, ft->vport);
		MLX5_SET(destroy_flow_group_in, in, other_vport, 1);
	}
359

360
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
361
362
}

363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
static int mlx5_set_extended_dest(struct mlx5_core_dev *dev,
				  struct fs_fte *fte, bool *extended_dest)
{
	int fw_log_max_fdb_encap_uplink =
		MLX5_CAP_ESW(dev, log_max_fdb_encap_uplink);
	int num_fwd_destinations = 0;
	struct mlx5_flow_rule *dst;
	int num_encap = 0;

	*extended_dest = false;
	if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
		return 0;

	list_for_each_entry(dst, &fte->node.children, node.list) {
		if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)
			continue;
		if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_VPORT &&
		    dst->dest_attr.vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)
			num_encap++;
		num_fwd_destinations++;
	}
	if (num_fwd_destinations > 1 && num_encap > 0)
		*extended_dest = true;

	if (*extended_dest && !fw_log_max_fdb_encap_uplink) {
		mlx5_core_warn(dev, "FW does not support extended destination");
		return -EOPNOTSUPP;
	}
	if (num_encap > (1 << fw_log_max_fdb_encap_uplink)) {
		mlx5_core_warn(dev, "FW does not support more than %d encaps",
			       1 << fw_log_max_fdb_encap_uplink);
		return -EOPNOTSUPP;
	}

	return 0;
}
399
400
401
402
403
404
static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
			    int opmod, int modify_mask,
			    struct mlx5_flow_table *ft,
			    unsigned group_id,
			    struct fs_fte *fte)
{
405
	u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
406
	bool extended_dest = false;
407
	struct mlx5_flow_rule *dst;
408
	void *in_flow_context, *vlan;
409
	void *in_match_value;
410
411
	unsigned int inlen;
	int dst_cnt_size;
412
413
414
415
	void *in_dests;
	u32 *in;
	int err;

416
417
418
419
420
421
422
423
424
	if (mlx5_set_extended_dest(dev, fte, &extended_dest))
		return -EOPNOTSUPP;

	if (!extended_dest)
		dst_cnt_size = MLX5_ST_SZ_BYTES(dest_format_struct);
	else
		dst_cnt_size = MLX5_ST_SZ_BYTES(extended_dest_format);

	inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fte->dests_size * dst_cnt_size;
425
426
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
427
428
429
430
431
432
433
434
		return -ENOMEM;

	MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
	MLX5_SET(set_fte_in, in, op_mod, opmod);
	MLX5_SET(set_fte_in, in, modify_enable_mask, modify_mask);
	MLX5_SET(set_fte_in, in, table_type, ft->type);
	MLX5_SET(set_fte_in, in, table_id,   ft->id);
	MLX5_SET(set_fte_in, in, flow_index, fte->index);
435
436
437
438
	if (ft->vport) {
		MLX5_SET(set_fte_in, in, vport_number, ft->vport);
		MLX5_SET(set_fte_in, in, other_vport, 1);
	}
439
440
441

	in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
	MLX5_SET(flow_context, in_flow_context, group_id, group_id);
442

443
444
	MLX5_SET(flow_context, in_flow_context, flow_tag,
		 fte->flow_context.flow_tag);
445
446
447
	MLX5_SET(flow_context, in_flow_context, flow_source,
		 fte->flow_context.flow_source);

448
449
450
451
452
453
454
455
456
457
458
	MLX5_SET(flow_context, in_flow_context, extended_destination,
		 extended_dest);
	if (extended_dest) {
		u32 action;

		action = fte->action.action &
			~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
		MLX5_SET(flow_context, in_flow_context, action, action);
	} else {
		MLX5_SET(flow_context, in_flow_context, action,
			 fte->action.action);
459
460
461
		if (fte->action.pkt_reformat)
			MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
				 fte->action.pkt_reformat->id);
462
	}
463
464
465
	if (fte->action.modify_hdr)
		MLX5_SET(flow_context, in_flow_context, modify_header_id,
			 fte->action.modify_hdr->id);
466
467
468

	vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan);

469
470
471
472
473
474
475
476
477
	MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[0].ethtype);
	MLX5_SET(vlan, vlan, vid, fte->action.vlan[0].vid);
	MLX5_SET(vlan, vlan, prio, fte->action.vlan[0].prio);

	vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan_2);

	MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[1].ethtype);
	MLX5_SET(vlan, vlan, vid, fte->action.vlan[1].vid);
	MLX5_SET(vlan, vlan, prio, fte->action.vlan[1].prio);
478

479
480
	in_match_value = MLX5_ADDR_OF(flow_context, in_flow_context,
				      match_value);
481
	memcpy(in_match_value, &fte->val, sizeof(fte->val));
482

483
	in_dests = MLX5_ADDR_OF(flow_context, in_flow_context, destination);
484
	if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
485
486
		int list_size = 0;

487
		list_for_each_entry(dst, &fte->node.children, node.list) {
488
			unsigned int id, type = dst->dest_attr.type;
489

490
			if (type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)
491
492
				continue;

493
494
495
496
497
498
			switch (type) {
			case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM:
				id = dst->dest_attr.ft_num;
				type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
				break;
			case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
499
				id = dst->dest_attr.ft->id;
500
501
				break;
			case MLX5_FLOW_DESTINATION_TYPE_VPORT:
502
503
504
				id = dst->dest_attr.vport.num;
				MLX5_SET(dest_format_struct, in_dests,
					 destination_eswitch_owner_vhca_id_valid,
505
506
					 !!(dst->dest_attr.vport.flags &
					    MLX5_FLOW_DEST_VPORT_VHCA_ID));
507
508
509
				MLX5_SET(dest_format_struct, in_dests,
					 destination_eswitch_owner_vhca_id,
					 dst->dest_attr.vport.vhca_id);
510
511
512
513
514
515
516
				if (extended_dest) {
					MLX5_SET(dest_format_struct, in_dests,
						 packet_reformat,
						 !!(dst->dest_attr.vport.flags &
						    MLX5_FLOW_DEST_VPORT_REFORMAT_ID));
					MLX5_SET(extended_dest_format, in_dests,
						 packet_reformat_id,
517
						 dst->dest_attr.vport.pkt_reformat->id);
518
				}
519
520
				break;
			default:
521
522
				id = dst->dest_attr.tir_num;
			}
523
524
525

			MLX5_SET(dest_format_struct, in_dests, destination_type,
				 type);
526
			MLX5_SET(dest_format_struct, in_dests, destination_id, id);
527
			in_dests += dst_cnt_size;
528
529
530
531
532
533
534
			list_size++;
		}

		MLX5_SET(flow_context, in_flow_context, destination_list_size,
			 list_size);
	}

535
	if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
536
537
538
		int max_list_size = BIT(MLX5_CAP_FLOWTABLE_TYPE(dev,
					log_max_flow_counter,
					ft->type));
539
540
541
542
543
544
545
546
		int list_size = 0;

		list_for_each_entry(dst, &fte->node.children, node.list) {
			if (dst->dest_attr.type !=
			    MLX5_FLOW_DESTINATION_TYPE_COUNTER)
				continue;

			MLX5_SET(flow_counter_list, in_dests, flow_counter_id,
547
				 dst->dest_attr.counter_id);
548
			in_dests += dst_cnt_size;
549
			list_size++;
550
		}
551
552
553
554
		if (list_size > max_list_size) {
			err = -EINVAL;
			goto err_out;
		}
555
556
557

		MLX5_SET(flow_context, in_flow_context, flow_counter_list_size,
			 list_size);
558
	}
559

560
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
561
err_out:
562
563
564
565
	kvfree(in);
	return err;
}

566
static int mlx5_cmd_create_fte(struct mlx5_flow_root_namespace *ns,
567
568
569
			       struct mlx5_flow_table *ft,
			       struct mlx5_flow_group *group,
			       struct fs_fte *fte)
570
{
571
	struct mlx5_core_dev *dev = ns->dev;
572
573
	unsigned int group_id = group->id;

574
	return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte);
575
576
}

577
static int mlx5_cmd_update_fte(struct mlx5_flow_root_namespace *ns,
578
			       struct mlx5_flow_table *ft,
579
			       struct mlx5_flow_group *fg,
580
581
			       int modify_mask,
			       struct fs_fte *fte)
582
583
{
	int opmod;
584
	struct mlx5_core_dev *dev = ns->dev;
585
586
587
588
	int atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
						flow_table_properties_nic_receive.
						flow_modify_en);
	if (!atomic_mod_cap)
589
		return -EOPNOTSUPP;
590
591
	opmod = 1;

592
	return	mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, fg->id, fte);
593
594
}

595
static int mlx5_cmd_delete_fte(struct mlx5_flow_root_namespace *ns,
596
			       struct mlx5_flow_table *ft,
597
			       struct fs_fte *fte)
598
{
599
600
	u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0};
	u32 in[MLX5_ST_SZ_DW(delete_fte_in)]   = {0};
601
	struct mlx5_core_dev *dev = ns->dev;
602
603
604
605

	MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
	MLX5_SET(delete_fte_in, in, table_type, ft->type);
	MLX5_SET(delete_fte_in, in, table_id, ft->id);
606
	MLX5_SET(delete_fte_in, in, flow_index, fte->index);
607
608
609
610
	if (ft->vport) {
		MLX5_SET(delete_fte_in, in, vport_number, ft->vport);
		MLX5_SET(delete_fte_in, in, other_vport, 1);
	}
611

612
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
613
}
614

615
616
617
int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
			   enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
			   u32 *id)
618
{
619
620
	u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)]   = {0};
	u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {0};
621
622
623
624
	int err;

	MLX5_SET(alloc_flow_counter_in, in, opcode,
		 MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
625
	MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
626

627
628
629
630
	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	if (!err)
		*id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
	return err;
631
632
}

633
634
635
636
637
int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id)
{
	return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
}

638
int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
639
{
640
641
	u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)]   = {0};
	u32 out[MLX5_ST_SZ_DW(dealloc_flow_counter_out)] = {0};
642
643
644
645

	MLX5_SET(dealloc_flow_counter_in, in, opcode,
		 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
	MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
646
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
647
648
}

649
int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
650
651
652
		      u64 *packets, u64 *bytes)
{
	u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
653
654
		MLX5_ST_SZ_BYTES(traffic_counter)]   = {0};
	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {0};
655
656
657
658
659
660
661
	void *stats;
	int err = 0;

	MLX5_SET(query_flow_counter_in, in, opcode,
		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
	MLX5_SET(query_flow_counter_in, in, op_mod, 0);
	MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
662
	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
663
664
665
666
667
668
669
670
	if (err)
		return err;

	stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
	*packets = MLX5_GET64(traffic_counter, stats, packets);
	*bytes = MLX5_GET64(traffic_counter, stats, octets);
	return 0;
}
671

672
int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
673
{
674
675
	return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
		MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
676
677
}

678
679
int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
			   u32 *out)
680
{
681
	int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
682
	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {0};
683
684
685
686

	MLX5_SET(query_flow_counter_in, in, opcode,
		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
	MLX5_SET(query_flow_counter_in, in, op_mod, 0);
687
688
689
	MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
	MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
690
}
691

692
693
694
695
696
697
static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
					  int reformat_type,
					  size_t size,
					  void *reformat_data,
					  enum mlx5_flow_namespace_type namespace,
					  struct mlx5_pkt_reformat *pkt_reformat)
698
{
699
	u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)];
700
	struct mlx5_core_dev *dev = ns->dev;
701
	void *packet_reformat_context_in;
702
	int max_encap_size;
703
	void *reformat;
704
	int inlen;
705
	int err;
706
	u32 *in;
707

708
709
710
711
712
	if (namespace == MLX5_FLOW_NAMESPACE_FDB)
		max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
	else
		max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);

713
714
715
	if (size > max_encap_size) {
		mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
			       size, max_encap_size);
716
		return -EINVAL;
717
	}
718

719
	in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) + size,
720
721
722
723
		     GFP_KERNEL);
	if (!in)
		return -ENOMEM;

724
725
726
727
728
729
	packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
						  in, packet_reformat_context);
	reformat = MLX5_ADDR_OF(packet_reformat_context_in,
				packet_reformat_context_in,
				reformat_data);
	inlen = reformat - (void *)in  + size;
730

731
	memset(in, 0, inlen);
732
733
734
735
736
737
738
	MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
		 MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
	MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
		 reformat_data_size, size);
	MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
		 reformat_type, reformat_type);
	memcpy(reformat, reformat_data, size);
739
740
741
742

	memset(out, 0, sizeof(out));
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));

743
744
	pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
				    out, packet_reformat_id);
745
	kfree(in);
746
747
748
	return err;
}

749
750
static void mlx5_cmd_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
					     struct mlx5_pkt_reformat *pkt_reformat)
751
{
752
753
	u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)];
	u32 out[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_out)];
754
	struct mlx5_core_dev *dev = ns->dev;
755
756

	memset(in, 0, sizeof(in));
757
758
759
	MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
		 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
	MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
760
		 pkt_reformat->id);
761
762
763

	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
764

765
766
767
768
static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
					u8 namespace, u8 num_actions,
					void *modify_actions,
					struct mlx5_modify_hdr *modify_hdr)
769
770
771
{
	u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)];
	int max_actions, actions_size, inlen, err;
772
	struct mlx5_core_dev *dev = ns->dev;
773
774
775
776
777
778
779
780
781
782
	void *actions_in;
	u8 table_type;
	u32 *in;

	switch (namespace) {
	case MLX5_FLOW_NAMESPACE_FDB:
		max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
		table_type = FS_FT_FDB;
		break;
	case MLX5_FLOW_NAMESPACE_KERNEL:
783
	case MLX5_FLOW_NAMESPACE_BYPASS:
784
785
786
		max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
		table_type = FS_FT_NIC_RX;
		break;
787
788
789
790
	case MLX5_FLOW_NAMESPACE_EGRESS:
		max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions);
		table_type = FS_FT_NIC_TX;
		break;
791
792
793
794
	case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
		max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
		table_type = FS_FT_ESW_INGRESS_ACL;
		break;
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
	default:
		return -EOPNOTSUPP;
	}

	if (num_actions > max_actions) {
		mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
			       num_actions, max_actions);
		return -EOPNOTSUPP;
	}

	actions_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto) * num_actions;
	inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;

	in = kzalloc(inlen, GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	MLX5_SET(alloc_modify_header_context_in, in, opcode,
		 MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
	MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
	MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);

	actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
	memcpy(actions_in, modify_actions, actions_size);

	memset(out, 0, sizeof(out));
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));

823
	modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
824
825
826
827
	kfree(in);
	return err;
}

828
829
static void mlx5_cmd_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
					   struct mlx5_modify_hdr *modify_hdr)
830
831
832
{
	u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)];
	u32 out[MLX5_ST_SZ_DW(dealloc_modify_header_context_out)];
833
	struct mlx5_core_dev *dev = ns->dev;
834
835
836
837
838

	memset(in, 0, sizeof(in));
	MLX5_SET(dealloc_modify_header_context_in, in, opcode,
		 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
	MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
839
		 modify_hdr->id);
840
841
842

	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
843
844
845
846
847
848
849
850
851
852
853

static const struct mlx5_flow_cmds mlx5_flow_cmds = {
	.create_flow_table = mlx5_cmd_create_flow_table,
	.destroy_flow_table = mlx5_cmd_destroy_flow_table,
	.modify_flow_table = mlx5_cmd_modify_flow_table,
	.create_flow_group = mlx5_cmd_create_flow_group,
	.destroy_flow_group = mlx5_cmd_destroy_flow_group,
	.create_fte = mlx5_cmd_create_fte,
	.update_fte = mlx5_cmd_update_fte,
	.delete_fte = mlx5_cmd_delete_fte,
	.update_root_ft = mlx5_cmd_update_root_ft,
854
855
856
	.packet_reformat_alloc = mlx5_cmd_packet_reformat_alloc,
	.packet_reformat_dealloc = mlx5_cmd_packet_reformat_dealloc,
	.modify_header_alloc = mlx5_cmd_modify_header_alloc,
857
858
859
860
	.modify_header_dealloc = mlx5_cmd_modify_header_dealloc,
	.set_peer = mlx5_cmd_stub_set_peer,
	.create_ns = mlx5_cmd_stub_create_ns,
	.destroy_ns = mlx5_cmd_stub_destroy_ns,
861
862
863
864
865
866
867
868
869
870
871
872
};

static const struct mlx5_flow_cmds mlx5_flow_cmd_stubs = {
	.create_flow_table = mlx5_cmd_stub_create_flow_table,
	.destroy_flow_table = mlx5_cmd_stub_destroy_flow_table,
	.modify_flow_table = mlx5_cmd_stub_modify_flow_table,
	.create_flow_group = mlx5_cmd_stub_create_flow_group,
	.destroy_flow_group = mlx5_cmd_stub_destroy_flow_group,
	.create_fte = mlx5_cmd_stub_create_fte,
	.update_fte = mlx5_cmd_stub_update_fte,
	.delete_fte = mlx5_cmd_stub_delete_fte,
	.update_root_ft = mlx5_cmd_stub_update_root_ft,
873
874
875
	.packet_reformat_alloc = mlx5_cmd_stub_packet_reformat_alloc,
	.packet_reformat_dealloc = mlx5_cmd_stub_packet_reformat_dealloc,
	.modify_header_alloc = mlx5_cmd_stub_modify_header_alloc,
876
877
878
879
	.modify_header_dealloc = mlx5_cmd_stub_modify_header_dealloc,
	.set_peer = mlx5_cmd_stub_set_peer,
	.create_ns = mlx5_cmd_stub_create_ns,
	.destroy_ns = mlx5_cmd_stub_destroy_ns,
880
881
};

882
const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void)
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
{
	return &mlx5_flow_cmds;
}

static const struct mlx5_flow_cmds *mlx5_fs_cmd_get_stub_cmds(void)
{
	return &mlx5_flow_cmd_stubs;
}

const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type)
{
	switch (type) {
	case FS_FT_NIC_RX:
	case FS_FT_ESW_EGRESS_ACL:
	case FS_FT_ESW_INGRESS_ACL:
	case FS_FT_FDB:
	case FS_FT_SNIFFER_RX:
	case FS_FT_SNIFFER_TX:
901
	case FS_FT_NIC_TX:
902
	case FS_FT_RDMA_RX:
903
		return mlx5_fs_cmd_get_fw_cmds();
904
905
906
907
	default:
		return mlx5_fs_cmd_get_stub_cmds();
	}
}