tls_rxtx.c 10.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/*
 * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include "en_accel/tls.h"
#include "en_accel/tls_rxtx.h"
36
37
#include "accel/accel.h"

38
39
40
41
42
43
#include <net/inet6_hashtables.h>
#include <linux/ipv6.h>

#define SYNDROM_DECRYPTED  0x30
#define SYNDROM_RESYNC_REQUEST 0x31
#define SYNDROM_AUTH_FAILED 0x32
44
45
46
47
48
49
50
51
52
53
54

#define SYNDROME_OFFLOAD_REQUIRED 32
#define SYNDROME_SYNC 33

struct sync_info {
	u64 rcd_sn;
	s32 sync_len;
	int nr_frags;
	skb_frag_t frags[MAX_SKB_FRAGS];
};

55
56
57
58
59
60
61
struct recv_metadata_content {
	u8 syndrome;
	u8 reserved;
	__be32 sync_seq;
} __packed;

struct send_metadata_content {
62
63
64
	/* One byte of syndrome followed by 3 bytes of swid */
	__be32 syndrome_swid;
	__be16 first_seq;
65
66
67
68
69
70
71
72
73
74
} __packed;

struct mlx5e_tls_metadata {
	union {
		/* from fpga to host */
		struct recv_metadata_content recv;
		/* from host to fpga */
		struct send_metadata_content send;
		unsigned char raw[6];
	} __packed content;
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
	/* packet type ID field	*/
	__be16 ethertype;
} __packed;

static int mlx5e_tls_add_metadata(struct sk_buff *skb, __be32 swid)
{
	struct mlx5e_tls_metadata *pet;
	struct ethhdr *eth;

	if (skb_cow_head(skb, sizeof(struct mlx5e_tls_metadata)))
		return -ENOMEM;

	eth = (struct ethhdr *)skb_push(skb, sizeof(struct mlx5e_tls_metadata));
	skb->mac_header -= sizeof(struct mlx5e_tls_metadata);
	pet = (struct mlx5e_tls_metadata *)(eth + 1);

	memmove(skb->data, skb->data + sizeof(struct mlx5e_tls_metadata),
		2 * ETH_ALEN);

	eth->h_proto = cpu_to_be16(MLX5E_METADATA_ETHER_TYPE);
95
96
	pet->content.send.syndrome_swid =
		htonl(SYNDROME_OFFLOAD_REQUIRED << 24) | swid;
97
98
99
100

	return 0;
}

101
static int mlx5e_tls_get_sync_data(struct mlx5e_tls_offload_context_tx *context,
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
				   u32 tcp_seq, struct sync_info *info)
{
	int remaining, i = 0, ret = -EINVAL;
	struct tls_record_info *record;
	unsigned long flags;
	s32 sync_size;

	spin_lock_irqsave(&context->base.lock, flags);
	record = tls_get_record(&context->base, tcp_seq, &info->rcd_sn);

	if (unlikely(!record))
		goto out;

	sync_size = tcp_seq - tls_record_start_seq(record);
	info->sync_len = sync_size;
	if (unlikely(sync_size < 0)) {
		if (tls_record_is_start_marker(record))
			goto done;

		goto out;
	}

	remaining = sync_size;
	while (remaining > 0) {
		info->frags[i] = record->frags[i];
		__skb_frag_ref(&info->frags[i]);
		remaining -= skb_frag_size(&info->frags[i]);

		if (remaining < 0)
			skb_frag_size_add(&info->frags[i], remaining);

		i++;
	}
	info->nr_frags = i;
done:
	ret = 0;
out:
	spin_unlock_irqrestore(&context->base.lock, flags);
	return ret;
}

static void mlx5e_tls_complete_sync_skb(struct sk_buff *skb,
					struct sk_buff *nskb, u32 tcp_seq,
					int headln, __be64 rcd_sn)
{
	struct mlx5e_tls_metadata *pet;
	u8 syndrome = SYNDROME_SYNC;
	struct iphdr *iph;
	struct tcphdr *th;
	int data_len, mss;

	nskb->dev = skb->dev;
	skb_reset_mac_header(nskb);
	skb_set_network_header(nskb, skb_network_offset(skb));
	skb_set_transport_header(nskb, skb_transport_offset(skb));
	memcpy(nskb->data, skb->data, headln);
	memcpy(nskb->data + headln, &rcd_sn, sizeof(rcd_sn));

	iph = ip_hdr(nskb);
	iph->tot_len = htons(nskb->len - skb_network_offset(nskb));
	th = tcp_hdr(nskb);
	data_len = nskb->len - headln;
	tcp_seq -= data_len;
	th->seq = htonl(tcp_seq);

	mss = nskb->dev->mtu - (headln - skb_network_offset(nskb));
	skb_shinfo(nskb)->gso_size = 0;
	if (data_len > mss) {
		skb_shinfo(nskb)->gso_size = mss;
		skb_shinfo(nskb)->gso_segs = DIV_ROUND_UP(data_len, mss);
	}
	skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type;

	pet = (struct mlx5e_tls_metadata *)(nskb->data + sizeof(struct ethhdr));
	memcpy(pet, &syndrome, sizeof(syndrome));
177
	pet->content.send.first_seq = htons(tcp_seq);
178
179
180
181
182
183
184
185
186
187

	/* MLX5 devices don't care about the checksum partial start, offset
	 * and pseudo header
	 */
	nskb->ip_summed = CHECKSUM_PARTIAL;

	nskb->queue_mapping = skb->queue_mapping;
}

static struct sk_buff *
188
mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context_tx *context,
189
190
		     struct mlx5e_txqsq *sq, struct sk_buff *skb,
		     struct mlx5e_tx_wqe **wqe,
191
192
		     u16 *pi,
		     struct mlx5e_tls *tls)
193
194
195
196
197
198
199
200
{
	u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
	struct sync_info info;
	struct sk_buff *nskb;
	int linear_len = 0;
	int headln;
	int i;

201
	sq->stats->tls_ooo++;
202

203
	if (mlx5e_tls_get_sync_data(context, tcp_seq, &info)) {
204
205
206
207
		/* We might get here if a retransmission reaches the driver
		 * after the relevant record is acked.
		 * It should be safe to drop the packet in this case
		 */
208
		atomic64_inc(&tls->sw_stats.tx_tls_drop_no_sync_data);
209
		goto err_out;
210
	}
211
212
213
214
215
216
217
218
219
220
221

	if (unlikely(info.sync_len < 0)) {
		u32 payload;

		headln = skb_transport_offset(skb) + tcp_hdrlen(skb);
		payload = skb->len - headln;
		if (likely(payload <= -info.sync_len))
			/* SKB payload doesn't require offload
			 */
			return skb;

222
		atomic64_inc(&tls->sw_stats.tx_tls_drop_bypass_required);
223
224
225
		goto err_out;
	}

226
227
	if (unlikely(mlx5e_tls_add_metadata(skb, context->swid))) {
		atomic64_inc(&tls->sw_stats.tx_tls_drop_metadata);
228
		goto err_out;
229
	}
230
231
232
233

	headln = skb_transport_offset(skb) + tcp_hdrlen(skb);
	linear_len += headln + sizeof(info.rcd_sn);
	nskb = alloc_skb(linear_len, GFP_ATOMIC);
234
235
	if (unlikely(!nskb)) {
		atomic64_inc(&tls->sw_stats.tx_tls_drop_resync_alloc);
236
		goto err_out;
237
	}
238
239
240
241
242
243
244
245
246

	context->expected_seq = tcp_seq + skb->len - headln;
	skb_put(nskb, linear_len);
	for (i = 0; i < info.nr_frags; i++)
		skb_shinfo(nskb)->frags[i] = info.frags[i];

	skb_shinfo(nskb)->nr_frags = info.nr_frags;
	nskb->data_len = info.sync_len;
	nskb->len += info.sync_len;
247
	sq->stats->tls_resync_bytes += nskb->len;
248
249
	mlx5e_tls_complete_sync_skb(skb, nskb, tcp_seq, headln,
				    cpu_to_be64(info.rcd_sn));
250
	mlx5e_sq_xmit(sq, nskb, *wqe, *pi, true);
251
	*wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi);
252
253
254
255
256
257
258
259
260
261
262
263
264
	return skb;

err_out:
	dev_kfree_skb_any(skb);
	return NULL;
}

struct sk_buff *mlx5e_tls_handle_tx_skb(struct net_device *netdev,
					struct mlx5e_txqsq *sq,
					struct sk_buff *skb,
					struct mlx5e_tx_wqe **wqe,
					u16 *pi)
{
265
	struct mlx5e_priv *priv = netdev_priv(netdev);
266
	struct mlx5e_tls_offload_context_tx *context;
267
268
269
270
271
	struct tls_context *tls_ctx;
	u32 expected_seq;
	int datalen;
	u32 skb_seq;

272
	if (MLX5_CAP_GEN(sq->channel->mdev, tls_tx)) {
273
274
275
276
		skb = mlx5e_ktls_handle_tx_skb(netdev, sq, skb, wqe, pi);
		goto out;
	}

277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
	if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
		goto out;

	datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb));
	if (!datalen)
		goto out;

	tls_ctx = tls_get_ctx(skb->sk);
	if (unlikely(tls_ctx->netdev != netdev))
		goto out;

	skb_seq = ntohl(tcp_hdr(skb)->seq);
	context = mlx5e_get_tls_tx_context(tls_ctx);
	expected_seq = context->expected_seq;

	if (unlikely(expected_seq != skb_seq)) {
293
		skb = mlx5e_tls_handle_ooo(context, sq, skb, wqe, pi, priv->tls);
294
295
296
297
		goto out;
	}

	if (unlikely(mlx5e_tls_add_metadata(skb, context->swid))) {
298
		atomic64_inc(&priv->tls->sw_stats.tx_tls_drop_metadata);
299
300
301
302
303
304
305
306
307
		dev_kfree_skb_any(skb);
		skb = NULL;
		goto out;
	}

	context->expected_seq = skb_seq + datalen;
out:
	return skb;
}
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

static int tls_update_resync_sn(struct net_device *netdev,
				struct sk_buff *skb,
				struct mlx5e_tls_metadata *mdata)
{
	struct sock *sk = NULL;
	struct iphdr *iph;
	struct tcphdr *th;
	__be32 seq;

	if (mdata->ethertype != htons(ETH_P_IP))
		return -EINVAL;

	iph = (struct iphdr *)(mdata + 1);

	th = ((void *)iph) + iph->ihl * 4;

	if (iph->version == 4) {
		sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,
					     iph->saddr, th->source, iph->daddr,
					     th->dest, netdev->ifindex);
#if IS_ENABLED(CONFIG_IPV6)
	} else {
		struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph;

		sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,
						&ipv6h->saddr, th->source,
335
						&ipv6h->daddr, ntohs(th->dest),
336
337
338
						netdev->ifindex, 0);
#endif
	}
339
340
341
342
	if (!sk || sk->sk_state == TCP_TIME_WAIT) {
		struct mlx5e_priv *priv = netdev_priv(netdev);

		atomic64_inc(&priv->tls->sw_stats.rx_tls_drop_resync_request);
343
		goto out;
344
	}
345
346
347
348
349
350
351
352
353
354
355
356
357
358

	skb->sk = sk;
	skb->destructor = sock_edemux;

	memcpy(&seq, &mdata->content.recv.sync_seq, sizeof(seq));
	tls_offload_rx_resync_request(sk, seq);
out:
	return 0;
}

void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb,
			     u32 *cqe_bcnt)
{
	struct mlx5e_tls_metadata *mdata;
359
	struct mlx5e_priv *priv;
360

361
	if (!is_metadata_hdr_valid(skb))
362
363
364
365
366
367
368
369
370
371
		return;

	/* Use the metadata */
	mdata = (struct mlx5e_tls_metadata *)(skb->data + ETH_HLEN);
	switch (mdata->content.recv.syndrome) {
	case SYNDROM_DECRYPTED:
		skb->decrypted = 1;
		break;
	case SYNDROM_RESYNC_REQUEST:
		tls_update_resync_sn(netdev, skb, mdata);
372
373
		priv = netdev_priv(netdev);
		atomic64_inc(&priv->tls->sw_stats.rx_tls_resync_request);
374
375
376
		break;
	case SYNDROM_AUTH_FAILED:
		/* Authentication failure will be observed and verified by kTLS */
377
378
		priv = netdev_priv(netdev);
		atomic64_inc(&priv->tls->sw_stats.rx_tls_auth_fail);
379
380
381
382
383
384
		break;
	default:
		/* Bypass the metadata header to others */
		return;
	}

385
	remove_metadata_hdr(skb);
386
387
	*cqe_bcnt -= MLX5E_METADATA_ETHER_LEN;
}