ib_isert.c 71.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
2
3
4
/*******************************************************************************
 * This file contains iSCSI extentions for RDMA (iSER) Verbs
 *
5
 * (c) Copyright 2013 Datera, Inc.
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 *
 * Nicholas A. Bellinger <nab@linux-iscsi.org>
 *
 ****************************************************************************/

#include <linux/string.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>
#include <target/target_core_base.h>
#include <target/target_core_fabric.h>
#include <target/iscsi/iscsi_transport.h>
22
#include <linux/semaphore.h>
23
24
25
26
27

#include "ib_isert.h"

#define	ISERT_MAX_CONN		8
#define ISER_MAX_RX_CQ_LEN	(ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
28
29
#define ISER_MAX_TX_CQ_LEN \
	((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
30
31
#define ISER_MAX_CQ_LEN		(ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
				 ISERT_MAX_CONN)
32

33
static int isert_debug_level;
34
35
36
module_param_named(debug_level, isert_debug_level, int, 0644);
MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");

37
38
39
static DEFINE_MUTEX(device_list_mutex);
static LIST_HEAD(device_list);
static struct workqueue_struct *isert_comp_wq;
40
static struct workqueue_struct *isert_release_wq;
41

42
43
static int
isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
44
static int
45
isert_login_post_recv(struct isert_conn *isert_conn);
46
47
static int
isert_rdma_accept(struct isert_conn *isert_conn);
48
struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
49

50
static void isert_release_work(struct work_struct *work);
51
52
53
54
static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
55

56
57
58
static inline bool
isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
{
59
	return (conn->pi_support &&
60
61
62
63
		cmd->prot_op != TARGET_PROT_NORMAL);
}


64
65
66
static void
isert_qp_event_callback(struct ib_event *e, void *context)
{
67
	struct isert_conn *isert_conn = context;
68

69
70
71
	isert_err("%s (%d): conn %p\n",
		  ib_event_msg(e->event), e->event, isert_conn);

72
73
	switch (e->event) {
	case IB_EVENT_COMM_EST:
74
		rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
75
76
		break;
	case IB_EVENT_QP_LAST_WQE_REACHED:
77
		isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
78
79
80
81
82
83
		break;
	default:
		break;
	}
}

84
85
static struct isert_comp *
isert_comp_get(struct isert_conn *isert_conn)
86
{
87
	struct isert_device *device = isert_conn->device;
88
	struct isert_comp *comp;
89
	int i, min = 0;
90
91

	mutex_lock(&device_list_mutex);
92
93
94
95
96
97
	for (i = 0; i < device->comps_used; i++)
		if (device->comps[i].active_qps <
		    device->comps[min].active_qps)
			min = i;
	comp = &device->comps[min];
	comp->active_qps++;
98
99
	mutex_unlock(&device_list_mutex);

100
	isert_info("conn %p, using comp %p min_index: %d\n",
101
		   isert_conn, comp, min);
102
103
104
105
106
107
108
109
110

	return comp;
}

static void
isert_comp_put(struct isert_comp *comp)
{
	mutex_lock(&device_list_mutex);
	comp->active_qps--;
111
	mutex_unlock(&device_list_mutex);
112
113
114
115
116
117
118
}

static struct ib_qp *
isert_create_qp(struct isert_conn *isert_conn,
		struct isert_comp *comp,
		struct rdma_cm_id *cma_id)
{
119
	struct isert_device *device = isert_conn->device;
120
121
	struct ib_qp_init_attr attr;
	int ret;
122
123
124
125

	memset(&attr, 0, sizeof(struct ib_qp_init_attr));
	attr.event_handler = isert_qp_event_callback;
	attr.qp_context = isert_conn;
126
127
	attr.send_cq = comp->cq;
	attr.recv_cq = comp->cq;
Sagi Grimberg's avatar
Sagi Grimberg committed
128
	attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
129
	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
130
	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
131
	attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
132
133
134
	attr.cap.max_recv_sge = 1;
	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
	attr.qp_type = IB_QPT_RC;
135
	if (device->pi_capable)
136
		attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
137

138
	ret = rdma_create_qp(cma_id, device->pd, &attr);
139
	if (ret) {
140
		isert_err("rdma_create_qp failed for cma_id %d\n", ret);
141
142
143
144
145
146
147
148
149
150
151
152
153
		return ERR_PTR(ret);
	}

	return cma_id->qp;
}

static int
isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
{
	struct isert_comp *comp;
	int ret;

	comp = isert_comp_get(isert_conn);
154
155
156
	isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
	if (IS_ERR(isert_conn->qp)) {
		ret = PTR_ERR(isert_conn->qp);
157
		goto err;
158
159
160
	}

	return 0;
161
err:
162
	isert_comp_put(comp);
163
	return ret;
164
165
166
167
168
}

static int
isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
{
169
	struct isert_device *device = isert_conn->device;
170
	struct ib_device *ib_dev = device->ib_device;
171
172
173
174
175
	struct iser_rx_desc *rx_desc;
	struct ib_sge *rx_sg;
	u64 dma_addr;
	int i, j;

Kees Cook's avatar
Kees Cook committed
176
177
178
	isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS,
				       sizeof(struct iser_rx_desc),
				       GFP_KERNEL);
179
	if (!isert_conn->rx_descs)
180
		return -ENOMEM;
181

182
	rx_desc = isert_conn->rx_descs;
183
184
185
186
187
188
189
190
191
192
193
194

	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
		dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
					ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
		if (ib_dma_mapping_error(ib_dev, dma_addr))
			goto dma_map_fail;

		rx_desc->dma_addr = dma_addr;

		rx_sg = &rx_desc->rx_sg;
		rx_sg->addr = rx_desc->dma_addr;
		rx_sg->length = ISER_RX_PAYLOAD_SIZE;
195
		rx_sg->lkey = device->pd->local_dma_lkey;
196
		rx_desc->rx_cqe.done = isert_recv_done;
197
198
199
200
201
	}

	return 0;

dma_map_fail:
202
	rx_desc = isert_conn->rx_descs;
203
204
205
206
	for (j = 0; j < i; j++, rx_desc++) {
		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	}
207
208
	kfree(isert_conn->rx_descs);
	isert_conn->rx_descs = NULL;
209
	isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
210
211
212
213
214
215
	return -ENOMEM;
}

static void
isert_free_rx_descriptors(struct isert_conn *isert_conn)
{
216
	struct ib_device *ib_dev = isert_conn->device->ib_device;
217
218
219
	struct iser_rx_desc *rx_desc;
	int i;

220
	if (!isert_conn->rx_descs)
221
222
		return;

223
	rx_desc = isert_conn->rx_descs;
224
225
226
227
228
	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	}

229
230
	kfree(isert_conn->rx_descs);
	isert_conn->rx_descs = NULL;
231
232
}

233
234
static void
isert_free_comps(struct isert_device *device)
235
{
236
	int i;
237

238
239
	for (i = 0; i < device->comps_used; i++) {
		struct isert_comp *comp = &device->comps[i];
240

241
242
		if (comp->cq)
			ib_free_cq(comp->cq);
243
	}
244
245
	kfree(device->comps);
}
246

247
static int
248
isert_alloc_comps(struct isert_device *device)
249
250
{
	int i, max_cqe, ret = 0;
251

252
	device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
253
254
				 device->ib_device->num_comp_vectors));

255
	isert_info("Using %d CQs, %s supports %d vectors support "
256
		   "pi_capable %d\n",
257
		   device->comps_used, dev_name(&device->ib_device->dev),
258
		   device->ib_device->num_comp_vectors,
259
260
261
262
		   device->pi_capable);

	device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
				GFP_KERNEL);
263
	if (!device->comps)
264
		return -ENOMEM;
265

266
	max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
267

268
269
270
271
	for (i = 0; i < device->comps_used; i++) {
		struct isert_comp *comp = &device->comps[i];

		comp->device = device;
272
273
		comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
				IB_POLL_WORKQUEUE);
274
		if (IS_ERR(comp->cq)) {
275
			isert_err("Unable to allocate cq\n");
276
277
			ret = PTR_ERR(comp->cq);
			comp->cq = NULL;
278
			goto out_cq;
279
		}
280
281
	}

282
283
284
285
286
287
288
289
290
	return 0;
out_cq:
	isert_free_comps(device);
	return ret;
}

static int
isert_create_device_ib_res(struct isert_device *device)
{
291
	struct ib_device *ib_dev = device->ib_device;
292
	int ret;
293

294
295
	isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
		  ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
296
	isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
297

298
	ret = isert_alloc_comps(device);
299
	if (ret)
300
		goto out;
301

302
	device->pd = ib_alloc_pd(ib_dev, 0);
303
304
305
306
307
308
309
	if (IS_ERR(device->pd)) {
		ret = PTR_ERR(device->pd);
		isert_err("failed to allocate pd, device %p, ret=%d\n",
			  device, ret);
		goto out_cq;
	}

310
	/* Check signature cap */
311
	device->pi_capable = ib_dev->attrs.device_cap_flags &
312
			     IB_DEVICE_INTEGRITY_HANDOVER ? true : false;
313

314
315
316
	return 0;

out_cq:
317
	isert_free_comps(device);
318
319
320
out:
	if (ret > 0)
		ret = -EINVAL;
321
322
323
324
325
326
	return ret;
}

static void
isert_free_device_ib_res(struct isert_device *device)
{
327
	isert_info("device %p\n", device);
328

329
	ib_dealloc_pd(device->pd);
330
	isert_free_comps(device);
331
332
333
}

static void
334
isert_device_put(struct isert_device *device)
335
336
337
{
	mutex_lock(&device_list_mutex);
	device->refcount--;
338
	isert_info("device %p refcount %d\n", device, device->refcount);
339
340
341
342
343
344
345
346
347
	if (!device->refcount) {
		isert_free_device_ib_res(device);
		list_del(&device->dev_node);
		kfree(device);
	}
	mutex_unlock(&device_list_mutex);
}

static struct isert_device *
348
isert_device_get(struct rdma_cm_id *cma_id)
349
350
351
352
353
354
355
356
{
	struct isert_device *device;
	int ret;

	mutex_lock(&device_list_mutex);
	list_for_each_entry(device, &device_list, dev_node) {
		if (device->ib_device->node_guid == cma_id->device->node_guid) {
			device->refcount++;
357
358
			isert_info("Found iser device %p refcount %d\n",
				   device, device->refcount);
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
			mutex_unlock(&device_list_mutex);
			return device;
		}
	}

	device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
	if (!device) {
		mutex_unlock(&device_list_mutex);
		return ERR_PTR(-ENOMEM);
	}

	INIT_LIST_HEAD(&device->dev_node);

	device->ib_device = cma_id->device;
	ret = isert_create_device_ib_res(device);
	if (ret) {
		kfree(device);
		mutex_unlock(&device_list_mutex);
		return ERR_PTR(ret);
	}

	device->refcount++;
	list_add_tail(&device->dev_node, &device_list);
382
383
	isert_info("Created a new iser device %p refcount %d\n",
		   device, device->refcount);
384
385
386
387
388
	mutex_unlock(&device_list_mutex);

	return device;
}

389
390
static void
isert_init_conn(struct isert_conn *isert_conn)
391
392
{
	isert_conn->state = ISER_CONN_INIT;
393
	INIT_LIST_HEAD(&isert_conn->node);
394
	init_completion(&isert_conn->login_comp);
395
	init_completion(&isert_conn->login_req_comp);
396
	init_waitqueue_head(&isert_conn->rem_wait);
397
398
	kref_init(&isert_conn->kref);
	mutex_init(&isert_conn->mutex);
399
	INIT_WORK(&isert_conn->release_work, isert_release_work);
400
}
401

402
403
404
static void
isert_free_login_buf(struct isert_conn *isert_conn)
{
405
	struct ib_device *ib_dev = isert_conn->device->ib_device;
406
407

	ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
408
			    ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
409
410
	kfree(isert_conn->login_rsp_buf);

411
	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
412
			    ISER_RX_PAYLOAD_SIZE,
413
			    DMA_FROM_DEVICE);
414
	kfree(isert_conn->login_req_buf);
415
416
417
418
419
420
421
}

static int
isert_alloc_login_buf(struct isert_conn *isert_conn,
		      struct ib_device *ib_dev)
{
	int ret;
422

423
424
	isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf),
			GFP_KERNEL);
425
	if (!isert_conn->login_req_buf)
426
		return -ENOMEM;
427
428

	isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
429
430
				isert_conn->login_req_buf,
				ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
431
432
	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
	if (ret) {
433
		isert_err("login_req_dma mapping error: %d\n", ret);
434
		isert_conn->login_req_dma = 0;
435
436
437
438
439
		goto out_free_login_req_buf;
	}

	isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
	if (!isert_conn->login_rsp_buf) {
440
		ret = -ENOMEM;
441
		goto out_unmap_login_req_buf;
442
443
444
	}

	isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
445
					isert_conn->login_rsp_buf,
446
					ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
447
448
	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
	if (ret) {
449
		isert_err("login_rsp_dma mapping error: %d\n", ret);
450
		isert_conn->login_rsp_dma = 0;
451
		goto out_free_login_rsp_buf;
452
453
	}

454
455
	return 0;

456
457
458
out_free_login_rsp_buf:
	kfree(isert_conn->login_rsp_buf);
out_unmap_login_req_buf:
459
	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
460
461
462
			    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
out_free_login_req_buf:
	kfree(isert_conn->login_req_buf);
463
464
465
	return ret;
}

466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
static void
isert_set_nego_params(struct isert_conn *isert_conn,
		      struct rdma_conn_param *param)
{
	struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;

	/* Set max inflight RDMA READ requests */
	isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
				attr->max_qp_init_rd_atom);
	isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);

	if (param->private_data) {
		u8 flags = *(u8 *)param->private_data;

		/*
		 * use remote invalidation if the both initiator
		 * and the HCA support it
		 */
		isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
					  (attr->device_cap_flags &
					   IB_DEVICE_MEM_MGT_EXTENSIONS);
		if (isert_conn->snd_w_inv)
			isert_info("Using remote invalidation\n");
	}
}

492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
static int
isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
	struct isert_np *isert_np = cma_id->context;
	struct iscsi_np *np = isert_np->np;
	struct isert_conn *isert_conn;
	struct isert_device *device;
	int ret = 0;

	spin_lock_bh(&np->np_thread_lock);
	if (!np->enabled) {
		spin_unlock_bh(&np->np_thread_lock);
		isert_dbg("iscsi_np is not enabled, reject connect request\n");
		return rdma_reject(cma_id, NULL, 0);
	}
	spin_unlock_bh(&np->np_thread_lock);

	isert_dbg("cma_id: %p, portal: %p\n",
		 cma_id, cma_id->context);

	isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
	if (!isert_conn)
		return -ENOMEM;

	isert_init_conn(isert_conn);
517
	isert_conn->cm_id = cma_id;
518
519
520
521
522

	ret = isert_alloc_login_buf(isert_conn, cma_id->device);
	if (ret)
		goto out;

523
	device = isert_device_get(cma_id);
524
525
526
527
	if (IS_ERR(device)) {
		ret = PTR_ERR(device);
		goto out_rsp_dma_map;
	}
528
	isert_conn->device = device;
529

530
	isert_set_nego_params(isert_conn, &event->param.conn);
531

532
	ret = isert_conn_setup_qp(isert_conn, cma_id);
533
534
535
	if (ret)
		goto out_conn_dev;

536
	ret = isert_login_post_recv(isert_conn);
537
538
539
540
541
542
543
	if (ret)
		goto out_conn_dev;

	ret = isert_rdma_accept(isert_conn);
	if (ret)
		goto out_conn_dev;

544
	mutex_lock(&isert_np->mutex);
545
	list_add_tail(&isert_conn->node, &isert_np->accepted);
546
	mutex_unlock(&isert_np->mutex);
547
548
549
550

	return 0;

out_conn_dev:
551
	isert_device_put(device);
552
out_rsp_dma_map:
553
	isert_free_login_buf(isert_conn);
554
555
out:
	kfree(isert_conn);
556
	rdma_reject(cma_id, NULL, 0);
557
558
559
560
561
562
	return ret;
}

static void
isert_connect_release(struct isert_conn *isert_conn)
{
563
	struct isert_device *device = isert_conn->device;
564

565
	isert_dbg("conn %p\n", isert_conn);
566

567
568
	BUG_ON(!device);

569
	isert_free_rx_descriptors(isert_conn);
570
571
	if (isert_conn->cm_id &&
	    !isert_conn->dev_removed)
572
		rdma_destroy_id(isert_conn->cm_id);
573

574
575
	if (isert_conn->qp) {
		struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
576

577
		isert_comp_put(comp);
578
		ib_destroy_qp(isert_conn->qp);
579
580
	}

581
	if (isert_conn->login_req_buf)
582
583
		isert_free_login_buf(isert_conn);

584
	isert_device_put(device);
585

586
587
588
589
	if (isert_conn->dev_removed)
		wake_up_interruptible(&isert_conn->rem_wait);
	else
		kfree(isert_conn);
590
591
592
593
594
}

static void
isert_connected_handler(struct rdma_cm_id *cma_id)
{
595
	struct isert_conn *isert_conn = cma_id->qp->qp_context;
596
	struct isert_np *isert_np = cma_id->context;
597

598
	isert_info("conn %p\n", isert_conn);
599

600
	mutex_lock(&isert_conn->mutex);
601
602
	isert_conn->state = ISER_CONN_UP;
	kref_get(&isert_conn->kref);
603
	mutex_unlock(&isert_conn->mutex);
604
605
606
607
608
609
610

	mutex_lock(&isert_np->mutex);
	list_move_tail(&isert_conn->node, &isert_np->pending);
	mutex_unlock(&isert_np->mutex);

	isert_info("np %p: Allow accept_np to continue\n", isert_np);
	up(&isert_np->sem);
611
612
613
}

static void
614
isert_release_kref(struct kref *kref)
615
616
{
	struct isert_conn *isert_conn = container_of(kref,
617
				struct isert_conn, kref);
618

619
620
	isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
		   current->pid);
621
622
623
624
625
626
627

	isert_connect_release(isert_conn);
}

static void
isert_put_conn(struct isert_conn *isert_conn)
{
628
	kref_put(&isert_conn->kref, isert_release_kref);
629
630
}

631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
static void
isert_handle_unbound_conn(struct isert_conn *isert_conn)
{
	struct isert_np *isert_np = isert_conn->cm_id->context;

	mutex_lock(&isert_np->mutex);
	if (!list_empty(&isert_conn->node)) {
		/*
		 * This means iscsi doesn't know this connection
		 * so schedule a cleanup ourselves
		 */
		list_del_init(&isert_conn->node);
		isert_put_conn(isert_conn);
		queue_work(isert_release_wq, &isert_conn->release_work);
	}
	mutex_unlock(&isert_np->mutex);
}

649
650
651
652
653
/**
 * isert_conn_terminate() - Initiate connection termination
 * @isert_conn: isert connection struct
 *
 * Notes:
654
 * In case the connection state is BOUND, move state
655
 * to TEMINATING and start teardown sequence (rdma_disconnect).
656
 * In case the connection state is UP, complete flush as well.
657
 *
658
 * This routine must be called with mutex held. Thus it is
659
660
661
662
663
664
665
 * safe to call multiple times.
 */
static void
isert_conn_terminate(struct isert_conn *isert_conn)
{
	int err;

666
667
668
669
670
671
672
673
674
675
	if (isert_conn->state >= ISER_CONN_TERMINATING)
		return;

	isert_info("Terminating conn %p state %d\n",
		   isert_conn, isert_conn->state);
	isert_conn->state = ISER_CONN_TERMINATING;
	err = rdma_disconnect(isert_conn->cm_id);
	if (err)
		isert_warn("Failed rdma_disconnect isert_conn %p\n",
			   isert_conn);
676
677
}

678
static int
679
680
isert_np_cma_handler(struct isert_np *isert_np,
		     enum rdma_cm_event_type event)
681
{
682
683
	isert_dbg("%s (%d): isert np %p\n",
		  rdma_event_msg(event), event, isert_np);
684

685
686
	switch (event) {
	case RDMA_CM_EVENT_DEVICE_REMOVAL:
687
		isert_np->cm_id = NULL;
688
689
		break;
	case RDMA_CM_EVENT_ADDR_CHANGE:
690
691
		isert_np->cm_id = isert_setup_id(isert_np);
		if (IS_ERR(isert_np->cm_id)) {
692
			isert_err("isert np %p setup id failed: %ld\n",
693
694
				  isert_np, PTR_ERR(isert_np->cm_id));
			isert_np->cm_id = NULL;
695
696
697
		}
		break;
	default:
698
		isert_err("isert np %p Unexpected event %d\n",
699
			  isert_np, event);
700
701
	}

702
703
704
705
706
707
708
	return -1;
}

static int
isert_disconnected_handler(struct rdma_cm_id *cma_id,
			   enum rdma_cm_event_type event)
{
709
	struct isert_conn *isert_conn = cma_id->qp->qp_context;
710

711
	mutex_lock(&isert_conn->mutex);
712
713
714
715
716
	switch (isert_conn->state) {
	case ISER_CONN_TERMINATING:
		break;
	case ISER_CONN_UP:
		isert_conn_terminate(isert_conn);
Sagi Grimberg's avatar
Sagi Grimberg committed
717
		ib_drain_qp(isert_conn->qp);
718
719
720
721
722
723
724
		isert_handle_unbound_conn(isert_conn);
		break;
	case ISER_CONN_BOUND:
	case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
		iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		break;
	default:
725
		isert_warn("conn %p terminating in state %d\n",
726
			   isert_conn, isert_conn->state);
727
	}
728
	mutex_unlock(&isert_conn->mutex);
729

730
	return 0;
731
732
}

733
static int
734
735
isert_connect_error(struct rdma_cm_id *cma_id)
{
736
	struct isert_conn *isert_conn = cma_id->qp->qp_context;
737

738
	ib_drain_qp(isert_conn->qp);
739
	list_del_init(&isert_conn->node);
740
	isert_conn->cm_id = NULL;
741
	isert_put_conn(isert_conn);
742
743

	return -1;
744
745
}

746
747
748
static int
isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
749
	struct isert_np *isert_np = cma_id->context;
750
	struct isert_conn *isert_conn;
751
752
	int ret = 0;

753
754
	isert_info("%s (%d): status %d id %p np %p\n",
		   rdma_event_msg(event->event), event->event,
755
		   event->status, cma_id, cma_id->context);
756

757
758
759
	if (isert_np->cm_id == cma_id)
		return isert_np_cma_handler(cma_id->context, event->event);

760
761
762
	switch (event->event) {
	case RDMA_CM_EVENT_CONNECT_REQUEST:
		ret = isert_connect_request(cma_id, event);
763
		if (ret)
764
			isert_err("failed handle connect request %d\n", ret);
765
766
767
768
		break;
	case RDMA_CM_EVENT_ESTABLISHED:
		isert_connected_handler(cma_id);
		break;
769
770
771
	case RDMA_CM_EVENT_ADDR_CHANGE:    /* FALLTHRU */
	case RDMA_CM_EVENT_DISCONNECTED:   /* FALLTHRU */
	case RDMA_CM_EVENT_TIMEWAIT_EXIT:  /* FALLTHRU */
772
		ret = isert_disconnected_handler(cma_id, event->event);
773
		break;
774
775
776
777
778
779
780
781
782
783
784
785
	case RDMA_CM_EVENT_DEVICE_REMOVAL:
		isert_conn = cma_id->qp->qp_context;
		isert_conn->dev_removed = true;
		isert_disconnected_handler(cma_id, event->event);
		wait_event_interruptible(isert_conn->rem_wait,
					 isert_conn->state == ISER_CONN_DOWN);
		kfree(isert_conn);
		/*
		 * return non-zero from the callback to destroy
		 * the rdma cm id
		 */
		return 1;
786
	case RDMA_CM_EVENT_REJECTED:
787
788
		isert_info("Connection rejected: %s\n",
			   rdma_reject_msg(cma_id, event->status));
789
790
		/* fall through */
	case RDMA_CM_EVENT_UNREACHABLE:
791
	case RDMA_CM_EVENT_CONNECT_ERROR:
792
		ret = isert_connect_error(cma_id);
793
		break;
794
	default:
795
		isert_err("Unhandled RDMA CMA event: %d\n", event->event);
796
797
798
799
800
801
802
		break;
	}

	return ret;
}

static int
803
isert_post_recvm(struct isert_conn *isert_conn, u32 count)
804
{
805
	struct ib_recv_wr *rx_wr;
806
807
808
	int i, ret;
	struct iser_rx_desc *rx_desc;

809
	for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
810
		rx_desc = &isert_conn->rx_descs[i];
811
812

		rx_wr->wr_cqe = &rx_desc->rx_cqe;
813
814
815
		rx_wr->sg_list = &rx_desc->rx_sg;
		rx_wr->num_sge = 1;
		rx_wr->next = rx_wr + 1;
816
		rx_desc->in_use = false;
817
818
819
820
	}
	rx_wr--;
	rx_wr->next = NULL; /* mark end of work requests list */

821
	ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL);
822
	if (ret)
823
		isert_err("ib_post_recv() failed with ret: %d\n", ret);
824
825
826
827
828
829
830

	return ret;
}

static int
isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
{
831
	struct ib_recv_wr rx_wr;
832
833
	int ret;

834
835
836
837
838
839
840
841
842
	if (!rx_desc->in_use) {
		/*
		 * if the descriptor is not in-use we already reposted it
		 * for recv, so just silently return
		 */
		return 0;
	}

	rx_desc->in_use = false;
843
	rx_wr.wr_cqe = &rx_desc->rx_cqe;
844
845
846
847
	rx_wr.sg_list = &rx_desc->rx_sg;
	rx_wr.num_sge = 1;
	rx_wr.next = NULL;

848
	ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
849
	if (ret)
850
851
		isert_err("ib_post_recv() failed with ret: %d\n", ret);

852
853
854
855
	return ret;
}

static int
856
isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
857
{
858
	struct ib_device *ib_dev = isert_conn->cm_id->device;
859
	struct ib_send_wr send_wr;
860
861
862
863
864
	int ret;

	ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
				      ISER_HEADERS_LEN, DMA_TO_DEVICE);

865
866
	tx_desc->tx_cqe.done = isert_login_send_done;

867
	send_wr.next	= NULL;
868
	send_wr.wr_cqe	= &tx_desc->tx_cqe;
869
870
871
872
873
	send_wr.sg_list	= tx_desc->tx_sg;
	send_wr.num_sge	= tx_desc->num_sge;
	send_wr.opcode	= IB_WR_SEND;
	send_wr.send_flags = IB_SEND_SIGNALED;

874
	ret = ib_post_send(isert_conn->qp, &send_wr, NULL);
875
	if (ret)
876
		isert_err("ib_post_send() failed, ret: %d\n", ret);
877
878
879
880
881

	return ret;
}

static void
882
883
__isert_create_send_desc(struct isert_device *device,
			 struct iser_tx_desc *tx_desc)
884
885
{

886
887
	memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
	tx_desc->iser_header.flags = ISCSI_CTRL;
888
889
890

	tx_desc->num_sge = 1;

891
892
	if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
		tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
893
		isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
894
895
896
	}
}

897
898
899
900
901
902
903
904
905
906
907
908
909
910
static void
isert_create_send_desc(struct isert_conn *isert_conn,
		       struct isert_cmd *isert_cmd,
		       struct iser_tx_desc *tx_desc)
{
	struct isert_device *device = isert_conn->device;
	struct ib_device *ib_dev = device->ib_device;

	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
				   ISER_HEADERS_LEN, DMA_TO_DEVICE);

	__isert_create_send_desc(device, tx_desc);
}

911
912
913
914
static int
isert_init_tx_hdrs(struct isert_conn *isert_conn,
		   struct iser_tx_desc *tx_desc)
{
915
	struct isert_device *device = isert_conn->device;
916
	struct ib_device *ib_dev = device->ib_device;
917
918
919
920
921
	u64 dma_addr;

	dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
			ISER_HEADERS_LEN, DMA_TO_DEVICE);
	if (ib_dma_mapping_error(ib_dev, dma_addr)) {
922
		isert_err("ib_dma_mapping_error() failed\n");
923
924
925
926
927
928
		return -ENOMEM;
	}

	tx_desc->dma_addr = dma_addr;
	tx_desc->tx_sg[0].addr	= tx_desc->dma_addr;
	tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
929
	tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
930

931
932
933
	isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
		  tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
		  tx_desc->tx_sg[0].lkey);
934
935
936
937
938

	return 0;
}

static void
939
isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
940
		   struct ib_send_wr *send_wr)
941
{
942
943
	struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;

944
945
	tx_desc->tx_cqe.done = isert_send_done;
	send_wr->wr_cqe = &tx_desc->tx_cqe;
946
947
948
949
950
951
952
953

	if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
		send_wr->opcode  = IB_WR_SEND_WITH_INV;
		send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
	} else {
		send_wr->opcode = IB_WR_SEND;
	}

954
	send_wr->sg_list = &tx_desc->tx_sg[0];
955
	send_wr->num_sge = isert_cmd->tx_desc.num_sge;
956
	send_wr->send_flags = IB_SEND_SIGNALED;
957
958
959
}

static int
960
isert_login_post_recv(struct isert_conn *isert_conn)
961
{
962
	struct ib_recv_wr rx_wr;
963
964
965
966
967
	struct ib_sge sge;
	int ret;

	memset(&sge, 0, sizeof(struct ib_sge));
	sge.addr = isert_conn->login_req_dma;
968
	sge.length = ISER_RX_PAYLOAD_SIZE;
969
	sge.lkey = isert_conn->device->pd->local_dma_lkey;
970

971
	isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
972
973
		sge.addr, sge.length, sge.lkey);

974
975
	isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done;

976
	memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
977
	rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe;
978
979
980
	rx_wr.sg_list = &sge;
	rx_wr.num_sge = 1;

981
	ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
982
	if (ret)
983
		isert_err("ib_post_recv() failed: %d\n", ret);
984
985
986
987
988
989
990
991
992

	return ret;
}

static int
isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
		   u32 length)
{
	struct isert_conn *isert_conn = conn->context;
993
	struct isert_device *device = isert_conn->device;
994
	struct ib_device *ib_dev = device->ib_device;
Sagi Grimberg's avatar <