ib_cm.c 34.4 KB
Newer Older
Andy Grover's avatar
Andy Grover committed
1
/*
2
 * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
Andy Grover's avatar
Andy Grover committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/in.h>
35
#include <linux/slab.h>
Andy Grover's avatar
Andy Grover committed
36
#include <linux/vmalloc.h>
37
#include <linux/ratelimit.h>
38
#include <net/addrconf.h>
Andy Grover's avatar
Andy Grover committed
39

40
#include "rds_single_path.h"
Andy Grover's avatar
Andy Grover committed
41
42
#include "rds.h"
#include "ib.h"
43
#include "ib_mr.h"
Andy Grover's avatar
Andy Grover committed
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

/*
 * Set the selected protocol version
 */
static void rds_ib_set_protocol(struct rds_connection *conn, unsigned int version)
{
	conn->c_version = version;
}

/*
 * Set up flow control
 */
static void rds_ib_set_flow_control(struct rds_connection *conn, u32 credits)
{
	struct rds_ib_connection *ic = conn->c_transport_data;

	if (rds_ib_sysctl_flow_control && credits != 0) {
		/* We're doing flow control */
		ic->i_flowctl = 1;
		rds_ib_send_add_credits(conn, credits);
	} else {
		ic->i_flowctl = 0;
	}
}

/*
 * Tune RNR behavior. Without flow control, we use a rather
 * low timeout, but not the absolute minimum - this should
 * be tunable.
 *
 * We already set the RNR retry count to 7 (which is the
 * smallest infinite number :-) above.
 * If flow control is off, we want to change this back to 0
 * so that we learn quickly when our credit accounting is
 * buggy.
 *
 * Caller passes in a qp_attr pointer - don't waste stack spacv
 * by allocation this twice.
 */
static void
rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)
{
	int ret;

	attr->min_rnr_timer = IB_RNR_TIMER_000_32;
	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
	if (ret)
		printk(KERN_NOTICE "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret);
}

/*
 * Connection established.
 * We get here for both outgoing and incoming connection.
 */
void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event)
{
	struct rds_ib_connection *ic = conn->c_transport_data;
101
	const union rds_ib_conn_priv *dp = NULL;
Andy Grover's avatar
Andy Grover committed
102
	struct ib_qp_attr qp_attr;
103
104
105
106
	__be64 ack_seq = 0;
	__be32 credit = 0;
	u8 major = 0;
	u8 minor = 0;
Andy Grover's avatar
Andy Grover committed
107
108
	int err;

109
110
111
112
113
114
115
116
117
118
119
120
121
	dp = event->param.conn.private_data;
	if (conn->c_isv6) {
		if (event->param.conn.private_data_len >=
		    sizeof(struct rds6_ib_connect_private)) {
			major = dp->ricp_v6.dp_protocol_major;
			minor = dp->ricp_v6.dp_protocol_minor;
			credit = dp->ricp_v6.dp_credit;
			/* dp structure start is not guaranteed to be 8 bytes
			 * aligned.  Since dp_ack_seq is 64-bit extended load
			 * operations can be used so go through get_unaligned
			 * to avoid unaligned errors.
			 */
			ack_seq = get_unaligned(&dp->ricp_v6.dp_ack_seq);
122
		}
123
124
125
126
127
128
129
130
131
132
133
134
	} else if (event->param.conn.private_data_len >=
		   sizeof(struct rds_ib_connect_private)) {
		major = dp->ricp_v4.dp_protocol_major;
		minor = dp->ricp_v4.dp_protocol_minor;
		credit = dp->ricp_v4.dp_credit;
		ack_seq = get_unaligned(&dp->ricp_v4.dp_ack_seq);
	}

	/* make sure it isn't empty data */
	if (major) {
		rds_ib_set_protocol(conn, RDS_PROTOCOL(major, minor));
		rds_ib_set_flow_control(conn, be32_to_cpu(credit));
Andy Grover's avatar
Andy Grover committed
135
136
	}

137
138
139
140
141
142
143
144
145
	if (conn->c_version < RDS_PROTOCOL_VERSION) {
		if (conn->c_version != RDS_PROTOCOL_COMPAT_VERSION) {
			pr_notice("RDS/IB: Connection <%pI6c,%pI6c> version %u.%u no longer supported\n",
				  &conn->c_laddr, &conn->c_faddr,
				  RDS_PROTOCOL_MAJOR(conn->c_version),
				  RDS_PROTOCOL_MINOR(conn->c_version));
			rds_conn_destroy(conn);
			return;
		}
146
	}
Andy Grover's avatar
Andy Grover committed
147

148
	pr_notice("RDS/IB: %s conn connected <%pI6c,%pI6c,%d> version %u.%u%s\n",
149
		  ic->i_active_side ? "Active" : "Passive",
150
		  &conn->c_laddr, &conn->c_faddr, conn->c_tos,
151
152
153
154
		  RDS_PROTOCOL_MAJOR(conn->c_version),
		  RDS_PROTOCOL_MINOR(conn->c_version),
		  ic->i_flowctl ? ", flow control" : "");

155
156
157
	/* receive sl from the peer */
	ic->i_sl = ic->i_cm_id->route.path_rec->sl;

158
159
	atomic_set(&ic->i_cq_quiesce, 0);

160
161
162
	/* Init rings and fill recv. this needs to wait until protocol
	 * negotiation is complete, since ring layout is different
	 * from 3.1 to 4.1.
163
164
165
166
167
	 */
	rds_ib_send_init_ring(ic);
	rds_ib_recv_init_ring(ic);
	/* Post receive buffers - as a side effect, this will update
	 * the posted credit count. */
168
	rds_ib_recv_refill(conn, 1, GFP_KERNEL);
169

Andy Grover's avatar
Andy Grover committed
170
171
172
173
174
175
176
177
	/* Tune RNR behavior */
	rds_ib_tune_rnr(ic, &qp_attr);

	qp_attr.qp_state = IB_QPS_RTS;
	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
	if (err)
		printk(KERN_NOTICE "ib_modify_qp(IB_QP_STATE, RTS): err=%d\n", err);

178
	/* update ib_device with this local ipaddr */
179
	err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);
Andy Grover's avatar
Andy Grover committed
180
	if (err)
181
182
		printk(KERN_ERR "rds_ib_update_ipaddr failed (%d)\n",
			err);
Andy Grover's avatar
Andy Grover committed
183
184
185

	/* If the peer gave us the last packet it saw, process this as if
	 * we had received a regular ACK. */
186
	if (dp) {
187
188
		if (ack_seq)
			rds_send_drop_acked(conn, be64_to_cpu(ack_seq),
189
190
					    NULL);
	}
Andy Grover's avatar
Andy Grover committed
191

192
	conn->c_proposed_version = conn->c_version;
Andy Grover's avatar
Andy Grover committed
193
194
195
196
	rds_connect_complete(conn);
}

static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
197
198
199
200
201
202
				      struct rdma_conn_param *conn_param,
				      union rds_ib_conn_priv *dp,
				      u32 protocol_version,
				      u32 max_responder_resources,
				      u32 max_initiator_depth,
				      bool isv6)
Andy Grover's avatar
Andy Grover committed
203
{
204
	struct rds_ib_connection *ic = conn->c_transport_data;
205
	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
206

Andy Grover's avatar
Andy Grover committed
207
	memset(conn_param, 0, sizeof(struct rdma_conn_param));
208
209
210
211
212

	conn_param->responder_resources =
		min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
	conn_param->initiator_depth =
		min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
213
	conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
Andy Grover's avatar
Andy Grover committed
214
215
216
217
	conn_param->rnr_retry_count = 7;

	if (dp) {
		memset(dp, 0, sizeof(*dp));
218
219
220
221
222
223
224
225
226
227
228
		if (isv6) {
			dp->ricp_v6.dp_saddr = conn->c_laddr;
			dp->ricp_v6.dp_daddr = conn->c_faddr;
			dp->ricp_v6.dp_protocol_major =
			    RDS_PROTOCOL_MAJOR(protocol_version);
			dp->ricp_v6.dp_protocol_minor =
			    RDS_PROTOCOL_MINOR(protocol_version);
			dp->ricp_v6.dp_protocol_minor_mask =
			    cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
			dp->ricp_v6.dp_ack_seq =
			    cpu_to_be64(rds_ib_piggyb_ack(ic));
229
			dp->ricp_v6.dp_cmn.ricpc_dp_toss = conn->c_tos;
230
231
232
233
234
235
236
237
238
239
240
241
242
243

			conn_param->private_data = &dp->ricp_v6;
			conn_param->private_data_len = sizeof(dp->ricp_v6);
		} else {
			dp->ricp_v4.dp_saddr = conn->c_laddr.s6_addr32[3];
			dp->ricp_v4.dp_daddr = conn->c_faddr.s6_addr32[3];
			dp->ricp_v4.dp_protocol_major =
			    RDS_PROTOCOL_MAJOR(protocol_version);
			dp->ricp_v4.dp_protocol_minor =
			    RDS_PROTOCOL_MINOR(protocol_version);
			dp->ricp_v4.dp_protocol_minor_mask =
			    cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
			dp->ricp_v4.dp_ack_seq =
			    cpu_to_be64(rds_ib_piggyb_ack(ic));
244
			dp->ricp_v4.dp_cmn.ricpc_dp_toss = conn->c_tos;
245
246
247
248

			conn_param->private_data = &dp->ricp_v4;
			conn_param->private_data_len = sizeof(dp->ricp_v4);
		}
Andy Grover's avatar
Andy Grover committed
249
250
251
252
253

		/* Advertise flow control */
		if (ic->i_flowctl) {
			unsigned int credits;

254
255
256
257
258
259
260
261
			credits = IB_GET_POST_CREDITS
				(atomic_read(&ic->i_credits));
			if (isv6)
				dp->ricp_v6.dp_credit = cpu_to_be32(credits);
			else
				dp->ricp_v4.dp_credit = cpu_to_be32(credits);
			atomic_sub(IB_SET_POST_CREDITS(credits),
				   &ic->i_credits);
Andy Grover's avatar
Andy Grover committed
262
263
264
265
266
267
		}
	}
}

static void rds_ib_cq_event_handler(struct ib_event *event, void *data)
{
268
	rdsdebug("event %u (%s) data %p\n",
269
		 event->event, ib_event_msg(event->event), data);
Andy Grover's avatar
Andy Grover committed
270
271
}

272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
/* Plucking the oldest entry from the ring can be done concurrently with
 * the thread refilling the ring.  Each ring operation is protected by
 * spinlocks and the transient state of refilling doesn't change the
 * recording of which entry is oldest.
 *
 * This relies on IB only calling one cq comp_handler for each cq so that
 * there will only be one caller of rds_recv_incoming() per RDS connection.
 */
static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
{
	struct rds_connection *conn = context;
	struct rds_ib_connection *ic = conn->c_transport_data;

	rdsdebug("conn %p cq %p\n", conn, cq);

	rds_ib_stats_inc(s_ib_evt_handler_call);

	tasklet_schedule(&ic->i_recv_tasklet);
}

292
293
static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
		     struct ib_wc *wcs)
294
{
295
	int nr, i;
296
297
298
299
300
301
302
303
	struct ib_wc *wc;

	while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
		for (i = 0; i < nr; i++) {
			wc = wcs + i;
			rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
				 (unsigned long long)wc->wr_id, wc->status,
				 wc->byte_len, be32_to_cpu(wc->ex.imm_data));
304

305
306
307
308
309
310
			if (wc->wr_id <= ic->i_send_ring.w_nr ||
			    wc->wr_id == RDS_IB_ACK_WR_ID)
				rds_ib_send_cqe_handler(ic, wc);
			else
				rds_ib_mr_cqe_handler(ic, wc);

311
312
313
314
		}
	}
}

315
316
317
318
319
320
321
static void rds_ib_tasklet_fn_send(unsigned long data)
{
	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
	struct rds_connection *conn = ic->conn;

	rds_ib_stats_inc(s_ib_tasklet_call);

322
323
324
325
	/* if cq has been already reaped, ignore incoming cq event */
	if (atomic_read(&ic->i_cq_quiesce))
		return;

326
	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
327
	ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
328
	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
329
330
331
332

	if (rds_conn_up(conn) &&
	    (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags) ||
	    test_bit(0, &conn->c_map_queued)))
333
		rds_send_xmit(&ic->conn->c_path[0]);
334
335
}

336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
		     struct ib_wc *wcs,
		     struct rds_ib_ack_state *ack_state)
{
	int nr, i;
	struct ib_wc *wc;

	while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
		for (i = 0; i < nr; i++) {
			wc = wcs + i;
			rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
				 (unsigned long long)wc->wr_id, wc->status,
				 wc->byte_len, be32_to_cpu(wc->ex.imm_data));

			rds_ib_recv_cqe_handler(ic, wc, ack_state);
		}
	}
}

355
356
357
358
359
360
361
static void rds_ib_tasklet_fn_recv(unsigned long data)
{
	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
	struct rds_connection *conn = ic->conn;
	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
	struct rds_ib_ack_state state;

362
363
	if (!rds_ibdev)
		rds_conn_drop(conn);
364
365
366

	rds_ib_stats_inc(s_ib_tasklet_call);

367
368
369
370
	/* if cq has been already reaped, ignore incoming cq event */
	if (atomic_read(&ic->i_cq_quiesce))
		return;

371
	memset(&state, 0, sizeof(state));
372
	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
373
	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
374
	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
375
376
377
378
379
380
381
382
383
384
385
386

	if (state.ack_next_valid)
		rds_ib_set_ack(ic, state.ack_next, state.ack_required);
	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
		rds_send_drop_acked(conn, state.ack_recv, NULL);
		ic->i_ack_recv = state.ack_recv;
	}

	if (rds_conn_up(conn))
		rds_ib_attempt_ack(ic);
}

Andy Grover's avatar
Andy Grover committed
387
388
389
390
391
static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
{
	struct rds_connection *conn = data;
	struct rds_ib_connection *ic = conn->c_transport_data;

392
	rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,
393
		 ib_event_msg(event->event));
Andy Grover's avatar
Andy Grover committed
394
395
396
397
398
399

	switch (event->event) {
	case IB_EVENT_COMM_EST:
		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
		break;
	default:
400
401
402
		rdsdebug("Fatal QP Event %u (%s) - connection %pI6c->%pI6c, reconnecting\n",
			 event->event, ib_event_msg(event->event),
			 &conn->c_laddr, &conn->c_faddr);
403
		rds_conn_drop(conn);
Andy Grover's avatar
Andy Grover committed
404
405
406
407
		break;
	}
}

408
409
410
411
412
413
414
415
416
417
418
419
static void rds_ib_cq_comp_handler_send(struct ib_cq *cq, void *context)
{
	struct rds_connection *conn = context;
	struct rds_ib_connection *ic = conn->c_transport_data;

	rdsdebug("conn %p cq %p\n", conn, cq);

	rds_ib_stats_inc(s_ib_evt_handler_call);

	tasklet_schedule(&ic->i_send_tasklet);
}

420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev)
{
	int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1];
	int index = rds_ibdev->dev->num_comp_vectors - 1;
	int i;

	for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) {
		if (rds_ibdev->vector_load[i] < min) {
			index = i;
			min = rds_ibdev->vector_load[i];
		}
	}

	rds_ibdev->vector_load[index]++;
	return index;
}

static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index)
{
	rds_ibdev->vector_load[index]--;
}

Andy Grover's avatar
Andy Grover committed
442
443
444
445
446
447
448
449
450
/*
 * This needs to be very careful to not leave IS_ERR pointers around for
 * cleanup to trip over.
 */
static int rds_ib_setup_qp(struct rds_connection *conn)
{
	struct rds_ib_connection *ic = conn->c_transport_data;
	struct ib_device *dev = ic->i_cm_id->device;
	struct ib_qp_init_attr attr;
451
	struct ib_cq_init_attr cq_attr = {};
Andy Grover's avatar
Andy Grover committed
452
	struct rds_ib_device *rds_ibdev;
453
	unsigned long max_wrs;
454
	int ret, fr_queue_space;
Andy Grover's avatar
Andy Grover committed
455

456
457
458
	/*
	 * It's normal to see a null device if an incoming connection races
	 * with device removal, so we don't print a warning.
Andy Grover's avatar
Andy Grover committed
459
	 */
460
461
	rds_ibdev = rds_ib_get_client_data(dev);
	if (!rds_ibdev)
Andy Grover's avatar
Andy Grover committed
462
		return -EOPNOTSUPP;
463

464
465
466
467
	/* The fr_queue_space is currently set to 512, to add extra space on
	 * completion queue and send queue. This extra space is used for FRMR
	 * registration and invalidation work requests
	 */
468
	fr_queue_space = (rds_ibdev->use_fastreg ? RDS_IB_DEFAULT_FR_WR : 0);
469

470
471
	/* add the conn now so that connection establishment has the dev */
	rds_ib_add_conn(rds_ibdev, conn);
Andy Grover's avatar
Andy Grover committed
472

473
474
475
476
477
478
479
480
481
	max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_send_wr + 1 ?
		rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_send_wr;
	if (ic->i_send_ring.w_nr != max_wrs)
		rds_ib_ring_resize(&ic->i_send_ring, max_wrs);

	max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_recv_wr + 1 ?
		rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_recv_wr;
	if (ic->i_recv_ring.w_nr != max_wrs)
		rds_ib_ring_resize(&ic->i_recv_ring, max_wrs);
Andy Grover's avatar
Andy Grover committed
482
483
484
485

	/* Protection domain and memory range */
	ic->i_pd = rds_ibdev->pd;

486
	ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
487
	cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
488
	cq_attr.comp_vector = ic->i_scq_vector;
489
	ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
Andy Grover's avatar
Andy Grover committed
490
				     rds_ib_cq_event_handler, conn,
491
				     &cq_attr);
Andy Grover's avatar
Andy Grover committed
492
493
494
	if (IS_ERR(ic->i_send_cq)) {
		ret = PTR_ERR(ic->i_send_cq);
		ic->i_send_cq = NULL;
495
		ibdev_put_vector(rds_ibdev, ic->i_scq_vector);
Andy Grover's avatar
Andy Grover committed
496
		rdsdebug("ib_create_cq send failed: %d\n", ret);
Zhu Yanjun's avatar
Zhu Yanjun committed
497
		goto rds_ibdev_out;
Andy Grover's avatar
Andy Grover committed
498
499
	}

500
	ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
501
	cq_attr.cqe = ic->i_recv_ring.w_nr;
502
	cq_attr.comp_vector = ic->i_rcq_vector;
503
	ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
Andy Grover's avatar
Andy Grover committed
504
				     rds_ib_cq_event_handler, conn,
505
				     &cq_attr);
Andy Grover's avatar
Andy Grover committed
506
507
508
	if (IS_ERR(ic->i_recv_cq)) {
		ret = PTR_ERR(ic->i_recv_cq);
		ic->i_recv_cq = NULL;
509
		ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);
Andy Grover's avatar
Andy Grover committed
510
		rdsdebug("ib_create_cq recv failed: %d\n", ret);
Zhu Yanjun's avatar
Zhu Yanjun committed
511
		goto send_cq_out;
Andy Grover's avatar
Andy Grover committed
512
513
514
515
516
	}

	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
	if (ret) {
		rdsdebug("ib_req_notify_cq send failed: %d\n", ret);
Zhu Yanjun's avatar
Zhu Yanjun committed
517
		goto recv_cq_out;
Andy Grover's avatar
Andy Grover committed
518
519
520
521
522
	}

	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
	if (ret) {
		rdsdebug("ib_req_notify_cq recv failed: %d\n", ret);
Zhu Yanjun's avatar
Zhu Yanjun committed
523
		goto recv_cq_out;
Andy Grover's avatar
Andy Grover committed
524
525
526
527
528
529
530
	}

	/* XXX negotiate max send/recv with remote? */
	memset(&attr, 0, sizeof(attr));
	attr.event_handler = rds_ib_qp_event_handler;
	attr.qp_context = conn;
	/* + 1 to allow for the single ack message */
531
	attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;
Andy Grover's avatar
Andy Grover committed
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
	attr.cap.max_send_sge = rds_ibdev->max_sge;
	attr.cap.max_recv_sge = RDS_IB_RECV_SGE;
	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
	attr.qp_type = IB_QPT_RC;
	attr.send_cq = ic->i_send_cq;
	attr.recv_cq = ic->i_recv_cq;

	/*
	 * XXX this can fail if max_*_wr is too large?  Are we supposed
	 * to back off until we get a value that the hardware can support?
	 */
	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
	if (ret) {
		rdsdebug("rdma_create_qp failed: %d\n", ret);
Zhu Yanjun's avatar
Zhu Yanjun committed
547
		goto recv_cq_out;
Andy Grover's avatar
Andy Grover committed
548
549
550
551
552
553
	}

	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
					   ic->i_send_ring.w_nr *
						sizeof(struct rds_header),
					   &ic->i_send_hdrs_dma, GFP_KERNEL);
554
	if (!ic->i_send_hdrs) {
Andy Grover's avatar
Andy Grover committed
555
556
		ret = -ENOMEM;
		rdsdebug("ib_dma_alloc_coherent send failed\n");
Zhu Yanjun's avatar
Zhu Yanjun committed
557
		goto qp_out;
Andy Grover's avatar
Andy Grover committed
558
559
560
561
562
563
	}

	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
					   ic->i_recv_ring.w_nr *
						sizeof(struct rds_header),
					   &ic->i_recv_hdrs_dma, GFP_KERNEL);
564
	if (!ic->i_recv_hdrs) {
Andy Grover's avatar
Andy Grover committed
565
566
		ret = -ENOMEM;
		rdsdebug("ib_dma_alloc_coherent recv failed\n");
Zhu Yanjun's avatar
Zhu Yanjun committed
567
		goto send_hdrs_dma_out;
Andy Grover's avatar
Andy Grover committed
568
569
570
571
	}

	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
				       &ic->i_ack_dma, GFP_KERNEL);
572
	if (!ic->i_ack) {
Andy Grover's avatar
Andy Grover committed
573
574
		ret = -ENOMEM;
		rdsdebug("ib_dma_alloc_coherent ack failed\n");
Zhu Yanjun's avatar
Zhu Yanjun committed
575
		goto recv_hdrs_dma_out;
Andy Grover's avatar
Andy Grover committed
576
577
	}

578
579
	ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),
					      ic->i_send_ring.w_nr),
580
				   ibdev_to_node(dev));
581
	if (!ic->i_sends) {
Andy Grover's avatar
Andy Grover committed
582
583
		ret = -ENOMEM;
		rdsdebug("send allocation failed\n");
Zhu Yanjun's avatar
Zhu Yanjun committed
584
		goto ack_dma_out;
Andy Grover's avatar
Andy Grover committed
585
586
	}

587
588
	ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),
					      ic->i_recv_ring.w_nr),
589
				   ibdev_to_node(dev));
590
	if (!ic->i_recvs) {
Andy Grover's avatar
Andy Grover committed
591
592
		ret = -ENOMEM;
		rdsdebug("recv allocation failed\n");
Zhu Yanjun's avatar
Zhu Yanjun committed
593
		goto sends_out;
Andy Grover's avatar
Andy Grover committed
594
595
596
597
	}

	rds_ib_recv_init_ack(ic);

598
	rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,
Andy Grover's avatar
Andy Grover committed
599
600
		 ic->i_send_cq, ic->i_recv_cq);

601
	goto out;
Zhu Yanjun's avatar
Zhu Yanjun committed
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618

sends_out:
	vfree(ic->i_sends);
ack_dma_out:
	ib_dma_free_coherent(dev, sizeof(struct rds_header),
			     ic->i_ack, ic->i_ack_dma);
recv_hdrs_dma_out:
	ib_dma_free_coherent(dev, ic->i_recv_ring.w_nr *
					sizeof(struct rds_header),
					ic->i_recv_hdrs, ic->i_recv_hdrs_dma);
send_hdrs_dma_out:
	ib_dma_free_coherent(dev, ic->i_send_ring.w_nr *
					sizeof(struct rds_header),
					ic->i_send_hdrs, ic->i_send_hdrs_dma);
qp_out:
	rdma_destroy_qp(ic->i_cm_id);
recv_cq_out:
619
620
	ib_destroy_cq(ic->i_recv_cq);
	ic->i_recv_cq = NULL;
Zhu Yanjun's avatar
Zhu Yanjun committed
621
send_cq_out:
622
623
	ib_destroy_cq(ic->i_send_cq);
	ic->i_send_cq = NULL;
Zhu Yanjun's avatar
Zhu Yanjun committed
624
625
rds_ibdev_out:
	rds_ib_remove_conn(rds_ibdev, conn);
626
out:
627
	rds_ib_dev_put(rds_ibdev);
Zhu Yanjun's avatar
Zhu Yanjun committed
628

Andy Grover's avatar
Andy Grover committed
629
630
631
	return ret;
}

632
static u32 rds_ib_protocol_compatible(struct rdma_cm_event *event, bool isv6)
Andy Grover's avatar
Andy Grover committed
633
{
634
635
	const union rds_ib_conn_priv *dp = event->param.conn.private_data;
	u8 data_len, major, minor;
Andy Grover's avatar
Andy Grover committed
636
	u32 version = 0;
637
638
	__be16 mask;
	u16 common;
Andy Grover's avatar
Andy Grover committed
639

640
641
	/*
	 * rdma_cm private data is odd - when there is any private data in the
Andy Grover's avatar
Andy Grover committed
642
643
644
645
646
	 * request, we will be given a pretty large buffer without telling us the
	 * original size. The only way to tell the difference is by looking at
	 * the contents, which are initialized to zero.
	 * If the protocol version fields aren't set, this is a connection attempt
	 * from an older version. This could could be 3.0 or 2.0 - we can't tell.
647
648
649
650
651
652
653
654
655
656
	 * We really should have changed this for OFED 1.3 :-(
	 */

	/* Be paranoid. RDS always has privdata */
	if (!event->param.conn.private_data_len) {
		printk(KERN_NOTICE "RDS incoming connection has no private data, "
			"rejecting\n");
		return 0;
	}

657
658
659
660
661
662
663
664
665
666
667
668
	if (isv6) {
		data_len = sizeof(struct rds6_ib_connect_private);
		major = dp->ricp_v6.dp_protocol_major;
		minor = dp->ricp_v6.dp_protocol_minor;
		mask = dp->ricp_v6.dp_protocol_minor_mask;
	} else {
		data_len = sizeof(struct rds_ib_connect_private);
		major = dp->ricp_v4.dp_protocol_major;
		minor = dp->ricp_v4.dp_protocol_minor;
		mask = dp->ricp_v4.dp_protocol_minor_mask;
	}

669
	/* Even if len is crap *now* I still want to check it. -ASG */
670
	if (event->param.conn.private_data_len < data_len || major == 0)
671
		return RDS_PROTOCOL_4_0;
Andy Grover's avatar
Andy Grover committed
672

673
	common = be16_to_cpu(mask) & RDS_IB_SUPPORTED_PROTOCOLS;
674
675
	if (major == 4 && common) {
		version = RDS_PROTOCOL_4_0;
Andy Grover's avatar
Andy Grover committed
676
677
		while ((common >>= 1) != 0)
			version++;
678
679
680
	} else if (RDS_PROTOCOL_COMPAT_VERSION ==
		   RDS_PROTOCOL(major, minor)) {
		version = RDS_PROTOCOL_COMPAT_VERSION;
681
682
683
684
685
686
687
688
	} else {
		if (isv6)
			printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI6c using incompatible protocol version %u.%u\n",
					   &dp->ricp_v6.dp_saddr, major, minor);
		else
			printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI4 using incompatible protocol version %u.%u\n",
					   &dp->ricp_v4.dp_saddr, major, minor);
	}
Andy Grover's avatar
Andy Grover committed
689
690
691
	return version;
}

Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
692
#if IS_ENABLED(CONFIG_IPV6)
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
693
/* Given an IPv6 address, find the net_device which hosts that address and
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
 * return its index.  This is used by the rds_ib_cm_handle_connect() code to
 * find the interface index of where an incoming request comes from when
 * the request is using a link local address.
 *
 * Note one problem in this search.  It is possible that two interfaces have
 * the same link local address.  Unfortunately, this cannot be solved unless
 * the underlying layer gives us the interface which an incoming RDMA connect
 * request comes from.
 */
static u32 __rds_find_ifindex(struct net *net, const struct in6_addr *addr)
{
	struct net_device *dev;
	int idx = 0;

	rcu_read_lock();
	for_each_netdev_rcu(net, dev) {
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
710
		if (ipv6_chk_addr(net, addr, dev, 1)) {
711
712
713
714
715
716
717
718
			idx = dev->ifindex;
			break;
		}
	}
	rcu_read_unlock();

	return idx;
}
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
719
#endif
720

Andy Grover's avatar
Andy Grover committed
721
int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
722
			     struct rdma_cm_event *event, bool isv6)
Andy Grover's avatar
Andy Grover committed
723
724
725
{
	__be64 lguid = cm_id->route.path_rec->sgid.global.interface_id;
	__be64 fguid = cm_id->route.path_rec->dgid.global.interface_id;
726
	const struct rds_ib_conn_priv_cmn *dp_cmn;
Andy Grover's avatar
Andy Grover committed
727
728
729
	struct rds_connection *conn = NULL;
	struct rds_ib_connection *ic = NULL;
	struct rdma_conn_param conn_param;
730
731
732
733
734
735
736
737
	const union rds_ib_conn_priv *dp;
	union rds_ib_conn_priv dp_rep;
	struct in6_addr s_mapped_addr;
	struct in6_addr d_mapped_addr;
	const struct in6_addr *saddr6;
	const struct in6_addr *daddr6;
	int destroy = 1;
	u32 ifindex = 0;
Andy Grover's avatar
Andy Grover committed
738
	u32 version;
739
	int err = 1;
Andy Grover's avatar
Andy Grover committed
740
741

	/* Check whether the remote protocol version matches ours. */
742
	version = rds_ib_protocol_compatible(event, isv6);
743
744
	if (!version) {
		err = RDS_RDMA_REJ_INCOMPAT;
Andy Grover's avatar
Andy Grover committed
745
		goto out;
746
	}
Andy Grover's avatar
Andy Grover committed
747

748
749
	dp = event->param.conn.private_data;
	if (isv6) {
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
750
#if IS_ENABLED(CONFIG_IPV6)
751
752
753
		dp_cmn = &dp->ricp_v6.dp_cmn;
		saddr6 = &dp->ricp_v6.dp_saddr;
		daddr6 = &dp->ricp_v6.dp_daddr;
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
754
		/* If either address is link local, need to find the
755
756
757
758
759
760
761
762
763
764
765
		 * interface index in order to create a proper RDS
		 * connection.
		 */
		if (ipv6_addr_type(daddr6) & IPV6_ADDR_LINKLOCAL) {
			/* Using init_net for now ..  */
			ifindex = __rds_find_ifindex(&init_net, daddr6);
			/* No index found...  Need to bail out. */
			if (ifindex == 0) {
				err = -EOPNOTSUPP;
				goto out;
			}
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
766
767
768
769
770
771
772
773
		} else if (ipv6_addr_type(saddr6) & IPV6_ADDR_LINKLOCAL) {
			/* Use our address to find the correct index. */
			ifindex = __rds_find_ifindex(&init_net, daddr6);
			/* No index found...  Need to bail out. */
			if (ifindex == 0) {
				err = -EOPNOTSUPP;
				goto out;
			}
774
		}
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
775
776
777
778
#else
		err = -EOPNOTSUPP;
		goto out;
#endif
779
780
781
782
783
784
785
786
	} else {
		dp_cmn = &dp->ricp_v4.dp_cmn;
		ipv6_addr_set_v4mapped(dp->ricp_v4.dp_saddr, &s_mapped_addr);
		ipv6_addr_set_v4mapped(dp->ricp_v4.dp_daddr, &d_mapped_addr);
		saddr6 = &s_mapped_addr;
		daddr6 = &d_mapped_addr;
	}

787
788
789
	rdsdebug("saddr %pI6c daddr %pI6c RDSv%u.%u lguid 0x%llx fguid 0x%llx, tos:%d\n",
		 saddr6, daddr6, RDS_PROTOCOL_MAJOR(version),
		 RDS_PROTOCOL_MINOR(version),
Andy Grover's avatar
Andy Grover committed
790
		 (unsigned long long)be64_to_cpu(lguid),
791
		 (unsigned long long)be64_to_cpu(fguid), dp_cmn->ricpc_dp_toss);
Andy Grover's avatar
Andy Grover committed
792

793
	/* RDS/IB is not currently netns aware, thus init_net */
794
	conn = rds_conn_create(&init_net, daddr6, saddr6,
795
796
			       &rds_ib_transport, dp_cmn->ricpc_dp_toss,
			       GFP_KERNEL, ifindex);
Andy Grover's avatar
Andy Grover committed
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
	if (IS_ERR(conn)) {
		rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
		conn = NULL;
		goto out;
	}

	/*
	 * The connection request may occur while the
	 * previous connection exist, e.g. in case of failover.
	 * But as connections may be initiated simultaneously
	 * by both hosts, we have a random backoff mechanism -
	 * see the comment above rds_queue_reconnect()
	 */
	mutex_lock(&conn->c_cm_lock);
	if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
		if (rds_conn_state(conn) == RDS_CONN_UP) {
			rdsdebug("incoming connect while connecting\n");
			rds_conn_drop(conn);
			rds_ib_stats_inc(s_ib_listen_closed_stale);
		} else
		if (rds_conn_state(conn) == RDS_CONN_CONNECTING) {
			/* Wait and see - our connect may still be succeeding */
			rds_ib_stats_inc(s_ib_connect_raced);
		}
		goto out;
	}

	ic = conn->c_transport_data;

	rds_ib_set_protocol(conn, version);
827
	rds_ib_set_flow_control(conn, be32_to_cpu(dp_cmn->ricpc_credit));
Andy Grover's avatar
Andy Grover committed
828
829
830

	/* If the peer gave us the last packet it saw, process this as if
	 * we had received a regular ACK. */
831
832
833
	if (dp_cmn->ricpc_ack_seq)
		rds_send_drop_acked(conn, be64_to_cpu(dp_cmn->ricpc_ack_seq),
				    NULL);
Andy Grover's avatar
Andy Grover committed
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850

	BUG_ON(cm_id->context);
	BUG_ON(ic->i_cm_id);

	ic->i_cm_id = cm_id;
	cm_id->context = conn;

	/* We got halfway through setting up the ib_connection, if we
	 * fail now, we have to take the long route out of this mess. */
	destroy = 0;

	err = rds_ib_setup_qp(conn);
	if (err) {
		rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", err);
		goto out;
	}

851
	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
852
853
				  event->param.conn.responder_resources,
				  event->param.conn.initiator_depth, isv6);
Andy Grover's avatar
Andy Grover committed
854
855

	/* rdma_accept() calls rdma_reject() internally if it fails */
856
857
	if (rdma_accept(cm_id, &conn_param))
		rds_ib_conn_error(conn, "rdma_accept failed\n");
Andy Grover's avatar
Andy Grover committed
858
859

out:
860
861
862
	if (conn)
		mutex_unlock(&conn->c_cm_lock);
	if (err)
863
		rdma_reject(cm_id, &err, sizeof(int));
Andy Grover's avatar
Andy Grover committed
864
865
866
867
	return destroy;
}


868
int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6)
Andy Grover's avatar
Andy Grover committed
869
870
871
872
{
	struct rds_connection *conn = cm_id->context;
	struct rds_ib_connection *ic = conn->c_transport_data;
	struct rdma_conn_param conn_param;
873
	union rds_ib_conn_priv dp;
Andy Grover's avatar
Andy Grover committed
874
875
876
877
	int ret;

	/* If the peer doesn't do protocol negotiation, we must
	 * default to RDSv3.0 */
878
	rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
Andy Grover's avatar
Andy Grover committed
879
880
881
882
883
884
885
886
	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */

	ret = rds_ib_setup_qp(conn);
	if (ret) {
		rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", ret);
		goto out;
	}

887
888
	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp,
				  conn->c_proposed_version,
889
				  UINT_MAX, UINT_MAX, isv6);
Andy Grover's avatar
Andy Grover committed
890
891
892
893
894
895
896
897
898
899
900
901
	ret = rdma_connect(cm_id, &conn_param);
	if (ret)
		rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret);

out:
	/* Beware - returning non-zero tells the rdma_cm to destroy
	 * the cm_id. We should certainly not do it as long as we still
	 * "own" the cm_id. */
	if (ret) {
		if (ic->i_cm_id == cm_id)
			ret = 0;
	}
902
	ic->i_active_side = true;
Andy Grover's avatar
Andy Grover committed
903
904
905
	return ret;
}

906
int rds_ib_conn_path_connect(struct rds_conn_path *cp)
Andy Grover's avatar
Andy Grover committed
907
{
908
	struct rds_connection *conn = cp->cp_conn;
909
910
911
	struct sockaddr_storage src, dest;
	rdma_cm_event_handler handler;
	struct rds_ib_connection *ic;
Andy Grover's avatar
Andy Grover committed
912
913
	int ret;

914
915
	ic = conn->c_transport_data;

Andy Grover's avatar
Andy Grover committed
916
917
	/* XXX I wonder what affect the port space has */
	/* delegate cm event handler to rdma_transport */
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
918
#if IS_ENABLED(CONFIG_IPV6)
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
919
920
921
	if (conn->c_isv6)
		handler = rds6_rdma_cm_event_handler;
	else
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
922
#endif
Ka-Cheong Poon's avatar
Ka-Cheong Poon committed
923
		handler = rds_rdma_cm_event_handler;
924
	ic->i_cm_id = rdma_create_id(&init_net, handler, conn,
925
				     RDMA_PS_TCP, IB_QPT_RC);
Andy Grover's avatar
Andy Grover committed
926
927
928
929
930
931
932
933
934
	if (IS_ERR(ic->i_cm_id)) {
		ret = PTR_ERR(ic->i_cm_id);
		ic->i_cm_id = NULL;
		rdsdebug("rdma_create_id() failed: %d\n", ret);
		goto out;
	}

	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);

935
936
937
938
939
940
941
	if (ipv6_addr_v4mapped(&conn->c_faddr)) {
		struct sockaddr_in *sin;

		sin = (struct sockaddr_in *)&src;
		sin->sin_family = AF_INET;
		sin->sin_addr.s_addr = conn->c_laddr.s6_addr32[3];
		sin->sin_port = 0;
Andy Grover's avatar
Andy Grover committed
942

943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
		sin = (struct sockaddr_in *)&dest;
		sin->sin_family = AF_INET;
		sin->sin_addr.s_addr = conn->c_faddr.s6_addr32[3];
		sin->sin_port = htons(RDS_PORT);
	} else {
		struct sockaddr_in6 *sin6;

		sin6 = (struct sockaddr_in6 *)&src;
		sin6->sin6_family = AF_INET6;
		sin6->sin6_addr = conn->c_laddr;
		sin6->sin6_port = 0;
		sin6->sin6_scope_id = conn->c_dev_if;

		sin6 = (struct sockaddr_in6 *)&dest;
		sin6->sin6_family = AF_INET6;
		sin6->sin6_addr = conn->c_faddr;
		sin6->sin6_port = htons(RDS_CM_PORT);
		sin6->sin6_scope_id = conn->c_dev_if;
	}
Andy Grover's avatar
Andy Grover committed
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981

	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
				(struct sockaddr *)&dest,
				RDS_RDMA_RESOLVE_TIMEOUT_MS);
	if (ret) {
		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
			 ret);
		rdma_destroy_id(ic->i_cm_id);
		ic->i_cm_id = NULL;
	}

out:
	return ret;
}

/*
 * This is so careful about only cleaning up resources that were built up
 * so that it can be called at any point during startup.  In fact it
 * can be called multiple times for a given connection.
 */
982
void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
Andy Grover's avatar
Andy Grover committed
983
{
984
	struct rds_connection *conn = cp->cp_conn;
Andy Grover's avatar
Andy Grover committed
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
	struct rds_ib_connection *ic = conn->c_transport_data;
	int err = 0;

	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);

	if (ic->i_cm_id) {
		struct ib_device *dev = ic->i_cm_id->device;

		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
		err = rdma_disconnect(ic->i_cm_id);
		if (err) {
			/* Actually this may happen quite frequently, when
			 * an outgoing connect raced with an incoming connect.
			 */