Commit fa438ccf authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

[NET]: Keep sk_backlog near sk_lock



sk_backlog is a critical field of struct sock. (known famous words)

It is (ab)used in hot paths, in particular in release_sock(), tcp_recvmsg(),
tcp_v4_rcv(), sk_receive_skb().

It really makes sense to place it next to sk_lock, because sk_backlog is only
used after sk_lock locked (and thus memory cache line in L1 cache). This
should reduce cache misses and sk_lock acquisition time.

(In theory, we could only move the head pointer near sk_lock, and leaving tail
far away, because 'tail' is normally not so hot, but keep it simple :) )

Signed-off-by: default avatarEric Dumazet <dada1@cosmosbay.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e317f6f6
...@@ -202,6 +202,15 @@ struct sock { ...@@ -202,6 +202,15 @@ struct sock {
unsigned short sk_type; unsigned short sk_type;
int sk_rcvbuf; int sk_rcvbuf;
socket_lock_t sk_lock; socket_lock_t sk_lock;
/*
* The backlog queue is special, it is always used with
* the per-socket spinlock held and requires low latency
* access. Therefore we special case it's implementation.
*/
struct {
struct sk_buff *head;
struct sk_buff *tail;
} sk_backlog;
wait_queue_head_t *sk_sleep; wait_queue_head_t *sk_sleep;
struct dst_entry *sk_dst_cache; struct dst_entry *sk_dst_cache;
struct xfrm_policy *sk_policy[2]; struct xfrm_policy *sk_policy[2];
...@@ -221,15 +230,6 @@ struct sock { ...@@ -221,15 +230,6 @@ struct sock {
int sk_rcvlowat; int sk_rcvlowat;
unsigned long sk_flags; unsigned long sk_flags;
unsigned long sk_lingertime; unsigned long sk_lingertime;
/*
* The backlog queue is special, it is always used with
* the per-socket spinlock held and requires low latency
* access. Therefore we special case it's implementation.
*/
struct {
struct sk_buff *head;
struct sk_buff *tail;
} sk_backlog;
struct sk_buff_head sk_error_queue; struct sk_buff_head sk_error_queue;
struct proto *sk_prot_creator; struct proto *sk_prot_creator;
rwlock_t sk_callback_lock; rwlock_t sk_callback_lock;
......
...@@ -904,6 +904,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority) ...@@ -904,6 +904,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
sk_node_init(&newsk->sk_node); sk_node_init(&newsk->sk_node);
sock_lock_init(newsk); sock_lock_init(newsk);
bh_lock_sock(newsk); bh_lock_sock(newsk);
newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
atomic_set(&newsk->sk_rmem_alloc, 0); atomic_set(&newsk->sk_rmem_alloc, 0);
atomic_set(&newsk->sk_wmem_alloc, 0); atomic_set(&newsk->sk_wmem_alloc, 0);
...@@ -923,7 +924,6 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority) ...@@ -923,7 +924,6 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
newsk->sk_wmem_queued = 0; newsk->sk_wmem_queued = 0;
newsk->sk_forward_alloc = 0; newsk->sk_forward_alloc = 0;
newsk->sk_send_head = NULL; newsk->sk_send_head = NULL;
newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
sock_reset_flag(newsk, SOCK_DONE); sock_reset_flag(newsk, SOCK_DONE);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment