From efe4208f47f907b86f528788da711e8ab9dea44d Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 3 Oct 2013 15:42:29 -0700 Subject: ipv6: make lookups simpler and faster TCP listener refactoring, part 4 : To speed up inet lookups, we moved IPv4 addresses from inet to struct sock_common Now is time to do the same for IPv6, because it permits us to have fast lookups for all kind of sockets, including upcoming SYN_RECV. Getting IPv6 addresses in TCP lookups currently requires two extra cache lines, plus a dereference (and memory stall). inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6 This patch is way bigger than its IPv4 counter part, because for IPv4, we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6, it's not doable easily. inet6_sk(sk)->daddr becomes sk->sk_v6_daddr inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr at the same offset. We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic macro. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/ipv6.h | 46 ++++++---------------------------------- include/net/inet6_hashtables.h | 5 ++--- include/net/inet_timewait_sock.h | 4 +++- include/net/ip.h | 2 +- include/net/ip6_checksum.h | 2 +- include/net/sock.h | 9 ++++++++ 6 files changed, 22 insertions(+), 46 deletions(-) (limited to 'include') diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index b7f1f3bb346d..35f6c1b562c4 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -141,8 +141,6 @@ struct ipv6_fl_socklist; */ struct ipv6_pinfo { struct in6_addr saddr; - struct in6_addr rcv_saddr; - struct in6_addr daddr; struct in6_pktinfo sticky_pktinfo; const struct in6_addr *daddr_cache; #ifdef CONFIG_IPV6_SUBTREES @@ -256,22 +254,10 @@ struct tcp6_sock { extern int inet6_sk_rebuild_header(struct sock *sk); -struct inet6_timewait_sock { - struct in6_addr tw_v6_daddr; - struct in6_addr tw_v6_rcv_saddr; -}; - struct tcp6_timewait_sock { struct tcp_timewait_sock tcp6tw_tcp; - struct inet6_timewait_sock tcp6tw_inet6; }; -static inline struct inet6_timewait_sock *inet6_twsk(const struct sock *sk) -{ - return (struct inet6_timewait_sock *)(((u8 *)sk) + - inet_twsk(sk)->tw_ipv6_offset); -} - #if IS_ENABLED(CONFIG_IPV6) static inline struct ipv6_pinfo * inet6_sk(const struct sock *__sk) { @@ -321,21 +307,11 @@ static inline void inet_sk_copy_descendant(struct sock *sk_to, #define __ipv6_only_sock(sk) (inet6_sk(sk)->ipv6only) #define ipv6_only_sock(sk) ((sk)->sk_family == PF_INET6 && __ipv6_only_sock(sk)) -static inline u16 inet6_tw_offset(const struct proto *prot) -{ - return prot->twsk_prot->twsk_obj_size - - sizeof(struct inet6_timewait_sock); -} - -static inline struct in6_addr *__inet6_rcv_saddr(const struct sock *sk) +static inline const struct in6_addr *inet6_rcv_saddr(const struct sock *sk) { - return likely(sk->sk_state != TCP_TIME_WAIT) ? - &inet6_sk(sk)->rcv_saddr : &inet6_twsk(sk)->tw_v6_rcv_saddr; -} - -static inline struct in6_addr *inet6_rcv_saddr(const struct sock *sk) -{ - return sk->sk_family == AF_INET6 ? __inet6_rcv_saddr(sk) : NULL; + if (sk->sk_family == AF_INET6) + return &sk->sk_v6_rcv_saddr; + return NULL; } static inline int inet_v6_ipv6only(const struct sock *sk) @@ -363,7 +339,6 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk) return NULL; } -#define __inet6_rcv_saddr(__sk) NULL #define inet6_rcv_saddr(__sk) NULL #define tcp_twsk_ipv6only(__sk) 0 #define inet_v6_ipv6only(__sk) 0 @@ -372,19 +347,10 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk) #define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \ (((__sk)->sk_portpair == (__ports)) && \ ((__sk)->sk_family == AF_INET6) && \ - ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr)) && \ - ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \ + ipv6_addr_equal(&(__sk)->sk_v6_daddr, (__saddr)) && \ + ipv6_addr_equal(&(__sk)->sk_v6_rcv_saddr, (__daddr)) && \ (!(__sk)->sk_bound_dev_if || \ ((__sk)->sk_bound_dev_if == (__dif))) && \ net_eq(sock_net(__sk), (__net))) -#define INET6_TW_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \ - (((__sk)->sk_portpair == (__ports)) && \ - ((__sk)->sk_family == AF_INET6) && \ - ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr)) && \ - ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr)) && \ - (!(__sk)->sk_bound_dev_if || \ - ((__sk)->sk_bound_dev_if == (__dif))) && \ - net_eq(sock_net(__sk), (__net))) - #endif /* _IPV6_H */ diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h index f52fa88feb64..a105d1a2fc00 100644 --- a/include/net/inet6_hashtables.h +++ b/include/net/inet6_hashtables.h @@ -43,9 +43,8 @@ static inline unsigned int inet6_ehashfn(struct net *net, static inline int inet6_sk_ehashfn(const struct sock *sk) { const struct inet_sock *inet = inet_sk(sk); - const struct ipv6_pinfo *np = inet6_sk(sk); - const struct in6_addr *laddr = &np->rcv_saddr; - const struct in6_addr *faddr = &np->daddr; + const struct in6_addr *laddr = &sk->sk_v6_rcv_saddr; + const struct in6_addr *faddr = &sk->sk_v6_daddr; const __u16 lport = inet->inet_num; const __be16 fport = inet->inet_dport; struct net *net = sock_net(sk); diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h index de9e3ab7d43d..b647c6270eb7 100644 --- a/include/net/inet_timewait_sock.h +++ b/include/net/inet_timewait_sock.h @@ -116,7 +116,9 @@ struct inet_timewait_sock { #define tw_prot __tw_common.skc_prot #define tw_net __tw_common.skc_net #define tw_daddr __tw_common.skc_daddr +#define tw_v6_daddr __tw_common.skc_v6_daddr #define tw_rcv_saddr __tw_common.skc_rcv_saddr +#define tw_v6_rcv_saddr __tw_common.skc_v6_rcv_saddr #define tw_dport __tw_common.skc_dport #define tw_num __tw_common.skc_num @@ -133,7 +135,7 @@ struct inet_timewait_sock { tw_transparent : 1, tw_pad : 6, /* 6 bits hole */ tw_tos : 8, - tw_ipv6_offset : 16; + tw_pad2 : 16 /* 16 bits hole */ kmemcheck_bitfield_end(flags); u32 tw_ttd; struct inet_bind_bucket *tw_tb; diff --git a/include/net/ip.h b/include/net/ip.h index b39ebe5339ac..217bc5bfc6c6 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -374,7 +374,7 @@ static __inline__ void inet_reset_saddr(struct sock *sk) struct ipv6_pinfo *np = inet6_sk(sk); memset(&np->saddr, 0, sizeof(np->saddr)); - memset(&np->rcv_saddr, 0, sizeof(np->rcv_saddr)); + memset(&sk->sk_v6_rcv_saddr, 0, sizeof(sk->sk_v6_rcv_saddr)); } #endif } diff --git a/include/net/ip6_checksum.h b/include/net/ip6_checksum.h index 7686e3f5033d..1944406949ba 100644 --- a/include/net/ip6_checksum.h +++ b/include/net/ip6_checksum.h @@ -70,7 +70,7 @@ static inline void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) { struct ipv6_pinfo *np = inet6_sk(sk); - __tcp_v6_send_check(skb, &np->saddr, &np->daddr); + __tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr); } int udp6_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto); diff --git a/include/net/sock.h b/include/net/sock.h index 3f3e48c4704d..7e50df5c71d4 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -191,6 +191,12 @@ struct sock_common { #ifdef CONFIG_NET_NS struct net *skc_net; #endif + +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr skc_v6_daddr; + struct in6_addr skc_v6_rcv_saddr; +#endif + /* * fields between dontcopy_begin/dontcopy_end * are not copied in sock_copy() @@ -314,6 +320,9 @@ struct sock { #define sk_bind_node __sk_common.skc_bind_node #define sk_prot __sk_common.skc_prot #define sk_net __sk_common.skc_net +#define sk_v6_daddr __sk_common.skc_v6_daddr +#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr + socket_lock_t sk_lock; struct sk_buff_head sk_receive_queue; /* -- cgit v1.2.3-59-g8ed1b