OpenWrt – Blame information for rev 3
?pathlinks?
Rev | Author | Line No. | Line |
---|---|---|---|
1 | office | 1 | From 9115e8cd2a0c6eaaa900c462721f12e1d45f326c Mon Sep 17 00:00:00 2001 |
2 | From: Eric Dumazet <edumazet@google.com> |
||
3 | Date: Sat, 3 Dec 2016 11:14:56 -0800 |
||
4 | Subject: [PATCH 07/10] net: reorganize struct sock for better data locality |
||
5 | |||
6 | Group fields used in TX path, and keep some cache lines mostly read |
||
7 | to permit sharing among cpus. |
||
8 | |||
9 | Gained two 4 bytes holes on 64bit arches. |
||
10 | |||
11 | Added a place holder for tcp tsq_flags, next to sk_wmem_alloc |
||
12 | to speed up tcp_wfree() in the following patch. |
||
13 | |||
14 | I have not added ____cacheline_aligned_in_smp, this might be done later. |
||
15 | I prefer doing this once inet and tcp/udp sockets reorg is also done. |
||
16 | |||
17 | Tested with both TCP and UDP. |
||
18 | |||
19 | UDP receiver performance under flood increased by ~20 % : |
||
20 | Accessing sk_filter/sk_wq/sk_napi_id no longer stalls because sk_drops |
||
21 | was moved away from a critical cache line, now mostly read and shared. |
||
22 | |||
23 | /* --- cacheline 4 boundary (256 bytes) --- */ |
||
24 | unsigned int sk_napi_id; /* 0x100 0x4 */ |
||
25 | int sk_rcvbuf; /* 0x104 0x4 */ |
||
26 | struct sk_filter * sk_filter; /* 0x108 0x8 */ |
||
27 | union { |
||
28 | struct socket_wq * sk_wq; /* 0x8 */ |
||
29 | struct socket_wq * sk_wq_raw; /* 0x8 */ |
||
30 | }; /* 0x110 0x8 */ |
||
31 | struct xfrm_policy * sk_policy[2]; /* 0x118 0x10 */ |
||
32 | struct dst_entry * sk_rx_dst; /* 0x128 0x8 */ |
||
33 | struct dst_entry * sk_dst_cache; /* 0x130 0x8 */ |
||
34 | atomic_t sk_omem_alloc; /* 0x138 0x4 */ |
||
35 | int sk_sndbuf; /* 0x13c 0x4 */ |
||
36 | /* --- cacheline 5 boundary (320 bytes) --- */ |
||
37 | int sk_wmem_queued; /* 0x140 0x4 */ |
||
38 | atomic_t sk_wmem_alloc; /* 0x144 0x4 */ |
||
39 | long unsigned int sk_tsq_flags; /* 0x148 0x8 */ |
||
40 | struct sk_buff * sk_send_head; /* 0x150 0x8 */ |
||
41 | struct sk_buff_head sk_write_queue; /* 0x158 0x18 */ |
||
42 | __s32 sk_peek_off; /* 0x170 0x4 */ |
||
43 | int sk_write_pending; /* 0x174 0x4 */ |
||
44 | long int sk_sndtimeo; /* 0x178 0x8 */ |
||
45 | |||
46 | Signed-off-by: Eric Dumazet <edumazet@google.com> |
||
47 | Tested-by: Paolo Abeni <pabeni@redhat.com> |
||
48 | Signed-off-by: David S. Miller <davem@davemloft.net> |
||
49 | --- |
||
50 | include/net/sock.h | 51 +++++++++++++++++++++++++++------------------------ |
||
51 | 1 file changed, 27 insertions(+), 24 deletions(-) |
||
52 | |||
53 | --- a/include/net/sock.h |
||
54 | +++ b/include/net/sock.h |
||
3 | office | 55 | @@ -343,6 +343,9 @@ struct sock { |
1 | office | 56 | #define sk_rxhash __sk_common.skc_rxhash |
57 | |||
58 | socket_lock_t sk_lock; |
||
59 | + atomic_t sk_drops; |
||
60 | + int sk_rcvlowat; |
||
61 | + struct sk_buff_head sk_error_queue; |
||
62 | struct sk_buff_head sk_receive_queue; |
||
63 | /* |
||
64 | * The backlog queue is special, it is always used with |
||
3 | office | 65 | @@ -359,14 +362,13 @@ struct sock { |
1 | office | 66 | struct sk_buff *tail; |
67 | } sk_backlog; |
||
68 | #define sk_rmem_alloc sk_backlog.rmem_alloc |
||
69 | - int sk_forward_alloc; |
||
70 | |||
71 | - __u32 sk_txhash; |
||
72 | + int sk_forward_alloc; |
||
73 | #ifdef CONFIG_NET_RX_BUSY_POLL |
||
74 | - unsigned int sk_napi_id; |
||
75 | unsigned int sk_ll_usec; |
||
76 | + /* ===== mostly read cache line ===== */ |
||
77 | + unsigned int sk_napi_id; |
||
78 | #endif |
||
79 | - atomic_t sk_drops; |
||
80 | int sk_rcvbuf; |
||
81 | |||
82 | struct sk_filter __rcu *sk_filter; |
||
3 | office | 83 | @@ -379,11 +381,30 @@ struct sock { |
1 | office | 84 | #endif |
85 | struct dst_entry *sk_rx_dst; |
||
86 | struct dst_entry __rcu *sk_dst_cache; |
||
87 | - /* Note: 32bit hole on 64bit arches */ |
||
88 | - atomic_t sk_wmem_alloc; |
||
89 | atomic_t sk_omem_alloc; |
||
90 | int sk_sndbuf; |
||
91 | + |
||
92 | + /* ===== cache line for TX ===== */ |
||
93 | + int sk_wmem_queued; |
||
94 | + atomic_t sk_wmem_alloc; |
||
95 | + unsigned long sk_tsq_flags; |
||
96 | + struct sk_buff *sk_send_head; |
||
97 | struct sk_buff_head sk_write_queue; |
||
98 | + __s32 sk_peek_off; |
||
99 | + int sk_write_pending; |
||
100 | + long sk_sndtimeo; |
||
101 | + struct timer_list sk_timer; |
||
102 | + __u32 sk_priority; |
||
103 | + __u32 sk_mark; |
||
104 | + u32 sk_pacing_rate; /* bytes per second */ |
||
105 | + u32 sk_max_pacing_rate; |
||
106 | + struct page_frag sk_frag; |
||
107 | + netdev_features_t sk_route_caps; |
||
108 | + netdev_features_t sk_route_nocaps; |
||
109 | + int sk_gso_type; |
||
110 | + unsigned int sk_gso_max_size; |
||
111 | + gfp_t sk_allocation; |
||
112 | + __u32 sk_txhash; |
||
113 | |||
114 | /* |
||
115 | * Because of non atomicity rules, all |
||
3 | office | 116 | @@ -399,41 +420,23 @@ struct sock { |
1 | office | 117 | #define SK_PROTOCOL_MAX U8_MAX |
118 | kmemcheck_bitfield_end(flags); |
||
119 | |||
120 | - int sk_wmem_queued; |
||
121 | - gfp_t sk_allocation; |
||
122 | - u32 sk_pacing_rate; /* bytes per second */ |
||
123 | - u32 sk_max_pacing_rate; |
||
124 | - netdev_features_t sk_route_caps; |
||
125 | - netdev_features_t sk_route_nocaps; |
||
126 | - int sk_gso_type; |
||
127 | - unsigned int sk_gso_max_size; |
||
128 | u16 sk_gso_max_segs; |
||
129 | - int sk_rcvlowat; |
||
130 | unsigned long sk_lingertime; |
||
131 | - struct sk_buff_head sk_error_queue; |
||
132 | struct proto *sk_prot_creator; |
||
133 | rwlock_t sk_callback_lock; |
||
134 | int sk_err, |
||
135 | sk_err_soft; |
||
136 | u32 sk_ack_backlog; |
||
137 | u32 sk_max_ack_backlog; |
||
138 | - __u32 sk_priority; |
||
139 | - __u32 sk_mark; |
||
140 | struct pid *sk_peer_pid; |
||
141 | const struct cred *sk_peer_cred; |
||
142 | long sk_rcvtimeo; |
||
143 | - long sk_sndtimeo; |
||
144 | - struct timer_list sk_timer; |
||
145 | ktime_t sk_stamp; |
||
3 | office | 146 | u16 sk_tsflags; |
147 | u8 sk_shutdown; |
||
1 | office | 148 | u32 sk_tskey; |
149 | struct socket *sk_socket; |
||
150 | void *sk_user_data; |
||
151 | - struct page_frag sk_frag; |
||
152 | - struct sk_buff *sk_send_head; |
||
153 | - __s32 sk_peek_off; |
||
154 | - int sk_write_pending; |
||
155 | #ifdef CONFIG_SECURITY |
||
156 | void *sk_security; |
||
157 | #endif |