Line data Source code
1 : /*
2 : * Copyright (c) 2019 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef included_gso_h
17 : #define included_gso_h
18 :
19 : #include <vnet/vnet.h>
20 : #include <vnet/gso/hdr_offset_parser.h>
21 : #include <vnet/ip/ip_psh_cksum.h>
22 :
23 : typedef struct
24 : {
25 : vlib_main_t *vlib_main;
26 : vnet_main_t *vnet_main;
27 : u16 msg_id_base;
28 : } gso_main_t;
29 :
30 : extern gso_main_t gso_main;
31 :
32 : int vnet_sw_interface_gso_enable_disable (u32 sw_if_index, u8 enable);
33 : u32 gso_segment_buffer (vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd,
34 : u32 bi, vlib_buffer_t *b, generic_header_offset_t *gho,
35 : u32 n_bytes_b, u8 is_l2, u8 is_ip6);
36 :
37 : static_always_inline void
38 187990 : gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
39 : u32 flags, u16 n_bufs, u16 hdr_sz)
40 : {
41 187990 : u32 i = n_bufs;
42 2887860 : while (i >= 4)
43 : {
44 : /* prefetches */
45 2699870 : CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
46 2699870 : CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
47 2699870 : vlib_prefetch_buffer_data (bufs[2], LOAD);
48 2699870 : vlib_prefetch_buffer_data (bufs[3], LOAD);
49 :
50 : /* copying objects from cacheline 0 */
51 2699870 : bufs[0]->current_data = 0;
52 2699870 : bufs[1]->current_data = 0;
53 :
54 2699870 : bufs[0]->current_length = hdr_sz;
55 2699870 : bufs[1]->current_length = hdr_sz;
56 :
57 2699870 : bufs[0]->flags = bufs[1]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
58 2699870 : bufs[0]->flow_id = bufs[1]->flow_id = b0->flow_id;
59 2699870 : bufs[0]->error = bufs[1]->error = b0->error;
60 2699870 : bufs[0]->current_config_index = bufs[1]->current_config_index =
61 2699870 : b0->current_config_index;
62 :
63 2699870 : clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque));
64 2699870 : clib_memcpy_fast (&bufs[1]->opaque, &b0->opaque, sizeof (b0->opaque));
65 :
66 : /* copying objects from cacheline 1 */
67 2699870 : bufs[0]->trace_handle = b0->trace_handle;
68 2699870 : bufs[1]->trace_handle = b0->trace_handle;
69 :
70 2699870 : bufs[0]->total_length_not_including_first_buffer = 0;
71 2699870 : bufs[1]->total_length_not_including_first_buffer = 0;
72 :
73 : /* copying data */
74 2699870 : clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
75 2699870 : clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz);
76 :
77 2699870 : bufs += 2;
78 2699870 : i -= 2;
79 : }
80 :
81 653001 : while (i > 0)
82 : {
83 : /* copying objects from cacheline 0 */
84 465011 : bufs[0]->current_data = 0;
85 465011 : bufs[0]->current_length = hdr_sz;
86 465011 : bufs[0]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
87 465011 : bufs[0]->flow_id = b0->flow_id;
88 465011 : bufs[0]->error = b0->error;
89 465011 : bufs[0]->current_config_index = b0->current_config_index;
90 465011 : clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque));
91 :
92 : /* copying objects from cacheline 1 */
93 465011 : bufs[0]->trace_handle = b0->trace_handle;
94 465011 : bufs[0]->total_length_not_including_first_buffer = 0;
95 :
96 : /* copying data */
97 465011 : clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
98 :
99 465011 : bufs++;
100 465011 : i--;
101 : }
102 187990 : }
103 :
104 : static_always_inline void
105 5864750 : gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
106 : int is_l2, int is_ip6, generic_header_offset_t *gho,
107 : clib_ip_csum_t *c, u8 tcp_flags)
108 : {
109 :
110 5864750 : ip4_header_t *ip4 =
111 5864750 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
112 5864750 : gho->outer_hdr_sz);
113 5864750 : ip6_header_t *ip6 =
114 5864750 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
115 5864750 : gho->outer_hdr_sz);
116 5864750 : tcp_header_t *tcp =
117 5864750 : (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset +
118 5864750 : gho->outer_hdr_sz);
119 :
120 5864750 : tcp->flags = tcp_flags;
121 5864750 : tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
122 5864750 : c->odd = 0;
123 :
124 5864750 : if (is_ip6)
125 : {
126 5206220 : ip6->payload_length = clib_host_to_net_u16 (
127 2603110 : b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz);
128 2603110 : vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
129 2603110 : ip6_psh_t psh = { 0 };
130 2603110 : u32 *p = (u32 *) &psh;
131 2603110 : psh.src = ip6->src_address;
132 2603110 : psh.dst = ip6->dst_address;
133 2603110 : psh.l4len = ip6->payload_length;
134 2603110 : psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol);
135 28634200 : for (int i = 0; i < 10; i++)
136 26031100 : c->sum += p[i];
137 : }
138 : else
139 : {
140 6523280 : ip4->length = clib_host_to_net_u16 (
141 3261640 : b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz);
142 3261640 : if (gho->gho_flags & GHO_F_IP4)
143 3261640 : ip4->checksum = ip4_header_checksum (ip4);
144 3261640 : vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
145 : VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
146 3261640 : c->sum += clib_mem_unaligned (&ip4->src_address, u32);
147 3261640 : c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
148 3261640 : c->sum += clib_host_to_net_u32 (
149 3261640 : (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
150 3261640 : (ip4->protocol << 16));
151 : }
152 5864750 : clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz);
153 5864750 : tcp->checksum = clib_ip_csum_fold (c);
154 :
155 5864750 : if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
156 : {
157 1739640 : u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
158 :
159 1739640 : ip_adjacency_t *adj0 = adj_get (adj_index0);
160 :
161 1739640 : if (adj0->lookup_next_index == IP_LOOKUP_NEXT_MIDCHAIN &&
162 405 : adj0->sub_type.midchain.fixup_func)
163 : /* calls e.g. ipip44_fixup */
164 405 : adj0->sub_type.midchain.fixup_func (
165 : vm, adj0, b0, adj0->sub_type.midchain.fixup_data);
166 : }
167 5864750 : }
168 :
169 : static_always_inline u32
170 187990 : gso_segment_buffer_inline (vlib_main_t *vm,
171 : vnet_interface_per_thread_data_t *ptd,
172 : vlib_buffer_t *b, generic_header_offset_t *gho,
173 : int is_l2, int is_ip6)
174 : {
175 187990 : vlib_buffer_t **bufs = 0;
176 187990 : u32 n_tx_bytes = 0;
177 187990 : u16 gso_size = vnet_buffer2 (b)->gso_size;
178 187990 : u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0;
179 187990 : u32 default_bflags =
180 187990 : b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
181 187990 : u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz;
182 187990 : u32 next_tcp_seq = 0, tcp_seq = 0;
183 187990 : u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz;
184 187990 : u16 size =
185 187990 : clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - hdr_sz);
186 187990 : u16 n_alloc = 0, n_bufs = ((data_size + size - 1) / size);
187 187990 : clib_ip_csum_t c = { .sum = 0, .odd = 0 };
188 : u8 *src_ptr, *dst_ptr;
189 : u16 src_left, dst_left, bytes_to_copy;
190 187990 : u32 i = 0;
191 :
192 187990 : vec_validate (ptd->split_buffers, n_bufs - 1);
193 187990 : n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
194 187990 : if (n_alloc < n_bufs)
195 : {
196 0 : vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
197 0 : return 0;
198 : }
199 :
200 187990 : vec_validate (bufs, n_bufs - 1);
201 187990 : vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs);
202 :
203 187990 : tcp_header_t *tcp =
204 187990 : (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset +
205 187990 : gho->outer_hdr_sz);
206 187990 : tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
207 : /* store original flags for last packet and reset FIN and PSH */
208 187990 : tcp_flags = tcp->flags;
209 187990 : tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
210 187990 : tcp->checksum = 0;
211 :
212 187990 : gso_init_bufs_from_template_base (bufs, b, default_bflags, n_bufs, hdr_sz);
213 :
214 187990 : src_ptr = vlib_buffer_get_current (b) + hdr_sz;
215 187990 : src_left = b->current_length - hdr_sz;
216 187990 : dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
217 187990 : dst_left = size;
218 :
219 10421200 : while (data_size)
220 : {
221 10421200 : bytes_to_copy = clib_min (src_left, dst_left);
222 10421200 : clib_ip_csum_and_copy_chunk (&c, src_ptr, dst_ptr, bytes_to_copy);
223 :
224 10421200 : src_left -= bytes_to_copy;
225 10421200 : src_ptr += bytes_to_copy;
226 10421200 : data_size -= bytes_to_copy;
227 10421200 : dst_left -= bytes_to_copy;
228 10421200 : dst_ptr += bytes_to_copy;
229 10421200 : next_tcp_seq += bytes_to_copy;
230 10421200 : bufs[i]->current_length += bytes_to_copy;
231 :
232 10421200 : if (0 == src_left)
233 : {
234 : /* init src to the next buffer in chain */
235 4844900 : if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
236 : {
237 4656900 : b = vlib_get_buffer (vm, b->next_buffer);
238 4656900 : src_left = b->current_length;
239 4656900 : src_ptr = vlib_buffer_get_current (b);
240 : }
241 : else
242 : {
243 187990 : ASSERT (data_size == 0);
244 187990 : break;
245 : }
246 : }
247 10233200 : if (0 == dst_left && data_size)
248 : {
249 5676760 : vlib_prefetch_buffer_header (bufs[i + 1], LOAD);
250 5676760 : vlib_prefetch_buffer_data (bufs[i + 1], LOAD);
251 :
252 5676760 : n_tx_bytes += bufs[i]->current_length;
253 5676760 : gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho,
254 : &c, tcp_flags_no_fin_psh);
255 5676760 : i++;
256 5676760 : dst_left = size;
257 5676760 : dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
258 5676760 : tcp_seq = next_tcp_seq;
259 : // reset clib_ip_csum_t
260 5676760 : c.odd = 0;
261 5676760 : c.sum = 0;
262 : }
263 : }
264 :
265 187990 : ASSERT ((i + 1) == n_alloc);
266 187990 : n_tx_bytes += bufs[i]->current_length;
267 187990 : gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c,
268 : tcp_flags);
269 :
270 187990 : vec_free (bufs);
271 187990 : return n_tx_bytes;
272 : }
273 :
274 : #endif /* included_gso_h */
275 :
276 : /*
277 : * fd.io coding-style-patch-verification: ON
278 : *
279 : * Local Variables:
280 : * eval: (c-set-style "gnu")
281 : * End:
282 : */
|
