Line data Source code
1 : /*
2 : * Copyright (c) 2020 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef included_gro_func_h
17 : #define included_gro_func_h
18 :
19 : #include <vnet/ethernet/ethernet.h>
20 : #include <vnet/gso/gro.h>
21 : #include <vnet/gso/hdr_offset_parser.h>
22 : #include <vnet/ip/ip4.h>
23 : #include <vnet/ip/ip6.h>
24 : #include <vnet/ip/ip6_inlines.h>
25 : #include <vnet/udp/udp_packet.h>
26 : #include <vnet/tcp/tcp_packet.h>
27 : #include <vnet/vnet.h>
28 : #include <vnet/interface.h>
29 :
30 : #define GRO_MIN_PACKET_SIZE 256
31 : #define GRO_PADDED_PACKET_SIZE 64
32 :
33 : static_always_inline u8
34 26831208 : gro_is_bad_packet (vlib_buffer_t * b, u8 flags, i16 l234_sz)
35 : {
36 26831208 : if (((b->current_length - l234_sz) <= 0) ||
37 24663208 : ((flags &= ~(TCP_FLAG_ACK | TCP_FLAG_PSH)) != 0))
38 2167980 : return 1;
39 24663208 : return 0;
40 : }
41 :
42 : static_always_inline void
43 14190020 : gro_get_ip4_flow_from_packet (u32 * sw_if_index,
44 : ip4_header_t * ip4, tcp_header_t * tcp,
45 : gro_flow_key_t * flow_key, int is_l2)
46 : {
47 14190020 : flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX];
48 14190020 : flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX];
49 14190020 : ip46_address_set_ip4 (&flow_key->src_address, &ip4->src_address);
50 14190020 : ip46_address_set_ip4 (&flow_key->dst_address, &ip4->dst_address);
51 14190020 : flow_key->src_port = tcp->src_port;
52 14190020 : flow_key->dst_port = tcp->dst_port;
53 14190020 : }
54 :
55 : static_always_inline void
56 10473288 : gro_get_ip6_flow_from_packet (u32 * sw_if_index,
57 : ip6_header_t * ip6, tcp_header_t * tcp,
58 : gro_flow_key_t * flow_key, int is_l2)
59 : {
60 10473288 : flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX];
61 10473288 : flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX];
62 10473288 : ip46_address_set_ip6 (&flow_key->src_address, &ip6->src_address);
63 10473288 : ip46_address_set_ip6 (&flow_key->dst_address, &ip6->dst_address);
64 10473288 : flow_key->src_port = tcp->src_port;
65 10473288 : flow_key->dst_port = tcp->dst_port;
66 10473288 : }
67 :
68 : static_always_inline u32
69 51516946 : gro_is_ip4_or_ip6_packet (vlib_buffer_t *b0, u8 is_l2)
70 : {
71 51516946 : if (b0->flags & VNET_BUFFER_F_IS_IP4)
72 0 : return VNET_BUFFER_F_IS_IP4;
73 51516946 : if (b0->flags & VNET_BUFFER_F_IS_IP6)
74 2 : return VNET_BUFFER_F_IS_IP6;
75 51516944 : if (is_l2)
76 : {
77 : ethernet_header_t *eh =
78 40140723 : (ethernet_header_t *) vlib_buffer_get_current (b0);
79 40140723 : u16 ethertype = clib_net_to_host_u16 (eh->type);
80 :
81 40140723 : if (ethernet_frame_is_tagged (ethertype))
82 : {
83 0 : ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
84 :
85 0 : ethertype = clib_net_to_host_u16 (vlan->type);
86 0 : if (ethertype == ETHERNET_TYPE_VLAN)
87 : {
88 0 : vlan++;
89 0 : ethertype = clib_net_to_host_u16 (vlan->type);
90 : }
91 : }
92 40140723 : if (ethertype == ETHERNET_TYPE_IP4)
93 23045338 : return VNET_BUFFER_F_IS_IP4;
94 17095485 : if (ethertype == ETHERNET_TYPE_IP6)
95 17095383 : return VNET_BUFFER_F_IS_IP6;
96 : }
97 : else
98 : {
99 11376221 : if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x40)
100 6585774 : return VNET_BUFFER_F_IS_IP4;
101 4790407 : if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x60)
102 4790407 : return VNET_BUFFER_F_IS_IP6;
103 : }
104 :
105 44 : return 0;
106 : }
107 :
108 : typedef enum
109 : {
110 : GRO_PACKET_ACTION_NONE = 0,
111 : GRO_PACKET_ACTION_ENQUEUE = 1,
112 : GRO_PACKET_ACTION_FLUSH = 2,
113 : } gro_packet_action_t;
114 :
115 : static_always_inline gro_packet_action_t
116 23422758 : gro_tcp_sequence_check (tcp_header_t * tcp0, tcp_header_t * tcp1,
117 : u32 payload_len0)
118 : {
119 23422758 : u32 next_tcp_seq0 = clib_net_to_host_u32 (tcp0->seq_number);
120 23422758 : u32 next_tcp_seq1 = clib_net_to_host_u32 (tcp1->seq_number);
121 :
122 : /* next packet, enqueue */
123 23422758 : if (PREDICT_TRUE (next_tcp_seq0 + payload_len0 == next_tcp_seq1))
124 23422558 : return GRO_PACKET_ACTION_ENQUEUE;
125 : /* flush all packets */
126 : else
127 172 : return GRO_PACKET_ACTION_FLUSH;
128 : }
129 :
130 : static_always_inline void
131 23311155 : gro_merge_buffers (vlib_main_t * vm, vlib_buffer_t * b0,
132 : vlib_buffer_t * b1, u32 bi1, u32 payload_len1,
133 : u16 l234_sz1)
134 : {
135 23311155 : vlib_buffer_t *pb = b0;
136 :
137 23311155 : if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
138 312175 : b0->total_length_not_including_first_buffer = 0;
139 :
140 566562662 : while (pb->flags & VLIB_BUFFER_NEXT_PRESENT)
141 543252207 : pb = vlib_get_buffer (vm, pb->next_buffer);
142 :
143 23311155 : vlib_buffer_advance (b1, l234_sz1);
144 23311155 : pb->flags |= VLIB_BUFFER_NEXT_PRESENT;
145 23311155 : pb->next_buffer = bi1;
146 23311155 : b0->total_length_not_including_first_buffer += payload_len1;
147 23311155 : b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
148 23311155 : }
149 :
150 : static_always_inline u32
151 24663208 : gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0,
152 : generic_header_offset_t * gho0, int is_ip4)
153 : {
154 24663208 : u32 flags = 0;
155 :
156 24663208 : if (b0->flags & VNET_BUFFER_F_OFFLOAD)
157 0 : return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
158 24663208 : vlib_buffer_advance (b0, gho0->l3_hdr_offset);
159 24663208 : if (is_ip4)
160 14190020 : flags = ip4_tcp_udp_validate_checksum (vm, b0);
161 : else
162 10473288 : flags = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
163 24663208 : vlib_buffer_advance (b0, -gho0->l3_hdr_offset);
164 24663208 : return flags;
165 : }
166 :
167 : static_always_inline u32
168 181 : gro_fix_padded_packet_len (vlib_buffer_t *b0, generic_header_offset_t *gho0,
169 : ip4_header_t *ip4_0, ip6_header_t *ip6_0,
170 : u32 pkt_len0, u16 l234_sz0)
171 : {
172 181 : u32 tcp_payload_len0 = 0;
173 181 : if (gho0->gho_flags & GHO_F_IP4)
174 : {
175 181 : tcp_payload_len0 = clib_net_to_host_u16 (ip4_0->length) -
176 181 : ip4_header_bytes (ip4_0) - gho0->l4_hdr_sz;
177 : }
178 : else
179 : {
180 0 : tcp_payload_len0 =
181 0 : clib_net_to_host_u16 (ip6_0->payload_length) - gho0->l4_hdr_sz;
182 : }
183 :
184 181 : ASSERT (l234_sz0 + tcp_payload_len0 <= pkt_len0);
185 :
186 181 : if (PREDICT_FALSE (l234_sz0 + tcp_payload_len0 < pkt_len0))
187 : {
188 : /* small packet with padding at the end, remove padding */
189 124 : b0->current_length = l234_sz0 + tcp_payload_len0;
190 124 : pkt_len0 = b0->current_length;
191 : }
192 181 : return pkt_len0;
193 : }
194 :
195 : static_always_inline u32
196 26831714 : gro_get_packet_data (vlib_main_t *vm, vlib_buffer_t *b0,
197 : generic_header_offset_t *gho0, gro_flow_key_t *flow_key0,
198 : u8 is_l2)
199 : {
200 26831714 : ip4_header_t *ip4_0 = 0;
201 26831714 : ip6_header_t *ip6_0 = 0;
202 26831714 : tcp_header_t *tcp0 = 0;
203 26831714 : u32 flags = 0;
204 26831714 : u32 pkt_len0 = 0;
205 26831714 : u16 l234_sz0 = 0;
206 26831714 : u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 };
207 :
208 26831714 : u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
209 :
210 26831714 : if (is_ip0 & VNET_BUFFER_F_IS_IP4)
211 15409520 : vnet_generic_header_offset_parser (b0, gho0, is_l2, 1 /* is_ip4 */ ,
212 : 0 /* is_ip6 */ );
213 11422094 : else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
214 11422092 : vnet_generic_header_offset_parser (b0, gho0, is_l2, 0 /* is_ip4 */ ,
215 : 1 /* is_ip6 */ );
216 : else
217 44 : return 0;
218 :
219 26831612 : if (PREDICT_FALSE ((gho0->gho_flags & GHO_F_TCP) == 0))
220 398 : return 0;
221 :
222 26831208 : ip4_0 =
223 26831208 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset);
224 26831208 : ip6_0 =
225 26831208 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset);
226 26831208 : tcp0 =
227 26831208 : (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0->l4_hdr_offset);
228 :
229 26831208 : l234_sz0 = gho0->hdr_sz;
230 26831208 : if (PREDICT_FALSE (gro_is_bad_packet (b0, tcp0->flags, l234_sz0)))
231 2167980 : return 0;
232 :
233 24663208 : sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX];
234 24663208 : sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX];
235 :
236 24663208 : if (gho0->gho_flags & GHO_F_IP4)
237 : {
238 14190020 : flags = gro_validate_checksum (vm, b0, gho0, 1);
239 14190020 : gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, flow_key0,
240 : is_l2);
241 : }
242 10473288 : else if (gho0->gho_flags & GHO_F_IP6)
243 : {
244 10473288 : flags = gro_validate_checksum (vm, b0, gho0, 0);
245 10473288 : gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, flow_key0,
246 : is_l2);
247 : }
248 : else
249 0 : return 0;
250 :
251 24663208 : if (PREDICT_FALSE ((flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) == 0))
252 0 : return 0;
253 :
254 24663208 : pkt_len0 = vlib_buffer_length_in_chain (vm, b0);
255 24663208 : if (PREDICT_FALSE (pkt_len0 >= TCP_MAX_GSO_SZ))
256 0 : return 0;
257 :
258 24663208 : if (PREDICT_FALSE (pkt_len0 <= GRO_PADDED_PACKET_SIZE))
259 : {
260 : pkt_len0 =
261 181 : gro_fix_padded_packet_len (b0, gho0, ip4_0, ip6_0, pkt_len0, l234_sz0);
262 : }
263 24663208 : return pkt_len0;
264 : }
265 :
266 : static_always_inline u32
267 : gro_coalesce_buffers (vlib_main_t *vm, vlib_buffer_t *b0, vlib_buffer_t *b1,
268 : u32 bi1, u8 is_l2)
269 : {
270 : generic_header_offset_t gho0 = { 0 };
271 : generic_header_offset_t gho1 = { 0 };
272 : gro_flow_key_t flow_key0, flow_key1;
273 : ip4_header_t *ip4_0, *ip4_1;
274 : ip6_header_t *ip6_0, *ip6_1;
275 : tcp_header_t *tcp0, *tcp1;
276 : u16 l234_sz0, l234_sz1;
277 : u32 pkt_len0, pkt_len1, payload_len0, payload_len1;
278 : u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 };
279 : u32 sw_if_index1[VLIB_N_RX_TX] = { ~0 };
280 :
281 : u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
282 : u32 is_ip1 = gro_is_ip4_or_ip6_packet (b1, is_l2);
283 :
284 : if (is_ip0 & VNET_BUFFER_F_IS_IP4)
285 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ ,
286 : 0 /* is_ip6 */ );
287 : else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
288 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ ,
289 : 1 /* is_ip6 */ );
290 : else
291 : return 0;
292 :
293 : if (is_ip1 & VNET_BUFFER_F_IS_IP4)
294 : vnet_generic_header_offset_parser (b1, &gho1, is_l2, 1 /* is_ip4 */ ,
295 : 0 /* is_ip6 */ );
296 : else if (is_ip1 & VNET_BUFFER_F_IS_IP6)
297 : vnet_generic_header_offset_parser (b1, &gho1, is_l2, 0 /* is_ip4 */ ,
298 : 1 /* is_ip6 */ );
299 : else
300 : return 0;
301 :
302 : pkt_len0 = vlib_buffer_length_in_chain (vm, b0);
303 : pkt_len1 = vlib_buffer_length_in_chain (vm, b1);
304 :
305 : if (((gho0.gho_flags & GHO_F_TCP) == 0 || pkt_len0 <= GRO_MIN_PACKET_SIZE) ||
306 : ((gho1.gho_flags & GHO_F_TCP) == 0 || pkt_len1 <= GRO_MIN_PACKET_SIZE))
307 : return 0;
308 :
309 : ip4_0 =
310 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
311 : ip4_1 =
312 : (ip4_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset);
313 : ip6_0 =
314 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
315 : ip6_1 =
316 : (ip6_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset);
317 :
318 : tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
319 : tcp1 = (tcp_header_t *) (vlib_buffer_get_current (b1) + gho1.l4_hdr_offset);
320 :
321 : l234_sz0 = gho0.hdr_sz;
322 : l234_sz1 = gho1.hdr_sz;
323 :
324 : if (gro_is_bad_packet (b0, tcp0->flags, l234_sz0)
325 : || gro_is_bad_packet (b1, tcp1->flags, l234_sz1))
326 : return 0;
327 :
328 : sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX];
329 : sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX];
330 :
331 : sw_if_index1[VLIB_RX] = vnet_buffer (b1)->sw_if_index[VLIB_RX];
332 : sw_if_index1[VLIB_TX] = vnet_buffer (b1)->sw_if_index[VLIB_TX];
333 :
334 : if ((gho0.gho_flags & GHO_F_IP4) && (gho1.gho_flags & GHO_F_IP4))
335 : {
336 : gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, &flow_key0,
337 : is_l2);
338 : gro_get_ip4_flow_from_packet (sw_if_index1, ip4_1, tcp1, &flow_key1,
339 : is_l2);
340 : }
341 : else if ((gho0.gho_flags & GHO_F_IP6) && (gho1.gho_flags & GHO_F_IP6))
342 : {
343 : gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, &flow_key0,
344 : is_l2);
345 : gro_get_ip6_flow_from_packet (sw_if_index1, ip6_1, tcp1, &flow_key1,
346 : is_l2);
347 : }
348 : else
349 : return 0;
350 :
351 : if (gro_flow_is_equal (&flow_key0, &flow_key1) == 0)
352 : return 0;
353 :
354 : payload_len0 = pkt_len0 - l234_sz0;
355 : payload_len1 = pkt_len1 - l234_sz1;
356 :
357 : if (pkt_len0 >= TCP_MAX_GSO_SZ || pkt_len1 >= TCP_MAX_GSO_SZ
358 : || (pkt_len0 + payload_len1) >= TCP_MAX_GSO_SZ)
359 : return 0;
360 :
361 : if (gro_tcp_sequence_check (tcp0, tcp1, payload_len0) ==
362 : GRO_PACKET_ACTION_ENQUEUE)
363 : {
364 : gro_merge_buffers (vm, b0, b1, bi1, payload_len1, l234_sz1);
365 : tcp0->flags |= tcp1->flags;
366 : return tcp1->ack_number;
367 : }
368 :
369 : return 0;
370 : }
371 :
372 : static_always_inline void
373 1262514 : gro_fixup_header (vlib_main_t *vm, vlib_buffer_t *b0, u32 ack_number, u8 is_l2)
374 : {
375 1262514 : generic_header_offset_t gho0 = { 0 };
376 :
377 1262514 : u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
378 :
379 1262514 : if (is_ip0 & VNET_BUFFER_F_IS_IP4)
380 674242 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ ,
381 : 0 /* is_ip6 */ );
382 588277 : else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
383 588277 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ ,
384 : 1 /* is_ip6 */ );
385 :
386 1262514 : vnet_buffer2 (b0)->gso_size = b0->current_length - gho0.hdr_sz;
387 1262514 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
388 :
389 1262514 : if (gho0.gho_flags & GHO_F_IP4)
390 : {
391 674242 : ip4_header_t *ip4 =
392 674242 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
393 674242 : ip4->length =
394 674242 : clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
395 674242 : gho0.l3_hdr_offset);
396 674242 : vnet_buffer (b0)->l3_hdr_offset = (u8 *) ip4 - b0->data;
397 674242 : b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
398 674242 : vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
399 : VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
400 : }
401 588277 : else if (gho0.gho_flags & GHO_F_IP6)
402 : {
403 588277 : ip6_header_t *ip6 =
404 588277 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
405 588277 : ip6->payload_length =
406 588277 : clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
407 588277 : gho0.l4_hdr_offset);
408 588277 : vnet_buffer (b0)->l3_hdr_offset = (u8 *) ip6 - b0->data;
409 588277 : b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
410 588277 : vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
411 : }
412 :
413 1262514 : tcp_header_t *tcp0 =
414 1262514 : (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
415 1262514 : vnet_buffer (b0)->l4_hdr_offset = (u8 *) tcp0 - b0->data;
416 1262514 : vnet_buffer2 (b0)->gso_l4_hdr_sz = tcp_header_bytes (tcp0);
417 1262514 : tcp0->ack_number = ack_number;
418 1262514 : b0->flags &= ~VLIB_BUFFER_IS_TRACED;
419 1262514 : }
420 :
421 : static_always_inline u32
422 16167005 : vnet_gro_flow_table_flush (vlib_main_t * vm, gro_flow_table_t * flow_table,
423 : u32 * to)
424 : {
425 16167005 : if (flow_table->flow_table_size > 0)
426 : {
427 : gro_flow_t *gro_flow;
428 175783 : u32 i = 0, j = 0;
429 2988307 : while (i < GRO_FLOW_TABLE_MAX_SIZE)
430 : {
431 2812526 : gro_flow = &flow_table->gro_flow[i];
432 2812526 : if (gro_flow->n_buffers && gro_flow_is_timeout (vm, gro_flow))
433 : {
434 : // flush the packet
435 : vlib_buffer_t *b0 =
436 175722 : vlib_get_buffer (vm, gro_flow->buffer_index);
437 175722 : gro_fixup_header (vm, b0, gro_flow->last_ack_number,
438 175722 : flow_table->is_l2);
439 175722 : to[j] = gro_flow->buffer_index;
440 175722 : gro_flow_table_reset_flow (flow_table, gro_flow);
441 175722 : flow_table->n_vectors++;
442 175722 : j++;
443 : }
444 2812526 : i++;
445 : }
446 :
447 175783 : return j;
448 : }
449 15991204 : return 0;
450 : }
451 :
452 : static_always_inline void
453 170057005 : vnet_gro_flow_table_schedule_node_on_dispatcher (vlib_main_t *vm,
454 : vnet_hw_if_tx_queue_t *txq,
455 : gro_flow_table_t *flow_table)
456 : {
457 170057005 : if (gro_flow_table_is_timeout (vm, flow_table))
458 : {
459 16167005 : u32 to[GRO_FLOW_TABLE_MAX_SIZE] = { 0 };
460 16167005 : u32 n_to = vnet_gro_flow_table_flush (vm, flow_table, to);
461 :
462 16167005 : if (n_to > 0)
463 : {
464 175722 : u32 node_index = flow_table->node_index;
465 175722 : vlib_frame_t *f = vlib_get_frame_to_node (vm, node_index);
466 175722 : vnet_hw_if_tx_frame_t *ft = vlib_frame_scalar_args (f);
467 175722 : u32 *f_to = vlib_frame_vector_args (f);
468 175722 : u32 i = 0;
469 :
470 175722 : ft->shared_queue = txq->shared_queue;
471 175722 : ft->queue_id = txq->queue_id;
472 :
473 351444 : while (i < n_to)
474 : {
475 175722 : f_to[f->n_vectors] = to[i];
476 175722 : i++;
477 175722 : f->n_vectors++;
478 : }
479 175722 : vlib_put_frame_to_node (vm, node_index, f);
480 : }
481 16167005 : gro_flow_table_set_timeout (vm, flow_table, GRO_FLOW_TABLE_FLUSH);
482 : }
483 170057005 : }
484 :
485 : static_always_inline u32
486 70616 : vnet_gro_flush_all_packets (vlib_main_t *vm, gro_flow_table_t *flow_table,
487 : gro_flow_t *gro_flow, vlib_buffer_t *b_s, u32 *to,
488 : u32 bi_s, u32 bi0, u8 is_l2)
489 : {
490 70616 : flow_table->n_vectors++;
491 70616 : flow_table->total_vectors++;
492 70616 : gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
493 70616 : gro_flow->n_buffers = 0;
494 70616 : gro_flow_table_reset_flow (flow_table, gro_flow);
495 70616 : to[0] = bi_s;
496 70616 : to[1] = bi0;
497 70616 : return 2;
498 : }
499 :
500 : static_always_inline u32
501 27007415 : vnet_gro_flow_table_inline (vlib_main_t * vm, gro_flow_table_t * flow_table,
502 : u32 bi0, u32 * to)
503 : {
504 27007415 : vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
505 27007415 : generic_header_offset_t gho0 = { 0 };
506 27007415 : gro_flow_t *gro_flow = 0;
507 27007415 : gro_flow_key_t flow_key0 = { };
508 27007415 : tcp_header_t *tcp0 = 0;
509 27007415 : u32 pkt_len0 = 0;
510 27007415 : u32 is_flush = 0;
511 27007415 : u8 is_l2 = flow_table->is_l2;
512 :
513 27007415 : if (!gro_flow_table_is_enable (flow_table))
514 : {
515 0 : to[0] = bi0;
516 0 : return 1;
517 : }
518 :
519 27007415 : if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
520 : {
521 175722 : to[0] = bi0;
522 175722 : return 1;
523 : }
524 :
525 26831714 : pkt_len0 = gro_get_packet_data (vm, b0, &gho0, &flow_key0, is_l2);
526 26831714 : if (pkt_len0 == 0)
527 : {
528 2168426 : to[0] = bi0;
529 2168426 : return 1;
530 : }
531 :
532 24663208 : tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
533 24663208 : if (PREDICT_TRUE (((tcp0->flags & TCP_FLAG_PSH) == 0) &&
534 : (pkt_len0 > GRO_MIN_PACKET_SIZE)))
535 23598659 : gro_flow = gro_flow_table_find_or_add_flow (flow_table, &flow_key0);
536 : else
537 : {
538 1064609 : is_flush = 1;
539 1064609 : gro_flow = gro_flow_table_get_flow (flow_table, &flow_key0);
540 : }
541 :
542 24663208 : if (!gro_flow)
543 : {
544 18961 : to[0] = bi0;
545 18961 : return 1;
546 : }
547 :
548 24644308 : if (PREDICT_FALSE (gro_flow->n_buffers == 0))
549 : {
550 1221540 : flow_table->total_vectors++;
551 1221540 : gro_flow_store_packet (gro_flow, bi0);
552 1221540 : gro_flow->last_ack_number = tcp0->ack_number;
553 1221540 : gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT);
554 1221540 : return 0;
555 : }
556 : else
557 : {
558 23422758 : generic_header_offset_t gho_s = { 0 };
559 : tcp_header_t *tcp_s;
560 : u16 l234_sz0, l234_sz_s;
561 : u32 pkt_len_s, payload_len0, payload_len_s;
562 23422758 : u32 bi_s = gro_flow->buffer_index;
563 :
564 23422758 : vlib_buffer_t *b_s = vlib_get_buffer (vm, bi_s);
565 23422758 : u32 is_ip_s = gro_is_ip4_or_ip6_packet (b_s, is_l2);
566 23422758 : if (is_ip_s & VNET_BUFFER_F_IS_IP4)
567 13547271 : vnet_generic_header_offset_parser (b_s, &gho_s, is_l2,
568 : 1 /* is_ip4 */ , 0 /* is_ip6 */ );
569 9875477 : else if (is_ip_s & VNET_BUFFER_F_IS_IP6)
570 9875477 : vnet_generic_header_offset_parser (b_s, &gho_s, is_l2,
571 : 0 /* is_ip4 */ , 1 /* is_ip6 */ );
572 :
573 23422758 : tcp_s =
574 23422758 : (tcp_header_t *) (vlib_buffer_get_current (b_s) +
575 23422758 : gho_s.l4_hdr_offset);
576 23422758 : pkt_len_s = vlib_buffer_length_in_chain (vm, b_s);
577 23422758 : l234_sz0 = gho0.hdr_sz;
578 23422758 : l234_sz_s = gho_s.hdr_sz;
579 23422758 : payload_len0 = pkt_len0 - l234_sz0;
580 23422758 : payload_len_s = pkt_len_s - l234_sz_s;
581 : gro_packet_action_t action =
582 23422758 : gro_tcp_sequence_check (tcp_s, tcp0, payload_len_s);
583 :
584 23422758 : if (PREDICT_TRUE (action == GRO_PACKET_ACTION_ENQUEUE))
585 : {
586 23422558 : if (PREDICT_TRUE (((pkt_len_s + payload_len0) < TCP_MAX_GSO_SZ) &&
587 : (gro_flow->n_buffers < GRO_FLOW_N_BUFFERS)))
588 : {
589 23311155 : flow_table->total_vectors++;
590 23311155 : gro_merge_buffers (vm, b_s, b0, bi0, payload_len0, l234_sz0);
591 23311155 : gro_flow_store_packet (gro_flow, bi0);
592 23311155 : gro_flow->last_ack_number = tcp0->ack_number;
593 23311155 : if (PREDICT_FALSE (is_flush))
594 : {
595 975200 : flow_table->n_vectors++;
596 975200 : tcp_s->flags |= tcp0->flags;
597 975200 : gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
598 975200 : gro_flow->n_buffers = 0;
599 975200 : gro_flow_table_reset_flow (flow_table, gro_flow);
600 975200 : to[0] = bi_s;
601 975200 : return 1;
602 : }
603 22336006 : return 0;
604 : }
605 111425 : else if (PREDICT_FALSE (is_flush))
606 : // flush the all (current and stored) packets
607 70444 : return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s,
608 : to, bi_s, bi0, is_l2);
609 : else
610 : {
611 : // flush the stored GSO size packet and buffer the current packet
612 40981 : flow_table->n_vectors++;
613 40981 : flow_table->total_vectors++;
614 40981 : gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
615 40981 : gro_flow->n_buffers = 0;
616 40981 : gro_flow_store_packet (gro_flow, bi0);
617 40981 : gro_flow->last_ack_number = tcp0->ack_number;
618 40981 : gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT);
619 40981 : to[0] = bi_s;
620 40981 : return 1;
621 : }
622 : }
623 : else
624 : {
625 : // flush the all (current and stored) packets
626 172 : return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s, to,
627 : bi_s, bi0, is_l2);
628 : }
629 : }
630 : }
631 :
632 : /**
633 : * coalesce buffers with flow tables
634 : */
635 : static_always_inline u32
636 523166 : vnet_gro_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, u32 * from,
637 : u16 n_left_from, u32 * to)
638 : {
639 523166 : u16 count = 0, i = 0;
640 :
641 27530526 : for (i = 0; i < n_left_from; i++)
642 27007415 : count += vnet_gro_flow_table_inline (vm, flow_table, from[i], &to[count]);
643 :
644 523166 : return count;
645 : }
646 :
647 : /**
648 : * coalesce buffers in opportunistic way without flow tables
649 : */
650 : static_always_inline u32
651 : vnet_gro_simple_inline (vlib_main_t * vm, u32 * from, u16 n_left_from,
652 : int is_l2)
653 : {
654 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
655 : vlib_get_buffers (vm, from, b, n_left_from);
656 : u32 bi = 1, ack_number = 0;
657 : if (PREDICT_TRUE (((b[0]->flags & VNET_BUFFER_F_GSO) == 0)))
658 : {
659 : while (n_left_from > 1)
660 : {
661 : if (PREDICT_TRUE (((b[bi]->flags & VNET_BUFFER_F_GSO) == 0)))
662 : {
663 : u32 ret;
664 : if ((ret =
665 : gro_coalesce_buffers (vm, b[0], b[bi], from[bi],
666 : is_l2)) != 0)
667 : {
668 : n_left_from -= 1;
669 : bi += 1;
670 : ack_number = ret;
671 : continue;
672 : }
673 : else
674 : break;
675 : }
676 : else
677 : break;
678 : }
679 :
680 : if (bi >= 2)
681 : {
682 : gro_fixup_header (vm, b[0], ack_number, is_l2);
683 : }
684 : }
685 : return bi;
686 : }
687 : #endif /* included_gro_func_h */
688 :
689 : /*
690 : * fd.io coding-style-patch-verification: ON
691 : *
692 : * Local Variables:
693 : * eval: (c-set-style "gnu")
694 : * End:
695 : */
|
