Line data Source code
1 : /*
2 : * Copyright (c) 2020 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef included_gro_func_h
17 : #define included_gro_func_h
18 :
19 : #include <vnet/ethernet/ethernet.h>
20 : #include <vnet/gso/gro.h>
21 : #include <vnet/gso/hdr_offset_parser.h>
22 : #include <vnet/ip/ip4.h>
23 : #include <vnet/ip/ip6.h>
24 : #include <vnet/ip/ip6_inlines.h>
25 : #include <vnet/udp/udp_packet.h>
26 : #include <vnet/tcp/tcp_packet.h>
27 : #include <vnet/vnet.h>
28 : #include <vnet/interface.h>
29 :
30 : #define GRO_MIN_PACKET_SIZE 256
31 : #define GRO_PADDED_PACKET_SIZE 64
32 :
33 : static_always_inline u8
34 26160308 : gro_is_bad_packet (vlib_buffer_t * b, u8 flags, i16 l234_sz)
35 : {
36 26160308 : if (((b->current_length - l234_sz) <= 0) ||
37 24311308 : ((flags &= ~(TCP_FLAG_ACK | TCP_FLAG_PSH)) != 0))
38 1849020 : return 1;
39 24311308 : return 0;
40 : }
41 :
42 : static_always_inline void
43 14003420 : gro_get_ip4_flow_from_packet (u32 * sw_if_index,
44 : ip4_header_t * ip4, tcp_header_t * tcp,
45 : gro_flow_key_t * flow_key, int is_l2)
46 : {
47 14003420 : flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX];
48 14003420 : flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX];
49 14003420 : ip46_address_set_ip4 (&flow_key->src_address, &ip4->src_address);
50 14003420 : ip46_address_set_ip4 (&flow_key->dst_address, &ip4->dst_address);
51 14003420 : flow_key->src_port = tcp->src_port;
52 14003420 : flow_key->dst_port = tcp->dst_port;
53 14003420 : }
54 :
55 : static_always_inline void
56 10307888 : gro_get_ip6_flow_from_packet (u32 * sw_if_index,
57 : ip6_header_t * ip6, tcp_header_t * tcp,
58 : gro_flow_key_t * flow_key, int is_l2)
59 : {
60 10307888 : flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX];
61 10307888 : flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX];
62 10307888 : ip46_address_set_ip6 (&flow_key->src_address, &ip6->src_address);
63 10307888 : ip46_address_set_ip6 (&flow_key->dst_address, &ip6->dst_address);
64 10307888 : flow_key->src_port = tcp->src_port;
65 10307888 : flow_key->dst_port = tcp->dst_port;
66 10307888 : }
67 :
68 : static_always_inline u32
69 50492873 : gro_is_ip4_or_ip6_packet (vlib_buffer_t *b0, u8 is_l2)
70 : {
71 50492873 : if (b0->flags & VNET_BUFFER_F_IS_IP4)
72 0 : return VNET_BUFFER_F_IS_IP4;
73 50492873 : if (b0->flags & VNET_BUFFER_F_IS_IP6)
74 2 : return VNET_BUFFER_F_IS_IP6;
75 50492871 : if (is_l2)
76 : {
77 : ethernet_header_t *eh =
78 38106444 : (ethernet_header_t *) vlib_buffer_get_current (b0);
79 38106444 : u16 ethertype = clib_net_to_host_u16 (eh->type);
80 :
81 38106444 : if (ethernet_frame_is_tagged (ethertype))
82 : {
83 0 : ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
84 :
85 0 : ethertype = clib_net_to_host_u16 (vlan->type);
86 0 : if (ethertype == ETHERNET_TYPE_VLAN)
87 : {
88 0 : vlan++;
89 0 : ethertype = clib_net_to_host_u16 (vlan->type);
90 : }
91 : }
92 38106444 : if (ethertype == ETHERNET_TYPE_IP4)
93 21872893 : return VNET_BUFFER_F_IS_IP4;
94 16233651 : if (ethertype == ETHERNET_TYPE_IP6)
95 16233549 : return VNET_BUFFER_F_IS_IP6;
96 : }
97 : else
98 : {
99 12386427 : if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x40)
100 7187881 : return VNET_BUFFER_F_IS_IP4;
101 5198556 : if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x60)
102 5198556 : return VNET_BUFFER_F_IS_IP6;
103 : }
104 :
105 44 : return 0;
106 : }
107 :
108 : typedef enum
109 : {
110 : GRO_PACKET_ACTION_NONE = 0,
111 : GRO_PACKET_ACTION_ENQUEUE = 1,
112 : GRO_PACKET_ACTION_FLUSH = 2,
113 : } gro_packet_action_t;
114 :
115 : static_always_inline gro_packet_action_t
116 23081358 : gro_tcp_sequence_check (tcp_header_t * tcp0, tcp_header_t * tcp1,
117 : u32 payload_len0)
118 : {
119 23081358 : u32 next_tcp_seq0 = clib_net_to_host_u32 (tcp0->seq_number);
120 23081358 : u32 next_tcp_seq1 = clib_net_to_host_u32 (tcp1->seq_number);
121 :
122 : /* next packet, enqueue */
123 23081358 : if (PREDICT_TRUE (next_tcp_seq0 + payload_len0 == next_tcp_seq1))
124 23081158 : return GRO_PACKET_ACTION_ENQUEUE;
125 : /* flush all packets */
126 : else
127 168 : return GRO_PACKET_ACTION_FLUSH;
128 : }
129 :
130 : static_always_inline void
131 22969655 : gro_merge_buffers (vlib_main_t * vm, vlib_buffer_t * b0,
132 : vlib_buffer_t * b1, u32 bi1, u32 payload_len1,
133 : u16 l234_sz1)
134 : {
135 22969655 : vlib_buffer_t *pb = b0;
136 :
137 22969655 : if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
138 310108 : b0->total_length_not_including_first_buffer = 0;
139 :
140 561004662 : while (pb->flags & VLIB_BUFFER_NEXT_PRESENT)
141 538034207 : pb = vlib_get_buffer (vm, pb->next_buffer);
142 :
143 22969655 : vlib_buffer_advance (b1, l234_sz1);
144 22969655 : pb->flags |= VLIB_BUFFER_NEXT_PRESENT;
145 22969655 : pb->next_buffer = bi1;
146 22969655 : b0->total_length_not_including_first_buffer += payload_len1;
147 22969655 : b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
148 22969655 : }
149 :
150 : static_always_inline u32
151 24311308 : gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0,
152 : generic_header_offset_t * gho0, int is_ip4)
153 : {
154 24311308 : u32 flags = 0;
155 :
156 24311308 : if (b0->flags & VNET_BUFFER_F_OFFLOAD)
157 0 : return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
158 24311308 : vlib_buffer_advance (b0, gho0->l3_hdr_offset);
159 24311308 : if (is_ip4)
160 14003420 : flags = ip4_tcp_udp_validate_checksum (vm, b0);
161 : else
162 10307888 : flags = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
163 24311308 : vlib_buffer_advance (b0, -gho0->l3_hdr_offset);
164 24311308 : return flags;
165 : }
166 :
167 : static_always_inline u32
168 180 : gro_fix_padded_packet_len (vlib_buffer_t *b0, generic_header_offset_t *gho0,
169 : ip4_header_t *ip4_0, ip6_header_t *ip6_0,
170 : u32 pkt_len0, u16 l234_sz0)
171 : {
172 180 : u32 tcp_payload_len0 = 0;
173 180 : if (gho0->gho_flags & GHO_F_IP4)
174 : {
175 180 : tcp_payload_len0 = clib_net_to_host_u16 (ip4_0->length) -
176 180 : ip4_header_bytes (ip4_0) - gho0->l4_hdr_sz;
177 : }
178 : else
179 : {
180 0 : tcp_payload_len0 =
181 0 : clib_net_to_host_u16 (ip6_0->payload_length) - gho0->l4_hdr_sz;
182 : }
183 :
184 180 : ASSERT (l234_sz0 + tcp_payload_len0 <= pkt_len0);
185 :
186 180 : if (PREDICT_FALSE (l234_sz0 + tcp_payload_len0 < pkt_len0))
187 : {
188 : /* small packet with padding at the end, remove padding */
189 124 : b0->current_length = l234_sz0 + tcp_payload_len0;
190 124 : pkt_len0 = b0->current_length;
191 : }
192 180 : return pkt_len0;
193 : }
194 :
195 : static_always_inline u32
196 26160814 : gro_get_packet_data (vlib_main_t *vm, vlib_buffer_t *b0,
197 : generic_header_offset_t *gho0, gro_flow_key_t *flow_key0,
198 : u8 is_l2)
199 : {
200 26160814 : ip4_header_t *ip4_0 = 0;
201 26160814 : ip6_header_t *ip6_0 = 0;
202 26160814 : tcp_header_t *tcp0 = 0;
203 26160814 : u32 flags = 0;
204 26160814 : u32 pkt_len0 = 0;
205 26160814 : u16 l234_sz0 = 0;
206 26160814 : u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 };
207 :
208 26160814 : u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
209 :
210 26160814 : if (is_ip0 & VNET_BUFFER_F_IS_IP4)
211 15026220 : vnet_generic_header_offset_parser (b0, gho0, is_l2, 1 /* is_ip4 */ ,
212 : 0 /* is_ip6 */ );
213 11134494 : else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
214 11134492 : vnet_generic_header_offset_parser (b0, gho0, is_l2, 0 /* is_ip4 */ ,
215 : 1 /* is_ip6 */ );
216 : else
217 44 : return 0;
218 :
219 26160712 : if (PREDICT_FALSE ((gho0->gho_flags & GHO_F_TCP) == 0))
220 385 : return 0;
221 :
222 26160308 : ip4_0 =
223 26160308 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset);
224 26160308 : ip6_0 =
225 26160308 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset);
226 26160308 : tcp0 =
227 26160308 : (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0->l4_hdr_offset);
228 :
229 26160308 : l234_sz0 = gho0->hdr_sz;
230 26160308 : if (PREDICT_FALSE (gro_is_bad_packet (b0, tcp0->flags, l234_sz0)))
231 1849020 : return 0;
232 :
233 24311308 : sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX];
234 24311308 : sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX];
235 :
236 24311308 : if (gho0->gho_flags & GHO_F_IP4)
237 : {
238 14003420 : flags = gro_validate_checksum (vm, b0, gho0, 1);
239 14003420 : gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, flow_key0,
240 : is_l2);
241 : }
242 10307888 : else if (gho0->gho_flags & GHO_F_IP6)
243 : {
244 10307888 : flags = gro_validate_checksum (vm, b0, gho0, 0);
245 10307888 : gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, flow_key0,
246 : is_l2);
247 : }
248 : else
249 0 : return 0;
250 :
251 24311308 : if (PREDICT_FALSE ((flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) == 0))
252 0 : return 0;
253 :
254 24311308 : pkt_len0 = vlib_buffer_length_in_chain (vm, b0);
255 24311308 : if (PREDICT_FALSE (pkt_len0 >= TCP_MAX_GSO_SZ))
256 0 : return 0;
257 :
258 24311308 : if (PREDICT_FALSE (pkt_len0 <= GRO_PADDED_PACKET_SIZE))
259 : {
260 : pkt_len0 =
261 180 : gro_fix_padded_packet_len (b0, gho0, ip4_0, ip6_0, pkt_len0, l234_sz0);
262 : }
263 24311308 : return pkt_len0;
264 : }
265 :
266 : static_always_inline u32
267 : gro_coalesce_buffers (vlib_main_t *vm, vlib_buffer_t *b0, vlib_buffer_t *b1,
268 : u32 bi1, u8 is_l2)
269 : {
270 : generic_header_offset_t gho0 = { 0 };
271 : generic_header_offset_t gho1 = { 0 };
272 : gro_flow_key_t flow_key0, flow_key1;
273 : ip4_header_t *ip4_0, *ip4_1;
274 : ip6_header_t *ip6_0, *ip6_1;
275 : tcp_header_t *tcp0, *tcp1;
276 : u16 l234_sz0, l234_sz1;
277 : u32 pkt_len0, pkt_len1, payload_len0, payload_len1;
278 : u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 };
279 : u32 sw_if_index1[VLIB_N_RX_TX] = { ~0 };
280 :
281 : u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
282 : u32 is_ip1 = gro_is_ip4_or_ip6_packet (b1, is_l2);
283 :
284 : if (is_ip0 & VNET_BUFFER_F_IS_IP4)
285 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ ,
286 : 0 /* is_ip6 */ );
287 : else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
288 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ ,
289 : 1 /* is_ip6 */ );
290 : else
291 : return 0;
292 :
293 : if (is_ip1 & VNET_BUFFER_F_IS_IP4)
294 : vnet_generic_header_offset_parser (b1, &gho1, is_l2, 1 /* is_ip4 */ ,
295 : 0 /* is_ip6 */ );
296 : else if (is_ip1 & VNET_BUFFER_F_IS_IP6)
297 : vnet_generic_header_offset_parser (b1, &gho1, is_l2, 0 /* is_ip4 */ ,
298 : 1 /* is_ip6 */ );
299 : else
300 : return 0;
301 :
302 : pkt_len0 = vlib_buffer_length_in_chain (vm, b0);
303 : pkt_len1 = vlib_buffer_length_in_chain (vm, b1);
304 :
305 : if (((gho0.gho_flags & GHO_F_TCP) == 0 || pkt_len0 <= GRO_MIN_PACKET_SIZE) ||
306 : ((gho1.gho_flags & GHO_F_TCP) == 0 || pkt_len1 <= GRO_MIN_PACKET_SIZE))
307 : return 0;
308 :
309 : ip4_0 =
310 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
311 : ip4_1 =
312 : (ip4_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset);
313 : ip6_0 =
314 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
315 : ip6_1 =
316 : (ip6_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset);
317 :
318 : tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
319 : tcp1 = (tcp_header_t *) (vlib_buffer_get_current (b1) + gho1.l4_hdr_offset);
320 :
321 : l234_sz0 = gho0.hdr_sz;
322 : l234_sz1 = gho1.hdr_sz;
323 :
324 : if (gro_is_bad_packet (b0, tcp0->flags, l234_sz0)
325 : || gro_is_bad_packet (b1, tcp1->flags, l234_sz1))
326 : return 0;
327 :
328 : sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX];
329 : sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX];
330 :
331 : sw_if_index1[VLIB_RX] = vnet_buffer (b1)->sw_if_index[VLIB_RX];
332 : sw_if_index1[VLIB_TX] = vnet_buffer (b1)->sw_if_index[VLIB_TX];
333 :
334 : if ((gho0.gho_flags & GHO_F_IP4) && (gho1.gho_flags & GHO_F_IP4))
335 : {
336 : gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, &flow_key0,
337 : is_l2);
338 : gro_get_ip4_flow_from_packet (sw_if_index1, ip4_1, tcp1, &flow_key1,
339 : is_l2);
340 : }
341 : else if ((gho0.gho_flags & GHO_F_IP6) && (gho1.gho_flags & GHO_F_IP6))
342 : {
343 : gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, &flow_key0,
344 : is_l2);
345 : gro_get_ip6_flow_from_packet (sw_if_index1, ip6_1, tcp1, &flow_key1,
346 : is_l2);
347 : }
348 : else
349 : return 0;
350 :
351 : if (gro_flow_is_equal (&flow_key0, &flow_key1) == 0)
352 : return 0;
353 :
354 : payload_len0 = pkt_len0 - l234_sz0;
355 : payload_len1 = pkt_len1 - l234_sz1;
356 :
357 : if (pkt_len0 >= TCP_MAX_GSO_SZ || pkt_len1 >= TCP_MAX_GSO_SZ
358 : || (pkt_len0 + payload_len1) >= TCP_MAX_GSO_SZ)
359 : return 0;
360 :
361 : if (gro_tcp_sequence_check (tcp0, tcp1, payload_len0) ==
362 : GRO_PACKET_ACTION_ENQUEUE)
363 : {
364 : gro_merge_buffers (vm, b0, b1, bi1, payload_len1, l234_sz1);
365 : tcp0->flags |= tcp1->flags;
366 : return tcp1->ack_number;
367 : }
368 :
369 : return 0;
370 : }
371 :
372 : static_always_inline void
373 1250821 : gro_fixup_header (vlib_main_t *vm, vlib_buffer_t *b0, u32 ack_number, u8 is_l2)
374 : {
375 1250821 : generic_header_offset_t gho0 = { 0 };
376 :
377 1250821 : u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
378 :
379 1250821 : if (is_ip0 & VNET_BUFFER_F_IS_IP4)
380 669786 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ ,
381 : 0 /* is_ip6 */ );
382 581031 : else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
383 581031 : vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ ,
384 : 1 /* is_ip6 */ );
385 :
386 1250821 : vnet_buffer2 (b0)->gso_size = b0->current_length - gho0.hdr_sz;
387 1250821 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
388 :
389 1250821 : if (gho0.gho_flags & GHO_F_IP4)
390 : {
391 669786 : ip4_header_t *ip4 =
392 669786 : (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
393 669786 : ip4->length =
394 669786 : clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
395 669786 : gho0.l3_hdr_offset);
396 669786 : vnet_buffer (b0)->l3_hdr_offset = (u8 *) ip4 - b0->data;
397 669786 : b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
398 669786 : vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
399 : VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
400 : }
401 581031 : else if (gho0.gho_flags & GHO_F_IP6)
402 : {
403 581031 : ip6_header_t *ip6 =
404 581031 : (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
405 581031 : ip6->payload_length =
406 581031 : clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
407 581031 : gho0.l4_hdr_offset);
408 581031 : vnet_buffer (b0)->l3_hdr_offset = (u8 *) ip6 - b0->data;
409 581031 : b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
410 581031 : vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
411 : }
412 :
413 1250821 : tcp_header_t *tcp0 =
414 1250821 : (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
415 1250821 : vnet_buffer (b0)->l4_hdr_offset = (u8 *) tcp0 - b0->data;
416 1250821 : vnet_buffer2 (b0)->gso_l4_hdr_sz = tcp_header_bytes (tcp0);
417 1250821 : tcp0->ack_number = ack_number;
418 1250821 : b0->flags &= ~VLIB_BUFFER_IS_TRACED;
419 1250821 : }
420 :
421 : static_always_inline u32
422 15703305 : vnet_gro_flow_table_flush (vlib_main_t * vm, gro_flow_table_t * flow_table,
423 : u32 * to)
424 : {
425 15703305 : if (flow_table->flow_table_size > 0)
426 : {
427 : gro_flow_t *gro_flow;
428 172919 : u32 i = 0, j = 0;
429 2939627 : while (i < GRO_FLOW_TABLE_MAX_SIZE)
430 : {
431 2766706 : gro_flow = &flow_table->gro_flow[i];
432 2766706 : if (gro_flow->n_buffers && gro_flow_is_timeout (vm, gro_flow))
433 : {
434 : // flush the packet
435 : vlib_buffer_t *b0 =
436 172849 : vlib_get_buffer (vm, gro_flow->buffer_index);
437 172849 : gro_fixup_header (vm, b0, gro_flow->last_ack_number,
438 172849 : flow_table->is_l2);
439 172849 : to[j] = gro_flow->buffer_index;
440 172849 : gro_flow_table_reset_flow (flow_table, gro_flow);
441 172849 : flow_table->n_vectors++;
442 172849 : j++;
443 : }
444 2766706 : i++;
445 : }
446 :
447 172919 : return j;
448 : }
449 15530404 : return 0;
450 : }
451 :
452 : static_always_inline void
453 159937005 : vnet_gro_flow_table_schedule_node_on_dispatcher (vlib_main_t *vm,
454 : vnet_hw_if_tx_queue_t *txq,
455 : gro_flow_table_t *flow_table)
456 : {
457 159937005 : if (gro_flow_table_is_timeout (vm, flow_table))
458 : {
459 15703305 : u32 to[GRO_FLOW_TABLE_MAX_SIZE] = { 0 };
460 15703305 : u32 n_to = vnet_gro_flow_table_flush (vm, flow_table, to);
461 :
462 15703305 : if (n_to > 0)
463 : {
464 172849 : u32 node_index = flow_table->node_index;
465 172849 : vlib_frame_t *f = vlib_get_frame_to_node (vm, node_index);
466 172849 : vnet_hw_if_tx_frame_t *ft = vlib_frame_scalar_args (f);
467 172849 : u32 *f_to = vlib_frame_vector_args (f);
468 172849 : u32 i = 0;
469 :
470 172849 : ft->shared_queue = txq->shared_queue;
471 172849 : ft->queue_id = txq->queue_id;
472 :
473 345698 : while (i < n_to)
474 : {
475 172849 : f_to[f->n_vectors] = to[i];
476 172849 : i++;
477 172849 : f->n_vectors++;
478 : }
479 172849 : vlib_put_frame_to_node (vm, node_index, f);
480 : }
481 15703305 : gro_flow_table_set_timeout (vm, flow_table, GRO_FLOW_TABLE_FLUSH);
482 : }
483 159937005 : }
484 :
485 : static_always_inline u32
486 70313 : vnet_gro_flush_all_packets (vlib_main_t *vm, gro_flow_table_t *flow_table,
487 : gro_flow_t *gro_flow, vlib_buffer_t *b_s, u32 *to,
488 : u32 bi_s, u32 bi0, u8 is_l2)
489 : {
490 70313 : flow_table->n_vectors++;
491 70313 : flow_table->total_vectors++;
492 70313 : gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
493 70313 : gro_flow->n_buffers = 0;
494 70313 : gro_flow_table_reset_flow (flow_table, gro_flow);
495 70313 : to[0] = bi_s;
496 70313 : to[1] = bi0;
497 70313 : return 2;
498 : }
499 :
500 : static_always_inline u32
501 26333615 : vnet_gro_flow_table_inline (vlib_main_t * vm, gro_flow_table_t * flow_table,
502 : u32 bi0, u32 * to)
503 : {
504 26333615 : vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
505 26333615 : generic_header_offset_t gho0 = { 0 };
506 26333615 : gro_flow_t *gro_flow = 0;
507 26333615 : gro_flow_key_t flow_key0 = { };
508 26333615 : tcp_header_t *tcp0 = 0;
509 26333615 : u32 pkt_len0 = 0;
510 26333615 : u32 is_flush = 0;
511 26333615 : u8 is_l2 = flow_table->is_l2;
512 :
513 26333615 : if (!gro_flow_table_is_enable (flow_table))
514 : {
515 0 : to[0] = bi0;
516 0 : return 1;
517 : }
518 :
519 26333615 : if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
520 : {
521 172849 : to[0] = bi0;
522 172849 : return 1;
523 : }
524 :
525 26160814 : pkt_len0 = gro_get_packet_data (vm, b0, &gho0, &flow_key0, is_l2);
526 26160814 : if (pkt_len0 == 0)
527 : {
528 1849456 : to[0] = bi0;
529 1849456 : return 1;
530 : }
531 :
532 24311308 : tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
533 24311308 : if (PREDICT_TRUE (((tcp0->flags & TCP_FLAG_PSH) == 0) &&
534 : (pkt_len0 > GRO_MIN_PACKET_SIZE)))
535 23254359 : gro_flow = gro_flow_table_find_or_add_flow (flow_table, &flow_key0);
536 : else
537 : {
538 1056969 : is_flush = 1;
539 1056969 : gro_flow = gro_flow_table_get_flow (flow_table, &flow_key0);
540 : }
541 :
542 24311308 : if (!gro_flow)
543 : {
544 20520 : to[0] = bi0;
545 20520 : return 1;
546 : }
547 :
548 24290808 : if (PREDICT_FALSE (gro_flow->n_buffers == 0))
549 : {
550 1209470 : flow_table->total_vectors++;
551 1209470 : gro_flow_store_packet (gro_flow, bi0);
552 1209470 : gro_flow->last_ack_number = tcp0->ack_number;
553 1209470 : gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT);
554 1209470 : return 0;
555 : }
556 : else
557 : {
558 23081358 : generic_header_offset_t gho_s = { 0 };
559 : tcp_header_t *tcp_s;
560 : u16 l234_sz0, l234_sz_s;
561 : u32 pkt_len_s, payload_len0, payload_len_s;
562 23081358 : u32 bi_s = gro_flow->buffer_index;
563 :
564 23081358 : vlib_buffer_t *b_s = vlib_get_buffer (vm, bi_s);
565 23081358 : u32 is_ip_s = gro_is_ip4_or_ip6_packet (b_s, is_l2);
566 23081358 : if (is_ip_s & VNET_BUFFER_F_IS_IP4)
567 13364771 : vnet_generic_header_offset_parser (b_s, &gho_s, is_l2,
568 : 1 /* is_ip4 */ , 0 /* is_ip6 */ );
569 9716607 : else if (is_ip_s & VNET_BUFFER_F_IS_IP6)
570 9716607 : vnet_generic_header_offset_parser (b_s, &gho_s, is_l2,
571 : 0 /* is_ip4 */ , 1 /* is_ip6 */ );
572 :
573 23081358 : tcp_s =
574 23081358 : (tcp_header_t *) (vlib_buffer_get_current (b_s) +
575 23081358 : gho_s.l4_hdr_offset);
576 23081358 : pkt_len_s = vlib_buffer_length_in_chain (vm, b_s);
577 23081358 : l234_sz0 = gho0.hdr_sz;
578 23081358 : l234_sz_s = gho_s.hdr_sz;
579 23081358 : payload_len0 = pkt_len0 - l234_sz0;
580 23081358 : payload_len_s = pkt_len_s - l234_sz_s;
581 : gro_packet_action_t action =
582 23081358 : gro_tcp_sequence_check (tcp_s, tcp0, payload_len_s);
583 :
584 23081358 : if (PREDICT_TRUE (action == GRO_PACKET_ACTION_ENQUEUE))
585 : {
586 23081158 : if (PREDICT_TRUE (((pkt_len_s + payload_len0) < TCP_MAX_GSO_SZ) &&
587 : (gro_flow->n_buffers < GRO_FLOW_N_BUFFERS)))
588 : {
589 22969655 : flow_table->total_vectors++;
590 22969655 : gro_merge_buffers (vm, b_s, b0, bi0, payload_len0, l234_sz0);
591 22969655 : gro_flow_store_packet (gro_flow, bi0);
592 22969655 : gro_flow->last_ack_number = tcp0->ack_number;
593 22969655 : if (PREDICT_FALSE (is_flush))
594 : {
595 966303 : flow_table->n_vectors++;
596 966303 : tcp_s->flags |= tcp0->flags;
597 966303 : gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
598 966303 : gro_flow->n_buffers = 0;
599 966303 : gro_flow_table_reset_flow (flow_table, gro_flow);
600 966303 : to[0] = bi_s;
601 966303 : return 1;
602 : }
603 22003406 : return 0;
604 : }
605 111497 : else if (PREDICT_FALSE (is_flush))
606 : // flush the all (current and stored) packets
607 70145 : return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s,
608 : to, bi_s, bi0, is_l2);
609 : else
610 : {
611 : // flush the stored GSO size packet and buffer the current packet
612 41352 : flow_table->n_vectors++;
613 41352 : flow_table->total_vectors++;
614 41352 : gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
615 41352 : gro_flow->n_buffers = 0;
616 41352 : gro_flow_store_packet (gro_flow, bi0);
617 41352 : gro_flow->last_ack_number = tcp0->ack_number;
618 41352 : gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT);
619 41352 : to[0] = bi_s;
620 41352 : return 1;
621 : }
622 : }
623 : else
624 : {
625 : // flush the all (current and stored) packets
626 168 : return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s, to,
627 : bi_s, bi0, is_l2);
628 : }
629 : }
630 : }
631 :
632 : /**
633 : * coalesce buffers with flow tables
634 : */
635 : static_always_inline u32
636 517697 : vnet_gro_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, u32 * from,
637 : u16 n_left_from, u32 * to)
638 : {
639 517697 : u16 count = 0, i = 0;
640 :
641 26851326 : for (i = 0; i < n_left_from; i++)
642 26333615 : count += vnet_gro_flow_table_inline (vm, flow_table, from[i], &to[count]);
643 :
644 517697 : return count;
645 : }
646 :
647 : /**
648 : * coalesce buffers in opportunistic way without flow tables
649 : */
650 : static_always_inline u32
651 : vnet_gro_simple_inline (vlib_main_t * vm, u32 * from, u16 n_left_from,
652 : int is_l2)
653 : {
654 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
655 : vlib_get_buffers (vm, from, b, n_left_from);
656 : u32 bi = 1, ack_number = 0;
657 : if (PREDICT_TRUE (((b[0]->flags & VNET_BUFFER_F_GSO) == 0)))
658 : {
659 : while (n_left_from > 1)
660 : {
661 : if (PREDICT_TRUE (((b[bi]->flags & VNET_BUFFER_F_GSO) == 0)))
662 : {
663 : u32 ret;
664 : if ((ret =
665 : gro_coalesce_buffers (vm, b[0], b[bi], from[bi],
666 : is_l2)) != 0)
667 : {
668 : n_left_from -= 1;
669 : bi += 1;
670 : ack_number = ret;
671 : continue;
672 : }
673 : else
674 : break;
675 : }
676 : else
677 : break;
678 : }
679 :
680 : if (bi >= 2)
681 : {
682 : gro_fixup_header (vm, b[0], ack_number, is_l2);
683 : }
684 : }
685 : return bi;
686 : }
687 : #endif /* included_gro_func_h */
688 :
689 : /*
690 : * fd.io coding-style-patch-verification: ON
691 : *
692 : * Local Variables:
693 : * eval: (c-set-style "gnu")
694 : * End:
695 : */
|
