Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * interface_output.c: interface output node
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #include <vnet/vnet.h>
41 : #include <vnet/ip/icmp46_packet.h>
42 : #include <vnet/ethernet/packet.h>
43 : #include <vnet/ip/format.h>
44 : #include <vnet/ip/ip4.h>
45 : #include <vnet/ip/ip6.h>
46 : #include <vnet/udp/udp_packet.h>
47 : #include <vnet/feature/feature.h>
48 : #include <vnet/classify/pcap_classify.h>
49 : #include <vnet/hash/hash.h>
50 : #include <vnet/interface_output.h>
51 : #include <vppinfra/vector/mask_compare.h>
52 : #include <vppinfra/vector/compress.h>
53 : #include <vppinfra/vector/count_equal.h>
54 : #include <vppinfra/vector/array_mask.h>
55 :
56 : typedef struct
57 : {
58 : u32 sw_if_index;
59 : u32 flags;
60 : u8 data[128 - 2 * sizeof (u32)];
61 : }
62 : interface_output_trace_t;
63 :
64 : #ifndef CLIB_MARCH_VARIANT
65 : u8 *
66 262874 : format_vnet_interface_output_trace (u8 * s, va_list * va)
67 : {
68 262874 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
69 262874 : vlib_node_t *node = va_arg (*va, vlib_node_t *);
70 262874 : interface_output_trace_t *t = va_arg (*va, interface_output_trace_t *);
71 262874 : vnet_main_t *vnm = vnet_get_main ();
72 : vnet_sw_interface_t *si;
73 : u32 indent;
74 :
75 262874 : if (t->sw_if_index != (u32) ~ 0)
76 : {
77 262874 : indent = format_get_indent (s);
78 :
79 262874 : if (pool_is_free_index
80 262874 : (vnm->interface_main.sw_interfaces, t->sw_if_index))
81 : {
82 : /* the interface may have been deleted by the time the trace is printed */
83 596 : s = format (s, "sw_if_index: %d ", t->sw_if_index);
84 : }
85 : else
86 : {
87 262278 : si = vnet_get_sw_interface (vnm, t->sw_if_index);
88 262278 : s = format (s, "%U flags 0x%08x", format_vnet_sw_interface_name, vnm,
89 : si, t->flags);
90 : }
91 : s =
92 262874 : format (s, "\n%U%U", format_white_space, indent,
93 262874 : node->format_buffer ? node->format_buffer : format_hex_bytes,
94 262874 : t->data, sizeof (t->data));
95 : }
96 262874 : return s;
97 : }
98 : #endif /* CLIB_MARCH_VARIANT */
99 :
100 : static void
101 12459 : vnet_interface_output_trace (vlib_main_t * vm,
102 : vlib_node_runtime_t * node,
103 : vlib_frame_t * frame, uword n_buffers)
104 : {
105 : u32 n_left, *from;
106 :
107 12459 : n_left = n_buffers;
108 12459 : from = vlib_frame_vector_args (frame);
109 :
110 195602 : while (n_left >= 4)
111 : {
112 : u32 bi0, bi1;
113 : vlib_buffer_t *b0, *b1;
114 : interface_output_trace_t *t0, *t1;
115 :
116 : /* Prefetch next iteration. */
117 183143 : vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
118 183143 : vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
119 :
120 183143 : bi0 = from[0];
121 183143 : bi1 = from[1];
122 :
123 183143 : b0 = vlib_get_buffer (vm, bi0);
124 183143 : b1 = vlib_get_buffer (vm, bi1);
125 :
126 183143 : if (b0->flags & VLIB_BUFFER_IS_TRACED)
127 : {
128 182554 : t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
129 182554 : t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
130 182554 : t0->flags = b0->flags;
131 182554 : clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
132 : sizeof (t0->data));
133 : }
134 183142 : if (b1->flags & VLIB_BUFFER_IS_TRACED)
135 : {
136 182552 : t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
137 182554 : t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX];
138 182554 : t1->flags = b1->flags;
139 182554 : clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1),
140 : sizeof (t1->data));
141 : }
142 183143 : from += 2;
143 183143 : n_left -= 2;
144 : }
145 :
146 35901 : while (n_left >= 1)
147 : {
148 : u32 bi0;
149 : vlib_buffer_t *b0;
150 : interface_output_trace_t *t0;
151 :
152 23441 : bi0 = from[0];
153 :
154 23441 : b0 = vlib_get_buffer (vm, bi0);
155 :
156 23441 : if (b0->flags & VLIB_BUFFER_IS_TRACED)
157 : {
158 23333 : t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
159 23334 : t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
160 23334 : t0->flags = b0->flags;
161 23334 : clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
162 : sizeof (t0->data));
163 : }
164 23442 : from += 1;
165 23442 : n_left -= 1;
166 : }
167 12460 : }
168 :
169 : static_always_inline void
170 7852160 : vnet_interface_output_handle_offload (vlib_main_t *vm, vlib_buffer_t *b)
171 : {
172 7852160 : if (b->flags & VNET_BUFFER_F_GSO)
173 8 : return;
174 7852150 : vnet_calc_checksums_inline (vm, b, b->flags & VNET_BUFFER_F_IS_IP4,
175 7852150 : b->flags & VNET_BUFFER_F_IS_IP6);
176 7852150 : vnet_calc_outer_checksums_inline (vm, b);
177 : }
178 :
179 : static_always_inline uword
180 1903060 : vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index,
181 : vlib_combined_counter_main_t *ccm,
182 : vlib_buffer_t **b, void **p,
183 : u32 config_index, u8 arc, u32 n_left,
184 : int processing_level)
185 : {
186 1903060 : u32 n_bytes = 0;
187 : u32 n_bytes0, n_bytes1, n_bytes2, n_bytes3;
188 1903060 : u32 ti = vm->thread_index;
189 :
190 18084000 : while (n_left >= 8)
191 : {
192 : u32 or_flags;
193 :
194 : /* Prefetch next iteration. */
195 16180900 : vlib_prefetch_buffer_header (b[4], LOAD);
196 16180900 : vlib_prefetch_buffer_header (b[5], LOAD);
197 16180900 : vlib_prefetch_buffer_header (b[6], LOAD);
198 16180900 : vlib_prefetch_buffer_header (b[7], LOAD);
199 :
200 16180900 : if (processing_level >= 1)
201 15288100 : or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
202 :
203 : /* Be grumpy about zero length buffers for benefit of
204 : driver tx function. */
205 16180900 : ASSERT (b[0]->current_length > 0);
206 16180900 : ASSERT (b[1]->current_length > 0);
207 16180900 : ASSERT (b[2]->current_length > 0);
208 16180900 : ASSERT (b[3]->current_length > 0);
209 :
210 16180900 : n_bytes += n_bytes0 = vlib_buffer_length_in_chain (vm, b[0]);
211 16180900 : n_bytes += n_bytes1 = vlib_buffer_length_in_chain (vm, b[1]);
212 16180900 : n_bytes += n_bytes2 = vlib_buffer_length_in_chain (vm, b[2]);
213 16180900 : n_bytes += n_bytes3 = vlib_buffer_length_in_chain (vm, b[3]);
214 :
215 16180900 : if (processing_level >= 3)
216 : {
217 0 : p[0] = vlib_buffer_get_current (b[0]);
218 0 : p[1] = vlib_buffer_get_current (b[1]);
219 0 : p[2] = vlib_buffer_get_current (b[2]);
220 0 : p[3] = vlib_buffer_get_current (b[3]);
221 0 : p += 4;
222 : }
223 :
224 16180900 : if (processing_level >= 2)
225 : {
226 : u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3;
227 1642 : tx_swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
228 1642 : tx_swif1 = vnet_buffer (b[1])->sw_if_index[VLIB_TX];
229 1642 : tx_swif2 = vnet_buffer (b[2])->sw_if_index[VLIB_TX];
230 1642 : tx_swif3 = vnet_buffer (b[3])->sw_if_index[VLIB_TX];
231 :
232 : /* update vlan subif tx counts, if required */
233 1642 : if (PREDICT_FALSE (tx_swif0 != sw_if_index))
234 1009 : vlib_increment_combined_counter (ccm, ti, tx_swif0, 1, n_bytes0);
235 :
236 1642 : if (PREDICT_FALSE (tx_swif1 != sw_if_index))
237 1009 : vlib_increment_combined_counter (ccm, ti, tx_swif1, 1, n_bytes1);
238 :
239 1642 : if (PREDICT_FALSE (tx_swif2 != sw_if_index))
240 1009 : vlib_increment_combined_counter (ccm, ti, tx_swif2, 1, n_bytes2);
241 :
242 1642 : if (PREDICT_FALSE (tx_swif3 != sw_if_index))
243 1009 : vlib_increment_combined_counter (ccm, ti, tx_swif3, 1, n_bytes3);
244 :
245 1642 : if (PREDICT_FALSE (config_index != ~0))
246 : {
247 1 : vnet_buffer (b[0])->feature_arc_index = arc;
248 1 : b[0]->current_config_index = config_index;
249 1 : vnet_buffer (b[1])->feature_arc_index = arc;
250 1 : b[1]->current_config_index = config_index;
251 1 : vnet_buffer (b[2])->feature_arc_index = arc;
252 1 : b[2]->current_config_index = config_index;
253 1 : vnet_buffer (b[3])->feature_arc_index = arc;
254 1 : b[3]->current_config_index = config_index;
255 : }
256 : }
257 :
258 16180900 : if (processing_level >= 1 && (or_flags & VNET_BUFFER_F_OFFLOAD))
259 : {
260 90286 : vnet_interface_output_handle_offload (vm, b[0]);
261 90286 : vnet_interface_output_handle_offload (vm, b[1]);
262 90286 : vnet_interface_output_handle_offload (vm, b[2]);
263 90286 : vnet_interface_output_handle_offload (vm, b[3]);
264 : }
265 :
266 16180900 : n_left -= 4;
267 16180900 : b += 4;
268 : }
269 :
270 10954500 : while (n_left)
271 : {
272 : /* Be grumpy about zero length buffers for benefit of
273 : driver tx function. */
274 9051420 : ASSERT (b[0]->current_length > 0);
275 :
276 9051420 : n_bytes += n_bytes0 = vlib_buffer_length_in_chain (vm, b[0]);
277 :
278 9051420 : if (processing_level >= 3)
279 : {
280 0 : p[0] = vlib_buffer_get_current (b[0]);
281 0 : p += 1;
282 : }
283 :
284 9051420 : if (processing_level >= 2)
285 : {
286 769 : u32 tx_swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
287 :
288 769 : if (PREDICT_FALSE (config_index != ~0))
289 : {
290 20 : vnet_buffer (b[0])->feature_arc_index = arc;
291 20 : b[0]->current_config_index = config_index;
292 : }
293 :
294 769 : if (PREDICT_FALSE (tx_swif0 != sw_if_index))
295 477 : vlib_increment_combined_counter (ccm, ti, tx_swif0, 1, n_bytes0);
296 : }
297 :
298 9051420 : if (processing_level >= 1)
299 7491010 : vnet_interface_output_handle_offload (vm, b[0]);
300 :
301 9051420 : n_left -= 1;
302 9051420 : b += 1;
303 : }
304 :
305 1903060 : return n_bytes;
306 : }
307 :
308 : static_always_inline void
309 2675950 : vnet_interface_pcap_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node,
310 : vlib_frame_t *frame, int in_interface_ouput)
311 : {
312 2675950 : vnet_main_t *vnm = vnet_get_main ();
313 : u32 n_left_from, *from;
314 2675950 : u32 sw_if_index = ~0, hw_if_index = ~0;
315 2675950 : vnet_pcap_t *pp = &vnm->pcap;
316 :
317 2675950 : if (PREDICT_TRUE (pp->pcap_tx_enable == 0))
318 2675940 : return;
319 :
320 9 : if (in_interface_ouput)
321 : {
322 : /* interface-output is called right before interface-output-template.
323 : * We only want to capture packets here if there is a per-interface
324 : * filter, in case it matches the sub-interface sw_if_index.
325 : * If there is no per-interface filter configured, let the
326 : * interface-output-template node deal with it */
327 0 : if (pp->pcap_sw_if_index == 0)
328 0 : return;
329 : }
330 : else
331 : {
332 9 : vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
333 9 : sw_if_index = rt->sw_if_index;
334 : }
335 :
336 9 : n_left_from = frame->n_vectors;
337 9 : from = vlib_frame_vector_args (frame);
338 :
339 188 : while (n_left_from > 0)
340 : {
341 179 : u32 bi0 = from[0];
342 179 : vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
343 179 : from++;
344 179 : n_left_from--;
345 :
346 179 : if (in_interface_ouput)
347 : {
348 0 : const u32 sii = vnet_buffer (b0)->sw_if_index[VLIB_TX];
349 0 : if (PREDICT_FALSE (sii != sw_if_index))
350 : {
351 : const vnet_hw_interface_t *hi =
352 0 : vnet_get_sup_hw_interface (vnm, sii);
353 0 : hw_if_index = hi->sw_if_index;
354 0 : sw_if_index = sii;
355 : }
356 0 : if (hw_if_index == sw_if_index)
357 0 : continue; /* defer to interface-output-template */
358 : }
359 :
360 179 : if (vnet_is_packet_pcaped (pp, b0, sw_if_index))
361 61 : pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt);
362 : }
363 : }
364 :
365 : static_always_inline void
366 0 : hash_func_with_mask (void **p, u32 *hash, u32 n_packets, u32 *lookup_table,
367 : u32 mask, vnet_hash_fn_t hf)
368 : {
369 0 : u32 n_left_from = n_packets;
370 :
371 0 : hf (p, hash, n_packets);
372 :
373 0 : clib_array_mask_u32 (hash, mask, n_packets);
374 :
375 0 : while (n_left_from >= 4)
376 : {
377 0 : hash[0] = lookup_table[hash[0]];
378 0 : hash[1] = lookup_table[hash[1]];
379 0 : hash[2] = lookup_table[hash[2]];
380 0 : hash[3] = lookup_table[hash[3]];
381 :
382 0 : hash += 4;
383 0 : n_left_from -= 4;
384 : }
385 :
386 0 : while (n_left_from > 0)
387 : {
388 0 : hash[0] = lookup_table[hash[0]];
389 :
390 0 : hash += 1;
391 0 : n_left_from -= 1;
392 : }
393 0 : }
394 :
395 : static_always_inline void
396 1902400 : store_tx_frame_scalar_data (vnet_hw_if_tx_frame_t *copy_frame,
397 : vnet_hw_if_tx_frame_t *tf)
398 : {
399 1902400 : if (copy_frame)
400 1885300 : clib_memcpy_fast (tf, copy_frame, sizeof (vnet_hw_if_tx_frame_t));
401 1902400 : }
402 :
403 : static_always_inline u32
404 1903060 : enqueue_one_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ppqi,
405 : u32 *from, vnet_hw_if_tx_frame_t *copy_frame,
406 : u32 n_vectors, u32 n_left, u32 next_index)
407 : {
408 : u32 tmp[VLIB_FRAME_SIZE];
409 1903060 : vlib_frame_bitmap_t mask = {};
410 : vlib_frame_t *f;
411 : vnet_hw_if_tx_frame_t *tf;
412 : u32 *to;
413 1903060 : u32 n_copy = 0, n_free = 0;
414 :
415 1903060 : f = vlib_get_next_frame_internal (vm, node, next_index, 0);
416 1903050 : tf = vlib_frame_scalar_args (f);
417 :
418 1903050 : if (f->n_vectors > 0 &&
419 0 : (!copy_frame || (tf->queue_id == copy_frame->queue_id)))
420 : {
421 : /* append current next frame */
422 657 : n_free = VLIB_FRAME_SIZE - f->n_vectors;
423 : /*
424 : * if frame contains enough space for worst case scenario,
425 : * we can avoid use of tmp
426 : */
427 657 : if (n_free >= n_left)
428 657 : to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
429 : else
430 0 : to = tmp;
431 : }
432 : else
433 : {
434 1902400 : if (f->n_vectors > 0)
435 : {
436 : /* current frame doesn't fit - grab empty one */
437 0 : f = vlib_get_next_frame_internal (vm, node, next_index, 1);
438 0 : tf = vlib_frame_scalar_args (f);
439 : }
440 :
441 : /* empty frame - store scalar data */
442 1902400 : store_tx_frame_scalar_data (copy_frame, tf);
443 1902400 : to = vlib_frame_vector_args (f);
444 1902400 : n_free = VLIB_FRAME_SIZE;
445 : }
446 :
447 : /*
448 : * per packet queue id array
449 : * compare with given queue_id, if match, copy respective buffer index from
450 : * -> to
451 : */
452 1903050 : if (ppqi)
453 : {
454 0 : clib_mask_compare_u32 (copy_frame->queue_id, ppqi, mask, n_vectors);
455 0 : n_copy = clib_compress_u32 (to, from, mask, n_vectors);
456 :
457 0 : if (n_copy == 0)
458 0 : return n_left;
459 : }
460 : else
461 : {
462 : /*
463 : * no work required, just copy all buffer indices from -> to
464 : */
465 1903050 : n_copy = n_left;
466 1903050 : vlib_buffer_copy_indices (to, from, n_copy);
467 : }
468 :
469 1903060 : if (to != tmp)
470 : {
471 : /* indices already written to frame, just close it */
472 1903060 : vlib_put_next_frame (vm, node, next_index, n_free - n_copy);
473 : }
474 0 : else if (n_free >= n_copy)
475 : {
476 : /* enough space in the existing frame */
477 0 : to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
478 0 : vlib_buffer_copy_indices (to, tmp, n_copy);
479 0 : vlib_put_next_frame (vm, node, next_index, n_free - n_copy);
480 : }
481 : else
482 : {
483 : /* full frame */
484 0 : to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
485 0 : vlib_buffer_copy_indices (to, tmp, n_free);
486 0 : vlib_put_next_frame (vm, node, next_index, 0);
487 :
488 : /* second frame */
489 0 : u32 n_2nd_frame = n_copy - n_free;
490 0 : f = vlib_get_next_frame_internal (vm, node, next_index, 1);
491 0 : tf = vlib_frame_scalar_args (f);
492 : /* empty frame - store scalar data */
493 0 : store_tx_frame_scalar_data (copy_frame, tf);
494 0 : to = vlib_frame_vector_args (f);
495 0 : vlib_buffer_copy_indices (to, tmp + n_free, n_2nd_frame);
496 0 : vlib_put_next_frame (vm, node, next_index,
497 : VLIB_FRAME_SIZE - n_2nd_frame);
498 : }
499 :
500 1903060 : return n_left - n_copy;
501 : }
502 :
503 : static_always_inline void
504 1903060 : enqueue_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node,
505 : vnet_hw_interface_t *hi, u32 next_index,
506 : vnet_hw_if_output_node_runtime_t *r, u32 *from, void **p,
507 : u32 n_vectors)
508 : {
509 1903060 : u32 n_left = n_vectors;
510 :
511 1903060 : ASSERT (n_vectors <= VLIB_FRAME_SIZE);
512 :
513 : /*
514 : * backward compatible for drivers not integrated with new tx infra.
515 : */
516 1903060 : if (r == 0)
517 : {
518 17758 : n_left = enqueue_one_to_tx_node (vm, node, NULL, from, NULL, n_vectors,
519 : n_left, next_index);
520 : }
521 : /*
522 : * only 1 tx queue of given interface is available on given thread
523 : */
524 1885300 : else if (r->n_queues == 1)
525 : {
526 1885300 : n_left = enqueue_one_to_tx_node (vm, node, NULL, from, r->frame,
527 : n_vectors, n_left, next_index);
528 : }
529 : /*
530 : * multi tx-queues use case
531 : */
532 0 : else if (r->n_queues > 1)
533 : {
534 : u32 qids[VLIB_FRAME_SIZE];
535 :
536 0 : hash_func_with_mask (p, qids, n_vectors, r->lookup_table,
537 0 : vec_len (r->lookup_table) - 1, hi->hf);
538 :
539 0 : for (u32 i = 0; i < r->n_queues; i++)
540 : {
541 0 : n_left = enqueue_one_to_tx_node (vm, node, qids, from, &r->frame[i],
542 : n_vectors, n_left, next_index);
543 0 : if (n_left == 0)
544 0 : break;
545 : }
546 : }
547 : else
548 0 : ASSERT (0);
549 1903060 : }
550 :
551 1905306 : VLIB_NODE_FN (vnet_interface_output_node)
552 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
553 : {
554 1903070 : vnet_main_t *vnm = vnet_get_main ();
555 1903070 : vnet_interface_main_t *im = &vnm->interface_main;
556 : vlib_combined_counter_main_t *ccm;
557 : vnet_hw_interface_t *hi;
558 : vnet_sw_interface_t *si;
559 1903070 : vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
560 1903070 : vnet_hw_if_output_node_runtime_t *r = 0;
561 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
562 1903070 : u32 n_bytes, n_buffers = frame->n_vectors;
563 1903070 : u32 config_index = ~0;
564 1903070 : u32 sw_if_index = rt->sw_if_index;
565 1903070 : u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX;
566 1903070 : u32 ti = vm->thread_index;
567 1903070 : u8 arc = im->output_feature_arc_index;
568 1903070 : int arc_or_subif = 0;
569 1903070 : int do_tx_offloads = 0;
570 1903070 : void *ptr[VLIB_FRAME_SIZE], **p = ptr;
571 1903070 : u8 is_parr = 0;
572 : u32 *from;
573 :
574 1903070 : if (node->flags & VLIB_NODE_FLAG_TRACE)
575 12459 : vnet_interface_output_trace (vm, node, frame, n_buffers);
576 :
577 1903070 : from = vlib_frame_vector_args (frame);
578 :
579 1903070 : if (rt->is_deleted)
580 0 : return vlib_error_drop_buffers (
581 : vm, node, from,
582 : /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP,
583 : node->node_index, VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DELETED);
584 :
585 1903070 : vnet_interface_pcap_tx_trace (vm, node, frame, 0 /* in_interface_ouput */);
586 :
587 1903070 : vlib_get_buffers (vm, from, bufs, n_buffers);
588 :
589 1903070 : si = vnet_get_sw_interface (vnm, sw_if_index);
590 1903070 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
591 :
592 1903070 : if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ||
593 1903060 : !(hi->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
594 : {
595 : vlib_simple_counter_main_t *cm;
596 :
597 16 : cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
598 : VNET_INTERFACE_COUNTER_TX_ERROR);
599 16 : vlib_increment_simple_counter (cm, ti, sw_if_index, n_buffers);
600 :
601 16 : return vlib_error_drop_buffers (
602 : vm, node, from,
603 : /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP,
604 : node->node_index, VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DOWN);
605 : }
606 :
607 1903060 : if (hi->output_node_thread_runtimes)
608 1885300 : r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index);
609 :
610 1903060 : if (r)
611 : {
612 : /*
613 : * tx queue of given interface is not available on given thread
614 : */
615 1885300 : if (r->n_queues == 0)
616 0 : return vlib_error_drop_buffers (
617 : vm, node, from,
618 : /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP,
619 : node->node_index, VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE);
620 : /*
621 : * multiple tx queues available on given thread
622 : */
623 1885300 : else if (r->n_queues > 1)
624 : /* construct array of pointer */
625 0 : is_parr = 1;
626 : }
627 :
628 : /* interface-output feature arc handling */
629 1903060 : if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index)))
630 : {
631 : vnet_feature_config_main_t *fcm;
632 9 : fcm = vnet_feature_get_config_main (arc);
633 9 : config_index = vnet_get_feature_config_index (arc, sw_if_index);
634 9 : vnet_get_config_data (&fcm->config_main, &config_index, &next_index, 0);
635 9 : arc_or_subif = 1;
636 : }
637 1903050 : else if (hash_elts (hi->sub_interface_sw_if_index_by_id))
638 346 : arc_or_subif = 1;
639 :
640 1903060 : ccm = im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX;
641 :
642 : /* if not all three flags IP4_,TCP_,UDP_CKSUM set, do compute them
643 : * here before sending to the interface */
644 1903060 : if ((hi->caps & VNET_HW_IF_CAP_TX_CKSUM) != VNET_HW_IF_CAP_TX_CKSUM)
645 1459780 : do_tx_offloads = 1;
646 :
647 : // basic processing
648 1903060 : if (do_tx_offloads == 0 && arc_or_subif == 0 && is_parr == 0)
649 443280 : n_bytes = vnet_interface_output_node_inline (
650 : vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 0);
651 : // basic processing + tx offloads
652 1459780 : else if (do_tx_offloads == 1 && arc_or_subif == 0 && is_parr == 0)
653 1459420 : n_bytes = vnet_interface_output_node_inline (
654 : vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 1);
655 : // basic processing + tx offloads + vlans + arcs
656 355 : else if (do_tx_offloads == 1 && arc_or_subif == 1 && is_parr == 0)
657 355 : n_bytes = vnet_interface_output_node_inline (
658 : vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 2);
659 : // basic processing + tx offloads + vlans + arcs + multi-txqs
660 : else
661 0 : n_bytes = vnet_interface_output_node_inline (
662 : vm, sw_if_index, ccm, bufs, p, config_index, arc, n_buffers, 3);
663 :
664 1903060 : from = vlib_frame_vector_args (frame);
665 1903060 : if (PREDICT_TRUE (next_index == VNET_INTERFACE_OUTPUT_NEXT_TX))
666 : {
667 1903050 : enqueue_to_tx_node (vm, node, hi, next_index, r, from, ptr,
668 1903050 : frame->n_vectors);
669 : }
670 : else
671 : {
672 9 : vlib_buffer_enqueue_to_single_next (vm, node, from, next_index,
673 9 : frame->n_vectors);
674 : }
675 :
676 : /* Update main interface stats. */
677 1903060 : vlib_increment_combined_counter (ccm, ti, sw_if_index, n_buffers, n_bytes);
678 1903060 : return n_buffers;
679 : }
680 :
681 178120 : VLIB_REGISTER_NODE (vnet_interface_output_node) = {
682 : .name = "interface-output-template",
683 : .vector_size = sizeof (u32),
684 : };
685 :
686 : /* Use buffer's sw_if_index[VNET_TX] to choose output interface. */
687 775113 : VLIB_NODE_FN (vnet_per_buffer_interface_output_node) (vlib_main_t * vm,
688 : vlib_node_runtime_t *
689 : node,
690 : vlib_frame_t * frame)
691 : {
692 772877 : vnet_main_t *vnm = vnet_get_main ();
693 : u32 n_left_to_next, *from, *to_next;
694 : u32 n_left_from, next_index;
695 :
696 772877 : vnet_interface_pcap_tx_trace (vm, node, frame, 1 /* in_interface_ouput */);
697 :
698 772877 : n_left_from = frame->n_vectors;
699 :
700 772877 : from = vlib_frame_vector_args (frame);
701 772877 : next_index = node->cached_next_index;
702 :
703 1547750 : while (n_left_from > 0)
704 : {
705 774877 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
706 :
707 14769700 : while (n_left_from >= 4 && n_left_to_next >= 2)
708 : {
709 : u32 bi0, bi1, next0, next1;
710 : vlib_buffer_t *b0, *b1;
711 : vnet_hw_interface_t *hi0, *hi1;
712 :
713 : /* Prefetch next iteration. */
714 13994800 : vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
715 13994800 : vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
716 :
717 13994800 : bi0 = from[0];
718 13994800 : bi1 = from[1];
719 13994800 : to_next[0] = bi0;
720 13994800 : to_next[1] = bi1;
721 13994800 : from += 2;
722 13994800 : to_next += 2;
723 13994800 : n_left_to_next -= 2;
724 13994800 : n_left_from -= 2;
725 :
726 13994800 : b0 = vlib_get_buffer (vm, bi0);
727 13994800 : b1 = vlib_get_buffer (vm, bi1);
728 :
729 : hi0 =
730 13994800 : vnet_get_sup_hw_interface (vnm,
731 13994800 : vnet_buffer (b0)->sw_if_index
732 : [VLIB_TX]);
733 : hi1 =
734 13994800 : vnet_get_sup_hw_interface (vnm,
735 13994800 : vnet_buffer (b1)->sw_if_index
736 : [VLIB_TX]);
737 :
738 13994800 : next0 = hi0->output_node_next_index;
739 13994800 : next1 = hi1->output_node_next_index;
740 :
741 13994800 : vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
742 : n_left_to_next, bi0, bi1, next0,
743 : next1);
744 : }
745 :
746 2667180 : while (n_left_from > 0 && n_left_to_next > 0)
747 : {
748 : u32 bi0, next0;
749 : vlib_buffer_t *b0;
750 : vnet_hw_interface_t *hi0;
751 :
752 1892310 : bi0 = from[0];
753 1892310 : to_next[0] = bi0;
754 1892310 : from += 1;
755 1892310 : to_next += 1;
756 1892310 : n_left_to_next -= 1;
757 1892310 : n_left_from -= 1;
758 :
759 1892310 : b0 = vlib_get_buffer (vm, bi0);
760 :
761 : hi0 =
762 1892310 : vnet_get_sup_hw_interface (vnm,
763 1892310 : vnet_buffer (b0)->sw_if_index
764 : [VLIB_TX]);
765 :
766 1892310 : next0 = hi0->output_node_next_index;
767 :
768 1892310 : vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
769 : n_left_to_next, bi0, next0);
770 : }
771 :
772 774877 : vlib_put_next_frame (vm, node, next_index, n_left_to_next);
773 : }
774 :
775 772877 : return frame->n_vectors;
776 : }
777 :
778 : typedef struct vnet_error_trace_t_
779 : {
780 : u32 sw_if_index;
781 : i8 details_valid;
782 : u8 is_ip6;
783 : u8 pad[2];
784 : u16 mactype;
785 : ip46_address_t src, dst;
786 : } vnet_error_trace_t;
787 :
788 : static u8 *
789 26195 : format_vnet_error_trace (u8 * s, va_list * va)
790 : {
791 26195 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
792 26195 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
793 26195 : vnet_error_trace_t *t = va_arg (*va, vnet_error_trace_t *);
794 :
795 : /* Normal, non-catchup trace */
796 26195 : if (t->details_valid == 0)
797 : {
798 26195 : s = format (s, "rx:%U", format_vnet_sw_if_index_name,
799 : vnet_get_main (), t->sw_if_index);
800 : }
801 0 : else if (t->details_valid == 1)
802 : {
803 : /* The trace capture code didn't understant the mactype */
804 0 : s = format (s, "mactype 0x%4x (not decoded)", t->mactype);
805 : }
806 0 : else if (t->details_valid == 2)
807 : {
808 : /* Dump the src/dst addresses */
809 0 : if (t->is_ip6 == 0)
810 0 : s = format (s, "IP4: %U -> %U",
811 : format_ip4_address, &t->src.ip4,
812 : format_ip4_address, &t->dst.ip4);
813 : else
814 0 : s = format (s, "IP6: %U -> %U",
815 : format_ip6_address, &t->src.ip6,
816 : format_ip6_address, &t->dst.ip6);
817 : }
818 26195 : return s;
819 : }
820 :
821 : static void
822 2583 : interface_trace_buffers (vlib_main_t * vm,
823 : vlib_node_runtime_t * node, vlib_frame_t * frame)
824 : {
825 : u32 n_left, *buffers;
826 :
827 2583 : buffers = vlib_frame_vector_args (frame);
828 2583 : n_left = frame->n_vectors;
829 :
830 23433 : while (n_left >= 4)
831 : {
832 : u32 bi0, bi1;
833 : vlib_buffer_t *b0, *b1;
834 : vnet_error_trace_t *t0, *t1;
835 :
836 : /* Prefetch next iteration. */
837 20850 : vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
838 20850 : vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
839 :
840 20850 : bi0 = buffers[0];
841 20850 : bi1 = buffers[1];
842 :
843 20850 : b0 = vlib_get_buffer (vm, bi0);
844 20850 : b1 = vlib_get_buffer (vm, bi1);
845 :
846 20850 : if (b0->flags & VLIB_BUFFER_IS_TRACED)
847 : {
848 20736 : t0 = vlib_add_trace (vm, node, b0,
849 : STRUCT_OFFSET_OF (vnet_error_trace_t, pad));
850 20736 : t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
851 20736 : t0->details_valid = 0;
852 : }
853 20850 : if (b1->flags & VLIB_BUFFER_IS_TRACED)
854 : {
855 20735 : t1 = vlib_add_trace (vm, node, b1,
856 : STRUCT_OFFSET_OF (vnet_error_trace_t, pad));
857 20735 : t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
858 20735 : t1->details_valid = 0;
859 : }
860 20850 : buffers += 2;
861 20850 : n_left -= 2;
862 : }
863 :
864 6994 : while (n_left >= 1)
865 : {
866 : u32 bi0;
867 : vlib_buffer_t *b0;
868 : vnet_error_trace_t *t0;
869 :
870 4411 : bi0 = buffers[0];
871 :
872 4411 : b0 = vlib_get_buffer (vm, bi0);
873 :
874 4411 : if (b0->flags & VLIB_BUFFER_IS_TRACED)
875 : {
876 4395 : t0 = vlib_add_trace (vm, node, b0,
877 : STRUCT_OFFSET_OF (vnet_error_trace_t, pad));
878 4395 : t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
879 4395 : t0->details_valid = 0;
880 : }
881 4411 : buffers += 1;
882 4411 : n_left -= 1;
883 : }
884 2583 : }
885 :
886 : typedef enum
887 : {
888 : VNET_ERROR_DISPOSITION_DROP,
889 : VNET_ERROR_DISPOSITION_PUNT,
890 : VNET_ERROR_N_DISPOSITION,
891 : } vnet_error_disposition_t;
892 :
893 : static void
894 0 : drop_catchup_trace (vlib_main_t * vm,
895 : vlib_node_runtime_t * node, vlib_buffer_t * b)
896 : {
897 : /* Can we safely rewind the buffer? If not, fagedaboudit */
898 0 : if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
899 : {
900 : vnet_error_trace_t *t;
901 : ip4_header_t *ip4;
902 : ip6_header_t *ip6;
903 : ethernet_header_t *eh;
904 : i16 delta;
905 :
906 0 : t = vlib_add_trace (vm, node, b, sizeof (*t));
907 0 : delta = vnet_buffer (b)->l2_hdr_offset - b->current_data;
908 0 : vlib_buffer_advance (b, delta);
909 :
910 0 : eh = vlib_buffer_get_current (b);
911 : /* Save mactype */
912 0 : t->mactype = clib_net_to_host_u16 (eh->type);
913 0 : t->details_valid = 1;
914 0 : switch (t->mactype)
915 : {
916 0 : case ETHERNET_TYPE_IP4:
917 0 : ip4 = (void *) (eh + 1);
918 0 : t->details_valid = 2;
919 0 : t->is_ip6 = 0;
920 0 : t->src.ip4.as_u32 = ip4->src_address.as_u32;
921 0 : t->dst.ip4.as_u32 = ip4->dst_address.as_u32;
922 0 : break;
923 :
924 0 : case ETHERNET_TYPE_IP6:
925 0 : ip6 = (void *) (eh + 1);
926 0 : t->details_valid = 2;
927 0 : t->is_ip6 = 1;
928 0 : clib_memcpy_fast (t->src.as_u8, ip6->src_address.as_u8,
929 : sizeof (ip6_address_t));
930 0 : clib_memcpy_fast (t->dst.as_u8, ip6->dst_address.as_u8,
931 : sizeof (ip6_address_t));
932 0 : break;
933 :
934 0 : default:
935 : /* Dunno, do nothing, leave details_valid alone */
936 0 : break;
937 : }
938 : /* Restore current data (probably unnecessary) */
939 0 : vlib_buffer_advance (b, -delta);
940 : }
941 0 : }
942 :
943 : static_always_inline uword
944 4104 : interface_drop_punt (vlib_main_t * vm,
945 : vlib_node_runtime_t * node,
946 : vlib_frame_t * frame,
947 : vnet_error_disposition_t disposition)
948 : {
949 : u32 *from, n_left, thread_index, *sw_if_index;
950 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
951 : u32 sw_if_indices[VLIB_FRAME_SIZE];
952 : vlib_simple_counter_main_t *cm;
953 : u16 nexts[VLIB_FRAME_SIZE];
954 : u32 n_trace;
955 : vnet_main_t *vnm;
956 :
957 4104 : vnm = vnet_get_main ();
958 4104 : thread_index = vm->thread_index;
959 4104 : from = vlib_frame_vector_args (frame);
960 4104 : n_left = frame->n_vectors;
961 4104 : b = bufs;
962 4104 : sw_if_index = sw_if_indices;
963 :
964 4104 : vlib_get_buffers (vm, from, bufs, n_left);
965 :
966 : /* "trace add error-drop NNN?" */
967 4104 : if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
968 : {
969 : /* If pkts aren't otherwise traced... */
970 0 : if ((node->flags & VLIB_NODE_FLAG_TRACE) == 0)
971 : {
972 : /* Trace them from here */
973 0 : node->flags |= VLIB_NODE_FLAG_TRACE;
974 0 : while (n_trace && n_left)
975 : {
976 0 : if (PREDICT_TRUE
977 : (vlib_trace_buffer (vm, node, 0 /* next_index */ , b[0],
978 : 0 /* follow chain */ )))
979 : {
980 : /*
981 : * Here we have a wireshark dissector problem.
982 : * Packets may be well-formed, or not. We
983 : * must not blow chunks in any case.
984 : *
985 : * Try to produce trace records which will help
986 : * folks understand what's going on.
987 : */
988 0 : drop_catchup_trace (vm, node, b[0]);
989 0 : n_trace--;
990 : }
991 0 : n_left--;
992 0 : b++;
993 : }
994 : }
995 :
996 0 : vlib_set_trace_count (vm, node, n_trace);
997 0 : b = bufs;
998 0 : n_left = frame->n_vectors;
999 : }
1000 :
1001 4104 : if (node->flags & VLIB_NODE_FLAG_TRACE)
1002 2583 : interface_trace_buffers (vm, node, frame);
1003 :
1004 : /* All going to drop regardless, this is just a counting exercise */
1005 4104 : clib_memset (nexts, 0, sizeof (nexts));
1006 :
1007 4104 : cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
1008 : (disposition == VNET_ERROR_DISPOSITION_PUNT
1009 : ? VNET_INTERFACE_COUNTER_PUNT
1010 : : VNET_INTERFACE_COUNTER_DROP));
1011 :
1012 : /* collect the array of interfaces first ... */
1013 15333 : while (n_left >= 4)
1014 : {
1015 11229 : if (n_left >= 12)
1016 : {
1017 : /* Prefetch 8 ahead - there's not much going on in each iteration */
1018 9550 : vlib_prefetch_buffer_header (b[4], LOAD);
1019 9550 : vlib_prefetch_buffer_header (b[5], LOAD);
1020 9550 : vlib_prefetch_buffer_header (b[6], LOAD);
1021 9550 : vlib_prefetch_buffer_header (b[7], LOAD);
1022 : }
1023 11229 : sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
1024 11229 : sw_if_index[1] = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
1025 11229 : sw_if_index[2] = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
1026 11229 : sw_if_index[3] = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
1027 :
1028 11229 : sw_if_index += 4;
1029 11229 : n_left -= 4;
1030 11229 : b += 4;
1031 : }
1032 9162 : while (n_left)
1033 : {
1034 5058 : sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
1035 :
1036 5058 : sw_if_index += 1;
1037 5058 : n_left -= 1;
1038 5058 : b += 1;
1039 : }
1040 :
1041 : /* ... then count against them in blocks */
1042 4104 : n_left = frame->n_vectors;
1043 :
1044 8450 : while (n_left)
1045 : {
1046 : vnet_sw_interface_t *sw_if0;
1047 : u16 off, count;
1048 :
1049 4346 : off = frame->n_vectors - n_left;
1050 :
1051 4346 : sw_if_index = sw_if_indices + off;
1052 :
1053 4346 : count = clib_count_equal_u32 (sw_if_index, n_left);
1054 4346 : n_left -= count;
1055 :
1056 4346 : vlib_increment_simple_counter (cm, thread_index, sw_if_index[0], count);
1057 :
1058 : /* Increment super-interface drop/punt counters for
1059 : sub-interfaces. */
1060 4346 : sw_if0 = vnet_get_sw_interface (vnm, sw_if_index[0]);
1061 4346 : if (sw_if0->sup_sw_if_index != sw_if_index[0])
1062 75 : vlib_increment_simple_counter
1063 : (cm, thread_index, sw_if0->sup_sw_if_index, count);
1064 : }
1065 :
1066 4104 : vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1067 :
1068 4104 : return frame->n_vectors;
1069 : }
1070 :
1071 : static inline void
1072 3 : pcap_drop_trace (vlib_main_t * vm,
1073 : vnet_interface_main_t * im,
1074 : vnet_pcap_t * pp, vlib_frame_t * f)
1075 : {
1076 : u32 *from;
1077 3 : u32 n_left = f->n_vectors;
1078 : vlib_buffer_t *b0, *p1;
1079 : u32 bi0;
1080 : i16 save_current_data;
1081 : u16 save_current_length;
1082 3 : vlib_error_main_t *em = &vm->error_main;
1083 :
1084 3 : from = vlib_frame_vector_args (f);
1085 :
1086 31 : while (n_left > 0)
1087 : {
1088 28 : if (PREDICT_TRUE (n_left > 1))
1089 : {
1090 25 : p1 = vlib_get_buffer (vm, from[1]);
1091 25 : vlib_prefetch_buffer_header (p1, LOAD);
1092 : }
1093 :
1094 28 : bi0 = from[0];
1095 28 : b0 = vlib_get_buffer (vm, bi0);
1096 28 : from++;
1097 28 : n_left--;
1098 :
1099 : /* See if we're pointedly ignoring this specific error */
1100 28 : if (im->pcap_drop_filter_hash
1101 28 : && hash_get (im->pcap_drop_filter_hash, b0->error))
1102 0 : continue;
1103 :
1104 28 : if (!vnet_is_packet_pcaped (pp, b0, ~0))
1105 1 : continue; /* not matching, skip */
1106 :
1107 : /* Trace all drops, or drops received on a specific interface */
1108 27 : save_current_data = b0->current_data;
1109 27 : save_current_length = b0->current_length;
1110 :
1111 : /*
1112 : * Typically, we'll need to rewind the buffer
1113 : * if l2_hdr_offset is valid, make sure to rewind to the start of
1114 : * the L2 header. This may not be the buffer start in case we pop-ed
1115 : * vlan tags.
1116 : * Otherwise, rewind to buffer start and hope for the best.
1117 : */
1118 27 : if (b0->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
1119 : {
1120 27 : if (b0->current_data > vnet_buffer (b0)->l2_hdr_offset)
1121 27 : vlib_buffer_advance (b0, vnet_buffer (b0)->l2_hdr_offset -
1122 27 : b0->current_data);
1123 : }
1124 0 : else if (b0->current_data > 0)
1125 : {
1126 0 : vlib_buffer_advance (b0, (word) -b0->current_data);
1127 : }
1128 :
1129 : {
1130 27 : vlib_buffer_t *last = b0;
1131 : u32 error_node_index;
1132 : int drop_string_len;
1133 : vlib_node_t *n;
1134 : /* Length of the error string */
1135 27 : int error_string_len =
1136 27 : clib_strnlen (em->counters_heap[b0->error].name, 128);
1137 :
1138 : /* Dig up the drop node */
1139 27 : error_node_index = vm->node_main.node_by_error[b0->error];
1140 27 : n = vlib_get_node (vm, error_node_index);
1141 :
1142 : /* Length of full drop string, w/ "nodename: " prepended */
1143 27 : drop_string_len = error_string_len + vec_len (n->name) + 2;
1144 :
1145 : /* Find the last buffer in the chain */
1146 27 : while (last->flags & VLIB_BUFFER_NEXT_PRESENT)
1147 0 : last = vlib_get_buffer (vm, last->next_buffer);
1148 :
1149 : /*
1150 : * Append <nodename>: <error-string> to the capture,
1151 : * only if we can do that without allocating a new buffer.
1152 : */
1153 27 : if (PREDICT_TRUE ((last->current_data + last->current_length) <
1154 : (VLIB_BUFFER_DEFAULT_DATA_SIZE - drop_string_len)))
1155 : {
1156 27 : clib_memcpy_fast (last->data + last->current_data +
1157 27 : last->current_length,
1158 54 : n->name, vec_len (n->name));
1159 54 : clib_memcpy_fast (last->data + last->current_data +
1160 27 : last->current_length + vec_len (n->name),
1161 : ": ", 2);
1162 54 : clib_memcpy_fast (last->data + last->current_data +
1163 27 : last->current_length + vec_len (n->name) + 2,
1164 27 : em->counters_heap[b0->error].name,
1165 : error_string_len);
1166 27 : last->current_length += drop_string_len;
1167 27 : b0->flags &= ~(VLIB_BUFFER_TOTAL_LENGTH_VALID);
1168 27 : pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt);
1169 27 : last->current_length -= drop_string_len;
1170 27 : b0->current_data = save_current_data;
1171 27 : b0->current_length = save_current_length;
1172 27 : continue;
1173 : }
1174 : }
1175 :
1176 : /*
1177 : * Didn't have space in the last buffer, here's the dropped
1178 : * packet as-is
1179 : */
1180 0 : pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt);
1181 :
1182 0 : b0->current_data = save_current_data;
1183 0 : b0->current_length = save_current_length;
1184 : }
1185 3 : }
1186 :
1187 : #ifndef CLIB_MARCH_VARIANT
1188 : void
1189 12857 : vnet_pcap_drop_trace_filter_add_del (u32 error_index, int is_add)
1190 : {
1191 12857 : vnet_interface_main_t *im = &vnet_get_main ()->interface_main;
1192 :
1193 12857 : if (im->pcap_drop_filter_hash == 0)
1194 559 : im->pcap_drop_filter_hash = hash_create (0, sizeof (uword));
1195 :
1196 12857 : if (is_add)
1197 12857 : hash_set (im->pcap_drop_filter_hash, error_index, 1);
1198 : else
1199 0 : hash_unset (im->pcap_drop_filter_hash, error_index);
1200 12857 : }
1201 : #endif /* CLIB_MARCH_VARIANT */
1202 :
1203 6169 : VLIB_NODE_FN (interface_drop) (vlib_main_t * vm,
1204 : vlib_node_runtime_t * node,
1205 : vlib_frame_t * frame)
1206 : {
1207 3933 : vnet_main_t *vnm = vnet_get_main ();
1208 3933 : vnet_interface_main_t *im = &vnet_get_main ()->interface_main;
1209 3933 : vnet_pcap_t *pp = &vnm->pcap;
1210 :
1211 3933 : if (PREDICT_FALSE (pp->pcap_drop_enable))
1212 3 : pcap_drop_trace (vm, im, pp, frame);
1213 :
1214 3933 : return interface_drop_punt (vm, node, frame, VNET_ERROR_DISPOSITION_DROP);
1215 : }
1216 :
1217 2407 : VLIB_NODE_FN (interface_punt) (vlib_main_t * vm,
1218 : vlib_node_runtime_t * node,
1219 : vlib_frame_t * frame)
1220 : {
1221 171 : return interface_drop_punt (vm, node, frame, VNET_ERROR_DISPOSITION_PUNT);
1222 : }
1223 :
1224 : /* *INDENT-OFF* */
1225 178120 : VLIB_REGISTER_NODE (interface_drop) = {
1226 : .name = "error-drop",
1227 : .vector_size = sizeof (u32),
1228 : .format_trace = format_vnet_error_trace,
1229 : .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
1230 : .n_next_nodes = 1,
1231 : .next_nodes = {
1232 : [0] = "drop",
1233 : },
1234 : };
1235 : /* *INDENT-ON* */
1236 :
1237 : /* *INDENT-OFF* */
1238 178120 : VLIB_REGISTER_NODE (interface_punt) = {
1239 : .name = "error-punt",
1240 : .vector_size = sizeof (u32),
1241 : .format_trace = format_vnet_error_trace,
1242 : .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
1243 : .n_next_nodes = 1,
1244 : .next_nodes = {
1245 : [0] = "punt",
1246 : },
1247 : };
1248 : /* *INDENT-ON* */
1249 :
1250 178120 : VLIB_REGISTER_NODE (vnet_per_buffer_interface_output_node) = {
1251 : .name = "interface-output",
1252 : .vector_size = sizeof (u32),
1253 : };
1254 :
1255 2245 : VLIB_NODE_FN (vnet_interface_output_arc_end_node)
1256 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1257 : {
1258 9 : vnet_main_t *vnm = vnet_get_main ();
1259 9 : vnet_interface_main_t *im = &vnm->interface_main;
1260 : vnet_hw_interface_t *hi;
1261 9 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1262 9 : u32 sw_if_indices[VLIB_FRAME_SIZE], *sw_if_index = sw_if_indices;
1263 9 : vlib_frame_bitmap_t used_elts = {}, mask = {};
1264 : u32 *tmp, *from, n_left, n_comp, n_p_comp, swif, off;
1265 : u16 next_index;
1266 9 : void *ptr[VLIB_FRAME_SIZE], **p = ptr;
1267 :
1268 9 : from = vlib_frame_vector_args (frame);
1269 9 : n_left = frame->n_vectors;
1270 9 : vlib_get_buffers (vm, from, bufs, n_left);
1271 :
1272 10 : while (n_left >= 8)
1273 : {
1274 1 : vlib_prefetch_buffer_header (b[4], LOAD);
1275 1 : vlib_prefetch_buffer_header (b[5], LOAD);
1276 1 : vlib_prefetch_buffer_header (b[6], LOAD);
1277 1 : vlib_prefetch_buffer_header (b[7], LOAD);
1278 :
1279 1 : p[0] = vlib_buffer_get_current (b[0]);
1280 1 : p[1] = vlib_buffer_get_current (b[1]);
1281 1 : p[2] = vlib_buffer_get_current (b[2]);
1282 1 : p[3] = vlib_buffer_get_current (b[3]);
1283 1 : sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
1284 1 : sw_if_index[1] = vnet_buffer (b[1])->sw_if_index[VLIB_TX];
1285 1 : sw_if_index[2] = vnet_buffer (b[2])->sw_if_index[VLIB_TX];
1286 1 : sw_if_index[3] = vnet_buffer (b[3])->sw_if_index[VLIB_TX];
1287 :
1288 1 : p += 4;
1289 1 : b += 4;
1290 1 : sw_if_index += 4;
1291 1 : n_left -= 4;
1292 : }
1293 :
1294 29 : while (n_left)
1295 : {
1296 20 : p[0] = vlib_buffer_get_current (b[0]);
1297 20 : sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
1298 20 : p++;
1299 20 : b++;
1300 20 : sw_if_index++;
1301 20 : n_left--;
1302 : }
1303 :
1304 9 : n_left = frame->n_vectors;
1305 9 : swif = sw_if_indices[0];
1306 9 : off = 0;
1307 :
1308 : /* a bit ugly but it allows us to reuse stack space for temporary store
1309 : * which may also improve memory latency */
1310 9 : tmp = (u32 *) bufs;
1311 :
1312 9 : more:
1313 9 : next_index = vec_elt (im->if_out_arc_end_next_index_by_sw_if_index, swif);
1314 9 : hi = vnet_get_sup_hw_interface (vnm, swif);
1315 9 : vnet_hw_if_output_node_runtime_t *r = 0;
1316 9 : void *ptr_tmp[VLIB_FRAME_SIZE], **p_tmp = ptr_tmp;
1317 :
1318 9 : if (hi->output_node_thread_runtimes)
1319 0 : r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index);
1320 :
1321 : /* compare and compress based on comparison mask */
1322 9 : clib_mask_compare_u32 (swif, sw_if_indices, mask, frame->n_vectors);
1323 9 : n_comp = clib_compress_u32 (tmp, from, mask, frame->n_vectors);
1324 :
1325 : /*
1326 : * tx queue of given interface is not available on given thread
1327 : */
1328 9 : if (r)
1329 : {
1330 0 : if (r->n_queues == 0)
1331 : {
1332 0 : vlib_error_drop_buffers (
1333 : vm, node, tmp,
1334 : /* buffer stride */ 1, n_comp, VNET_INTERFACE_OUTPUT_NEXT_DROP,
1335 : node->node_index, VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE);
1336 0 : goto drop;
1337 : }
1338 0 : else if (r->n_queues > 1)
1339 : {
1340 0 : n_p_comp = clib_compress_u64 ((u64 *) p_tmp, (u64 *) ptr, mask,
1341 0 : frame->n_vectors);
1342 0 : ASSERT (n_p_comp == n_comp);
1343 : }
1344 : }
1345 :
1346 9 : enqueue_to_tx_node (vm, node, hi, next_index, r, tmp, ptr_tmp, n_comp);
1347 :
1348 9 : drop:
1349 9 : n_left -= n_comp;
1350 9 : if (n_left)
1351 : {
1352 : /* store comparison mask so we can find next unused element */
1353 0 : vlib_frame_bitmap_or (used_elts, mask);
1354 :
1355 : /* fine first unused sw_if_index by scanning trough used_elts bitmap */
1356 0 : while (PREDICT_FALSE (used_elts[off] == ~0))
1357 0 : off++;
1358 :
1359 0 : swif =
1360 0 : sw_if_indices[(off << 6) + count_trailing_zeros (~used_elts[off])];
1361 0 : goto more;
1362 : }
1363 :
1364 9 : return frame->n_vectors;
1365 : }
1366 :
1367 178120 : VLIB_REGISTER_NODE (vnet_interface_output_arc_end_node) = {
1368 : .name = "interface-output-arc-end",
1369 : .vector_size = sizeof (u32),
1370 : .n_next_nodes = 1,
1371 : .next_nodes = {
1372 : [0] = "error-drop",
1373 : },
1374 : };
1375 :
1376 1119 : VNET_FEATURE_ARC_INIT (interface_output, static) = {
1377 : .arc_name = "interface-output",
1378 : .start_nodes = VNET_FEATURES (0),
1379 : .last_in_arc = "interface-output-arc-end",
1380 : .arc_index_ptr = &vnet_main.interface_main.output_feature_arc_index,
1381 : };
1382 :
1383 70583 : VNET_FEATURE_INIT (span_tx, static) = {
1384 : .arc_name = "interface-output",
1385 : .node_name = "span-output",
1386 : .runs_before = VNET_FEATURES ("interface-output-arc-end"),
1387 : };
1388 :
1389 70583 : VNET_FEATURE_INIT (ipsec_if_tx, static) = {
1390 : .arc_name = "interface-output",
1391 : .node_name = "ipsec-if-output",
1392 : .runs_before = VNET_FEATURES ("interface-output-arc-end"),
1393 : };
1394 :
1395 70583 : VNET_FEATURE_INIT (interface_output_arc_end, static) = {
1396 : .arc_name = "interface-output",
1397 : .node_name = "interface-output-arc-end",
1398 : .runs_before = 0,
1399 : };
1400 :
1401 : #ifndef CLIB_MARCH_VARIANT
1402 : clib_error_t *
1403 10415 : vnet_per_buffer_interface_output_hw_interface_add_del (vnet_main_t * vnm,
1404 : u32 hw_if_index,
1405 : u32 is_create)
1406 : {
1407 10415 : vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
1408 : u32 next_index;
1409 :
1410 10415 : if (hi->output_node_index == 0)
1411 3840 : return 0;
1412 :
1413 13150 : next_index = vlib_node_add_next
1414 6575 : (vnm->vlib_main, vnet_per_buffer_interface_output_node.index,
1415 6575 : hi->output_node_index);
1416 6575 : hi->output_node_next_index = next_index;
1417 :
1418 6575 : return 0;
1419 : }
1420 :
1421 1119 : VNET_HW_INTERFACE_ADD_DEL_FUNCTION
1422 : (vnet_per_buffer_interface_output_hw_interface_add_del);
1423 :
1424 : void
1425 3178 : vnet_set_interface_output_node (vnet_main_t * vnm,
1426 : u32 hw_if_index, u32 node_index)
1427 : {
1428 3178 : ASSERT (node_index);
1429 3178 : vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
1430 6356 : u32 next_index = vlib_node_add_next
1431 3178 : (vnm->vlib_main, vnet_per_buffer_interface_output_node.index, node_index);
1432 3178 : hi->output_node_next_index = next_index;
1433 3178 : hi->output_node_index = node_index;
1434 3178 : }
1435 : #endif /* CLIB_MARCH_VARIANT */
1436 :
1437 : /*
1438 : * fd.io coding-style-patch-verification: ON
1439 : *
1440 : * Local Variables:
1441 : * eval: (c-set-style "gnu")
1442 : * End:
1443 : */
|
