Line data Source code
1 : /*
2 : * Copyright (c) 2018 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * ethernet_node.c: ethernet packet processing
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #include <vlib/vlib.h>
41 : #include <vnet/pg/pg.h>
42 : #include <vnet/ethernet/ethernet.h>
43 : #include <vnet/ethernet/p2p_ethernet.h>
44 : #include <vnet/devices/pipe/pipe.h>
45 : #include <vppinfra/sparse_vec.h>
46 : #include <vnet/l2/l2_bvi.h>
47 : #include <vnet/classify/pcap_classify.h>
48 :
49 : #define foreach_ethernet_input_next \
50 : _ (PUNT, "error-punt") \
51 : _ (DROP, "error-drop") \
52 : _ (LLC, "llc-input") \
53 : _ (IP4_INPUT, "ip4-input") \
54 : _ (IP4_INPUT_NCS, "ip4-input-no-checksum")
55 :
56 : typedef enum
57 : {
58 : #define _(s,n) ETHERNET_INPUT_NEXT_##s,
59 : foreach_ethernet_input_next
60 : #undef _
61 : ETHERNET_INPUT_N_NEXT,
62 : } ethernet_input_next_t;
63 :
64 : typedef struct
65 : {
66 : u8 packet_data[32];
67 : u16 frame_flags;
68 : ethernet_input_frame_t frame_data;
69 : } ethernet_input_trace_t;
70 :
71 : static u8 *
72 355288 : format_ethernet_input_trace (u8 * s, va_list * va)
73 : {
74 355288 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
75 355288 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
76 355288 : ethernet_input_trace_t *t = va_arg (*va, ethernet_input_trace_t *);
77 355288 : u32 indent = format_get_indent (s);
78 :
79 355288 : if (t->frame_flags)
80 : {
81 353225 : s = format (s, "frame: flags 0x%x", t->frame_flags);
82 353225 : if (t->frame_flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
83 353225 : s = format (s, ", hw-if-index %u, sw-if-index %u",
84 : t->frame_data.hw_if_index, t->frame_data.sw_if_index);
85 353225 : s = format (s, "\n%U", format_white_space, indent);
86 : }
87 355288 : s = format (s, "%U", format_ethernet_header, t->packet_data);
88 :
89 355288 : return s;
90 : }
91 :
92 : extern vlib_node_registration_t ethernet_input_node;
93 :
94 : typedef enum
95 : {
96 : ETHERNET_INPUT_VARIANT_ETHERNET,
97 : ETHERNET_INPUT_VARIANT_ETHERNET_TYPE,
98 : ETHERNET_INPUT_VARIANT_NOT_L2,
99 : } ethernet_input_variant_t;
100 :
101 :
102 : // Parse the ethernet header to extract vlan tags and innermost ethertype
103 : static_always_inline void
104 180486 : parse_header (ethernet_input_variant_t variant,
105 : vlib_buffer_t * b0,
106 : u16 * type,
107 : u16 * orig_type,
108 : u16 * outer_id, u16 * inner_id, u32 * match_flags)
109 : {
110 : u8 vlan_count;
111 :
112 180486 : if (variant == ETHERNET_INPUT_VARIANT_ETHERNET
113 0 : || variant == ETHERNET_INPUT_VARIANT_NOT_L2)
114 180486 : {
115 : ethernet_header_t *e0;
116 :
117 180486 : e0 = vlib_buffer_get_current (b0);
118 :
119 180486 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
120 180486 : b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
121 :
122 180486 : vlib_buffer_advance (b0, sizeof (e0[0]));
123 :
124 180486 : *type = clib_net_to_host_u16 (e0->type);
125 : }
126 0 : else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE)
127 : {
128 : // here when prior node was LLC/SNAP processing
129 : u16 *e0;
130 :
131 0 : e0 = vlib_buffer_get_current (b0);
132 :
133 0 : vlib_buffer_advance (b0, sizeof (e0[0]));
134 :
135 0 : *type = clib_net_to_host_u16 (e0[0]);
136 : }
137 :
138 : // save for distinguishing between dot1q and dot1ad later
139 180486 : *orig_type = *type;
140 :
141 : // default the tags to 0 (used if there is no corresponding tag)
142 180486 : *outer_id = 0;
143 180486 : *inner_id = 0;
144 :
145 180486 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_0_TAG;
146 180486 : vlan_count = 0;
147 :
148 : // check for vlan encaps
149 180486 : if (ethernet_frame_is_tagged (*type))
150 : {
151 : ethernet_vlan_header_t *h0;
152 : u16 tag;
153 :
154 514 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_1_TAG;
155 :
156 514 : h0 = vlib_buffer_get_current (b0);
157 :
158 514 : tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
159 :
160 514 : *outer_id = tag & 0xfff;
161 514 : if (0 == *outer_id)
162 0 : *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG;
163 :
164 514 : *type = clib_net_to_host_u16 (h0->type);
165 :
166 514 : vlib_buffer_advance (b0, sizeof (h0[0]));
167 514 : vlan_count = 1;
168 :
169 514 : if (*type == ETHERNET_TYPE_VLAN)
170 : {
171 : // Double tagged packet
172 0 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_2_TAG;
173 :
174 0 : h0 = vlib_buffer_get_current (b0);
175 :
176 0 : tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
177 :
178 0 : *inner_id = tag & 0xfff;
179 :
180 0 : *type = clib_net_to_host_u16 (h0->type);
181 :
182 0 : vlib_buffer_advance (b0, sizeof (h0[0]));
183 0 : vlan_count = 2;
184 0 : if (*type == ETHERNET_TYPE_VLAN)
185 : {
186 : // More than double tagged packet
187 0 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_3_TAG;
188 :
189 0 : vlib_buffer_advance (b0, sizeof (h0[0]));
190 0 : vlan_count = 3; // "unknown" number, aka, 3-or-more
191 : }
192 : }
193 : }
194 180486 : ethernet_buffer_set_vlan_count (b0, vlan_count);
195 180486 : }
196 :
197 : static_always_inline void
198 : ethernet_input_inline_dmac_check (vnet_hw_interface_t * hi,
199 : u64 * dmacs, u8 * dmacs_bad,
200 : u32 n_packets, ethernet_interface_t * ei,
201 : u8 have_sec_dmac);
202 :
203 : // Determine the subinterface for this packet, given the result of the
204 : // vlan table lookups and vlan header parsing. Check the most specific
205 : // matches first.
206 : static_always_inline void
207 180486 : identify_subint (ethernet_main_t * em,
208 : vnet_hw_interface_t * hi,
209 : vlib_buffer_t * b0,
210 : u32 match_flags,
211 : main_intf_t * main_intf,
212 : vlan_intf_t * vlan_intf,
213 : qinq_intf_t * qinq_intf,
214 : u32 * new_sw_if_index, u8 * error0, u32 * is_l2)
215 : {
216 : u32 matched;
217 180486 : ethernet_interface_t *ei = ethernet_get_interface (em, hi->hw_if_index);
218 :
219 180486 : matched = eth_identify_subint (hi, match_flags, main_intf, vlan_intf,
220 : qinq_intf, new_sw_if_index, error0, is_l2);
221 :
222 180486 : if (matched)
223 : {
224 : // Perform L3 my-mac filter
225 : // A unicast packet arriving on an L3 interface must have a dmac
226 : // matching the interface mac. If interface has STATUS_L3 bit set
227 : // mac filter is already done.
228 180486 : if ((!*is_l2) && ei &&
229 27838 : (!(ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3)))
230 : {
231 : u64 dmacs[2];
232 : u8 dmacs_bad[2];
233 : ethernet_header_t *e0;
234 :
235 27838 : e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset);
236 27838 : dmacs[0] = *(u64 *) e0;
237 :
238 27838 : if (vec_len (ei->secondary_addrs))
239 27838 : ethernet_input_inline_dmac_check (hi, dmacs, dmacs_bad,
240 : 1 /* n_packets */, ei,
241 : 1 /* have_sec_dmac */);
242 : else
243 0 : ethernet_input_inline_dmac_check (hi, dmacs, dmacs_bad,
244 : 1 /* n_packets */, ei,
245 : 0 /* have_sec_dmac */);
246 27838 : if (dmacs_bad[0])
247 0 : *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
248 : }
249 :
250 : // Check for down subinterface
251 180486 : *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN;
252 : }
253 180486 : }
254 :
255 : static_always_inline void
256 4493840 : determine_next_node (ethernet_main_t * em,
257 : ethernet_input_variant_t variant,
258 : u32 is_l20,
259 : u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0)
260 : {
261 4493840 : vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
262 4493840 : b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
263 :
264 4493840 : if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE))
265 : {
266 : // some error occurred
267 47 : *next0 = ETHERNET_INPUT_NEXT_DROP;
268 : }
269 4493790 : else if (is_l20)
270 : {
271 : // record the L2 len and reset the buffer so the L2 header is preserved
272 152648 : u32 eth_start = vnet_buffer (b0)->l2_hdr_offset;
273 152648 : vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start;
274 152648 : *next0 = em->l2_next;
275 152648 : ASSERT (vnet_buffer (b0)->l2.l2_len ==
276 : ethernet_buffer_header_size (b0));
277 152648 : vlib_buffer_advance (b0, -(vnet_buffer (b0)->l2.l2_len));
278 :
279 : // check for common IP/MPLS ethertypes
280 : }
281 4341140 : else if (type0 == ETHERNET_TYPE_IP4)
282 : {
283 2506580 : *next0 = em->l3_next.input_next_ip4;
284 : }
285 1834560 : else if (type0 == ETHERNET_TYPE_IP6)
286 : {
287 1834440 : *next0 = em->l3_next.input_next_ip6;
288 : }
289 122 : else if (type0 == ETHERNET_TYPE_MPLS)
290 : {
291 0 : *next0 = em->l3_next.input_next_mpls;
292 :
293 : }
294 122 : else if (em->redirect_l3)
295 : {
296 : // L3 Redirect is on, the cached common next nodes will be
297 : // pointing to the redirect node, catch the uncommon types here
298 0 : *next0 = em->redirect_l3_next;
299 : }
300 : else
301 : {
302 : // uncommon ethertype, check table
303 : u32 i0;
304 122 : i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0);
305 122 : *next0 = vec_elt (em->l3_next.input_next_by_type, i0);
306 122 : *error0 =
307 : i0 ==
308 : SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0;
309 :
310 : // The table is not populated with LLC values, so check that now.
311 : // If variant is variant_ethernet then we came from LLC processing. Don't
312 : // go back there; drop instead using by keeping the drop/bad table result.
313 122 : if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET))
314 : {
315 0 : *next0 = ETHERNET_INPUT_NEXT_LLC;
316 : }
317 : }
318 4493840 : }
319 :
320 :
321 : /* following vector code relies on following assumptions */
322 : STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_data, 0);
323 : STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_length, 2);
324 : STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, flags, 4);
325 : STATIC_ASSERT (STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l2_hdr_offset) ==
326 : STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l3_hdr_offset) - 2,
327 : "l3_hdr_offset must follow l2_hdr_offset");
328 :
329 : static_always_inline void
330 14898400 : eth_input_adv_and_flags_x4 (vlib_buffer_t ** b, int is_l3)
331 : {
332 14898400 : i16 adv = sizeof (ethernet_header_t);
333 14898400 : u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
334 : VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
335 :
336 : #ifdef CLIB_HAVE_VEC256
337 : /* to reduce number of small loads/stores we are loading first 64 bits
338 : of each buffer metadata into 256-bit register so we can advance
339 : current_data, current_length and flags.
340 : Observed saving of this code is ~2 clocks per packet */
341 : u64x4 r, radv;
342 :
343 : /* vector if signed 16 bit integers used in signed vector add operation
344 : to advnce current_data and current_length */
345 14898400 : u32x8 flags4 = { 0, flags, 0, flags, 0, flags, 0, flags };
346 14898400 : i16x16 adv4 = {
347 : adv, -adv, 0, 0, adv, -adv, 0, 0,
348 : adv, -adv, 0, 0, adv, -adv, 0, 0
349 : };
350 :
351 : /* load 4 x 64 bits */
352 14898400 : r = u64x4_gather (b[0], b[1], b[2], b[3]);
353 :
354 : /* set flags */
355 14898400 : r |= (u64x4) flags4;
356 :
357 : /* advance buffer */
358 14898400 : radv = (u64x4) ((i16x16) r + adv4);
359 :
360 : /* write 4 x 64 bits */
361 14898400 : u64x4_scatter (is_l3 ? radv : r, b[0], b[1], b[2], b[3]);
362 :
363 : /* use old current_data as l2_hdr_offset and new current_data as
364 : l3_hdr_offset */
365 14898400 : r = (u64x4) u16x16_blend (r, radv << 16, 0xaa);
366 :
367 : /* store both l2_hdr_offset and l3_hdr_offset in single store operation */
368 14898400 : u32x8_scatter_one ((u32x8) r, 0, &vnet_buffer (b[0])->l2_hdr_offset);
369 14898400 : u32x8_scatter_one ((u32x8) r, 2, &vnet_buffer (b[1])->l2_hdr_offset);
370 14898400 : u32x8_scatter_one ((u32x8) r, 4, &vnet_buffer (b[2])->l2_hdr_offset);
371 14898400 : u32x8_scatter_one ((u32x8) r, 6, &vnet_buffer (b[3])->l2_hdr_offset);
372 :
373 14898400 : if (is_l3)
374 : {
375 3318040 : ASSERT (b[0]->current_data == vnet_buffer (b[0])->l3_hdr_offset);
376 3318040 : ASSERT (b[1]->current_data == vnet_buffer (b[1])->l3_hdr_offset);
377 3318040 : ASSERT (b[2]->current_data == vnet_buffer (b[2])->l3_hdr_offset);
378 3318040 : ASSERT (b[3]->current_data == vnet_buffer (b[3])->l3_hdr_offset);
379 :
380 3318040 : ASSERT (b[0]->current_data - vnet_buffer (b[0])->l2_hdr_offset == adv);
381 3318040 : ASSERT (b[1]->current_data - vnet_buffer (b[1])->l2_hdr_offset == adv);
382 3318040 : ASSERT (b[2]->current_data - vnet_buffer (b[2])->l2_hdr_offset == adv);
383 3318040 : ASSERT (b[3]->current_data - vnet_buffer (b[3])->l2_hdr_offset == adv);
384 : }
385 : else
386 : {
387 11580400 : ASSERT (b[0]->current_data == vnet_buffer (b[0])->l2_hdr_offset);
388 11580400 : ASSERT (b[1]->current_data == vnet_buffer (b[1])->l2_hdr_offset);
389 11580400 : ASSERT (b[2]->current_data == vnet_buffer (b[2])->l2_hdr_offset);
390 11580400 : ASSERT (b[3]->current_data == vnet_buffer (b[3])->l2_hdr_offset);
391 :
392 11580400 : ASSERT (b[0]->current_data - vnet_buffer (b[0])->l3_hdr_offset == -adv);
393 11580400 : ASSERT (b[1]->current_data - vnet_buffer (b[1])->l3_hdr_offset == -adv);
394 11580400 : ASSERT (b[2]->current_data - vnet_buffer (b[2])->l3_hdr_offset == -adv);
395 11580400 : ASSERT (b[3]->current_data - vnet_buffer (b[3])->l3_hdr_offset == -adv);
396 : }
397 :
398 : #else
399 0 : vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
400 0 : vnet_buffer (b[1])->l2_hdr_offset = b[1]->current_data;
401 0 : vnet_buffer (b[2])->l2_hdr_offset = b[2]->current_data;
402 0 : vnet_buffer (b[3])->l2_hdr_offset = b[3]->current_data;
403 0 : vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
404 0 : vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data + adv;
405 0 : vnet_buffer (b[2])->l3_hdr_offset = b[2]->current_data + adv;
406 0 : vnet_buffer (b[3])->l3_hdr_offset = b[3]->current_data + adv;
407 :
408 0 : if (is_l3)
409 : {
410 0 : vlib_buffer_advance (b[0], adv);
411 0 : vlib_buffer_advance (b[1], adv);
412 0 : vlib_buffer_advance (b[2], adv);
413 0 : vlib_buffer_advance (b[3], adv);
414 : }
415 :
416 0 : b[0]->flags |= flags;
417 0 : b[1]->flags |= flags;
418 0 : b[2]->flags |= flags;
419 0 : b[3]->flags |= flags;
420 : #endif
421 :
422 14898400 : if (!is_l3)
423 : {
424 11580400 : vnet_buffer (b[0])->l2.l2_len = adv;
425 11580400 : vnet_buffer (b[1])->l2.l2_len = adv;
426 11580400 : vnet_buffer (b[2])->l2.l2_len = adv;
427 11580400 : vnet_buffer (b[3])->l2.l2_len = adv;
428 : }
429 14898400 : }
430 :
431 : static_always_inline void
432 1856300 : eth_input_adv_and_flags_x1 (vlib_buffer_t ** b, int is_l3)
433 : {
434 1856300 : i16 adv = sizeof (ethernet_header_t);
435 1856300 : u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
436 : VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
437 :
438 1856300 : vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
439 1856300 : vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
440 :
441 1856300 : if (is_l3)
442 406462 : vlib_buffer_advance (b[0], adv);
443 1856300 : b[0]->flags |= flags;
444 1856300 : if (!is_l3)
445 1449840 : vnet_buffer (b[0])->l2.l2_len = adv;
446 1856300 : }
447 :
448 :
449 : static_always_inline void
450 61450000 : eth_input_get_etype_and_tags (vlib_buffer_t ** b, u16 * etype, u64 * tags,
451 : u64 * dmacs, int offset, int dmac_check)
452 : {
453 : ethernet_header_t *e;
454 61450000 : e = vlib_buffer_get_current (b[offset]);
455 : #ifdef CLIB_HAVE_VEC128
456 61450000 : u64x2 r = u64x2_load_unaligned (((u8 *) & e->type) - 6);
457 61450000 : etype[offset] = ((u16x8) r)[3];
458 61450000 : tags[offset] = r[1];
459 : #else
460 : etype[offset] = e->type;
461 : tags[offset] = *(u64 *) (e + 1);
462 : #endif
463 :
464 61450000 : if (dmac_check)
465 13678800 : dmacs[offset] = *(u64 *) e;
466 61450000 : }
467 :
468 : static_always_inline u16
469 2103 : eth_input_next_by_type (u16 etype)
470 : {
471 2103 : ethernet_main_t *em = ðernet_main;
472 :
473 4198 : return (etype < 0x600) ? ETHERNET_INPUT_NEXT_LLC :
474 2095 : vec_elt (em->l3_next.input_next_by_type,
475 : sparse_vec_index (em->l3_next.input_next_by_type, etype));
476 : }
477 :
478 : typedef struct
479 : {
480 : u64 tag, mask;
481 : u32 sw_if_index;
482 : u16 type, len, next;
483 : i16 adv;
484 : u8 err, n_tags;
485 : u64 n_packets, n_bytes;
486 : } eth_input_tag_lookup_t;
487 :
488 : static_always_inline void
489 4761 : eth_input_update_if_counters (vlib_main_t * vm, vnet_main_t * vnm,
490 : eth_input_tag_lookup_t * l)
491 : {
492 4761 : if (l->n_packets == 0 || l->sw_if_index == ~0)
493 4514 : return;
494 :
495 247 : if (l->adv > 0)
496 158 : l->n_bytes += l->n_packets * l->len;
497 :
498 247 : vlib_increment_combined_counter
499 : (vnm->interface_main.combined_sw_if_counters +
500 : VNET_INTERFACE_COUNTER_RX, vm->thread_index, l->sw_if_index,
501 : l->n_packets, l->n_bytes);
502 : }
503 :
504 : static_always_inline void
505 4990 : eth_input_tag_lookup (vlib_main_t * vm, vnet_main_t * vnm,
506 : vlib_node_runtime_t * node, vnet_hw_interface_t * hi,
507 : u64 tag, u16 * next, vlib_buffer_t * b,
508 : eth_input_tag_lookup_t * l, u8 dmac_bad, int is_dot1ad,
509 : int main_is_l3, int check_dmac)
510 : {
511 4990 : ethernet_main_t *em = ðernet_main;
512 :
513 4990 : if ((tag ^ l->tag) & l->mask)
514 : {
515 251 : main_intf_t *mif = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
516 : vlan_intf_t *vif;
517 : qinq_intf_t *qif;
518 : vlan_table_t *vlan_table;
519 : qinq_table_t *qinq_table;
520 251 : u16 *t = (u16 *) & tag;
521 251 : u16 vlan1 = clib_net_to_host_u16 (t[0]) & 0xFFF;
522 251 : u16 vlan2 = clib_net_to_host_u16 (t[2]) & 0xFFF;
523 : u32 matched, is_l2, new_sw_if_index;
524 :
525 251 : vlan_table = vec_elt_at_index (em->vlan_pool, is_dot1ad ?
526 : mif->dot1ad_vlans : mif->dot1q_vlans);
527 251 : vif = &vlan_table->vlans[vlan1];
528 251 : qinq_table = vec_elt_at_index (em->qinq_pool, vif->qinqs);
529 251 : qif = &qinq_table->vlans[vlan2];
530 251 : l->err = ETHERNET_ERROR_NONE;
531 251 : l->type = clib_net_to_host_u16 (t[1]);
532 :
533 251 : if (l->type == ETHERNET_TYPE_VLAN)
534 : {
535 53 : l->type = clib_net_to_host_u16 (t[3]);
536 53 : l->n_tags = 2;
537 53 : matched = eth_identify_subint (hi, SUBINT_CONFIG_VALID |
538 : SUBINT_CONFIG_MATCH_2_TAG, mif, vif,
539 : qif, &new_sw_if_index, &l->err,
540 : &is_l2);
541 : }
542 : else
543 : {
544 198 : l->n_tags = 1;
545 198 : if (vlan1 == 0)
546 : {
547 4 : new_sw_if_index = hi->sw_if_index;
548 4 : l->err = ETHERNET_ERROR_NONE;
549 4 : matched = 1;
550 4 : is_l2 = main_is_l3 == 0;
551 : }
552 : else
553 194 : matched = eth_identify_subint (hi, SUBINT_CONFIG_VALID |
554 : SUBINT_CONFIG_MATCH_1_TAG, mif,
555 : vif, qif, &new_sw_if_index,
556 : &l->err, &is_l2);
557 : }
558 :
559 251 : if (l->sw_if_index != new_sw_if_index)
560 : {
561 247 : eth_input_update_if_counters (vm, vnm, l);
562 247 : l->n_packets = 0;
563 247 : l->n_bytes = 0;
564 247 : l->sw_if_index = new_sw_if_index;
565 : }
566 251 : l->tag = tag;
567 502 : l->mask = (l->n_tags == 2) ?
568 251 : clib_net_to_host_u64 (0xffffffffffffffff) :
569 198 : clib_net_to_host_u64 (0xffffffff00000000);
570 :
571 251 : if (matched && l->sw_if_index == ~0)
572 0 : l->err = ETHERNET_ERROR_DOWN;
573 :
574 251 : l->len = sizeof (ethernet_header_t) +
575 251 : l->n_tags * sizeof (ethernet_vlan_header_t);
576 251 : if (main_is_l3)
577 387 : l->adv = is_l2 ? -(int) sizeof (ethernet_header_t) :
578 162 : l->n_tags * sizeof (ethernet_vlan_header_t);
579 : else
580 26 : l->adv = is_l2 ? 0 : l->len;
581 :
582 251 : if (PREDICT_FALSE (l->err != ETHERNET_ERROR_NONE))
583 4 : l->next = ETHERNET_INPUT_NEXT_DROP;
584 247 : else if (is_l2)
585 89 : l->next = em->l2_next;
586 158 : else if (l->type == ETHERNET_TYPE_IP4)
587 57 : l->next = em->l3_next.input_next_ip4;
588 101 : else if (l->type == ETHERNET_TYPE_IP6)
589 73 : l->next = em->l3_next.input_next_ip6;
590 28 : else if (l->type == ETHERNET_TYPE_MPLS)
591 0 : l->next = em->l3_next.input_next_mpls;
592 28 : else if (em->redirect_l3)
593 0 : l->next = em->redirect_l3_next;
594 : else
595 : {
596 28 : l->next = eth_input_next_by_type (l->type);
597 28 : if (l->next == ETHERNET_INPUT_NEXT_PUNT)
598 0 : l->err = ETHERNET_ERROR_UNKNOWN_TYPE;
599 : }
600 : }
601 :
602 4990 : if (check_dmac && l->adv > 0 && dmac_bad)
603 : {
604 0 : l->err = ETHERNET_ERROR_L3_MAC_MISMATCH;
605 0 : next[0] = ETHERNET_INPUT_NEXT_PUNT;
606 : }
607 : else
608 4990 : next[0] = l->next;
609 :
610 4990 : vlib_buffer_advance (b, l->adv);
611 4990 : vnet_buffer (b)->l2.l2_len = l->len;
612 4990 : vnet_buffer (b)->l3_hdr_offset = vnet_buffer (b)->l2_hdr_offset + l->len;
613 :
614 4990 : if (l->err == ETHERNET_ERROR_NONE)
615 : {
616 4954 : vnet_buffer (b)->sw_if_index[VLIB_RX] = l->sw_if_index;
617 4954 : ethernet_buffer_set_vlan_count (b, l->n_tags);
618 : }
619 : else
620 36 : b->error = node->errors[l->err];
621 :
622 : /* update counters */
623 4990 : l->n_packets += 1;
624 4990 : l->n_bytes += vlib_buffer_length_in_chain (vm, b);
625 4990 : }
626 :
627 : #define DMAC_MASK clib_net_to_host_u64 (0xFFFFFFFFFFFF0000)
628 : #define DMAC_IGBIT clib_net_to_host_u64 (0x0100000000000000)
629 :
630 : #ifdef CLIB_HAVE_VEC256
631 : static_always_inline u32
632 3687420 : is_dmac_bad_x4 (u64 * dmacs, u64 hwaddr)
633 : {
634 3687420 : u64x4 r0 = u64x4_load_unaligned (dmacs) & u64x4_splat (DMAC_MASK);
635 3687420 : r0 = (r0 != u64x4_splat (hwaddr)) & ((r0 & u64x4_splat (DMAC_IGBIT)) == 0);
636 3687420 : return u8x32_msb_mask ((u8x32) (r0));
637 : }
638 : #endif
639 :
640 : static_always_inline u8
641 4763310 : is_dmac_bad (u64 dmac, u64 hwaddr)
642 : {
643 4763310 : u64 r0 = dmac & DMAC_MASK;
644 4763310 : return (r0 != hwaddr) && ((r0 & DMAC_IGBIT) == 0);
645 : }
646 :
647 : static_always_inline u8
648 734 : is_sec_dmac_bad (u64 dmac, u64 hwaddr)
649 : {
650 734 : return ((dmac & DMAC_MASK) != hwaddr);
651 : }
652 :
653 : #ifdef CLIB_HAVE_VEC256
654 : static_always_inline u32
655 5248 : is_sec_dmac_bad_x4 (u64 * dmacs, u64 hwaddr)
656 : {
657 5248 : u64x4 r0 = u64x4_load_unaligned (dmacs) & u64x4_splat (DMAC_MASK);
658 5248 : r0 = (r0 != u64x4_splat (hwaddr));
659 5248 : return u8x32_msb_mask ((u8x32) (r0));
660 : }
661 : #endif
662 :
663 : static_always_inline u8
664 734 : eth_input_sec_dmac_check_x1 (u64 hwaddr, u64 * dmac, u8 * dmac_bad)
665 : {
666 734 : dmac_bad[0] &= is_sec_dmac_bad (dmac[0], hwaddr);
667 734 : return dmac_bad[0];
668 : }
669 :
670 : static_always_inline u32
671 5248 : eth_input_sec_dmac_check_x4 (u64 hwaddr, u64 * dmac, u8 * dmac_bad)
672 : {
673 : #ifdef CLIB_HAVE_VEC256
674 5248 : *(u32 *) (dmac_bad + 0) &= is_sec_dmac_bad_x4 (dmac + 0, hwaddr);
675 : #else
676 0 : dmac_bad[0] &= is_sec_dmac_bad (dmac[0], hwaddr);
677 0 : dmac_bad[1] &= is_sec_dmac_bad (dmac[1], hwaddr);
678 0 : dmac_bad[2] &= is_sec_dmac_bad (dmac[2], hwaddr);
679 0 : dmac_bad[3] &= is_sec_dmac_bad (dmac[3], hwaddr);
680 : #endif
681 5248 : return *(u32 *) dmac_bad;
682 : }
683 :
684 : /*
685 : * DMAC check for ethernet_input_inline()
686 : *
687 : * dmacs and dmacs_bad are arrays that are 2 elements long
688 : * n_packets should be 1 or 2 for ethernet_input_inline()
689 : */
690 : static_always_inline void
691 2381650 : ethernet_input_inline_dmac_check (vnet_hw_interface_t * hi,
692 : u64 * dmacs, u8 * dmacs_bad,
693 : u32 n_packets, ethernet_interface_t * ei,
694 : u8 have_sec_dmac)
695 : {
696 2381650 : u64 hwaddr = ei->address.as_u64;
697 2381650 : u8 bad = 0;
698 :
699 2381650 : ASSERT (0 == ei->address.zero);
700 :
701 2381650 : dmacs_bad[0] = is_dmac_bad (dmacs[0], hwaddr);
702 2381650 : dmacs_bad[1] = ((n_packets > 1) & is_dmac_bad (dmacs[1], hwaddr));
703 :
704 2381650 : bad = dmacs_bad[0] | dmacs_bad[1];
705 :
706 2381650 : if (PREDICT_FALSE (bad && have_sec_dmac))
707 : {
708 : ethernet_interface_address_t *sec_addr;
709 :
710 3 : vec_foreach (sec_addr, ei->secondary_addrs)
711 : {
712 2 : ASSERT (0 == sec_addr->zero);
713 2 : hwaddr = sec_addr->as_u64;
714 :
715 2 : bad = (eth_input_sec_dmac_check_x1 (hwaddr, dmacs, dmacs_bad) |
716 2 : eth_input_sec_dmac_check_x1 (hwaddr, dmacs + 1,
717 : dmacs_bad + 1));
718 :
719 2 : if (!bad)
720 0 : return;
721 : }
722 : }
723 : }
724 :
725 : static_always_inline void
726 283479 : eth_input_process_frame_dmac_check (vnet_hw_interface_t * hi,
727 : u64 * dmacs, u8 * dmacs_bad,
728 : u32 n_packets, ethernet_interface_t * ei,
729 : u8 have_sec_dmac)
730 : {
731 283479 : u64 hwaddr = ei->address.as_u64;
732 283479 : u64 *dmac = dmacs;
733 283479 : u8 *dmac_bad = dmacs_bad;
734 283479 : u32 bad = 0;
735 283479 : i32 n_left = n_packets;
736 :
737 283479 : ASSERT (0 == ei->address.zero);
738 :
739 : #ifdef CLIB_HAVE_VEC256
740 2127190 : while (n_left > 0)
741 : {
742 1843710 : bad |= *(u32 *) (dmac_bad + 0) = is_dmac_bad_x4 (dmac + 0, hwaddr);
743 1843710 : bad |= *(u32 *) (dmac_bad + 4) = is_dmac_bad_x4 (dmac + 4, hwaddr);
744 :
745 : /* next */
746 1843710 : dmac += 8;
747 1843710 : dmac_bad += 8;
748 1843710 : n_left -= 8;
749 : }
750 : #else
751 0 : while (n_left > 0)
752 : {
753 0 : bad |= dmac_bad[0] = is_dmac_bad (dmac[0], hwaddr);
754 0 : bad |= dmac_bad[1] = is_dmac_bad (dmac[1], hwaddr);
755 0 : bad |= dmac_bad[2] = is_dmac_bad (dmac[2], hwaddr);
756 0 : bad |= dmac_bad[3] = is_dmac_bad (dmac[3], hwaddr);
757 :
758 : /* next */
759 0 : dmac += 4;
760 0 : dmac_bad += 4;
761 0 : n_left -= 4;
762 : }
763 : #endif
764 :
765 283479 : if (have_sec_dmac && bad)
766 : {
767 : ethernet_interface_address_t *addr;
768 :
769 178263 : vec_foreach (addr, ei->secondary_addrs)
770 : {
771 178142 : u64 hwaddr = addr->as_u64;
772 178142 : i32 n_left = n_packets;
773 178142 : u64 *dmac = dmacs;
774 178142 : u8 *dmac_bad = dmacs_bad;
775 :
776 178142 : ASSERT (0 == addr->zero);
777 :
778 178142 : bad = 0;
779 :
780 5317540 : while (n_left > 0)
781 : {
782 5139400 : int adv = 0;
783 : int n_bad;
784 :
785 : /* skip any that have already matched */
786 5139400 : if (!dmac_bad[0])
787 : {
788 5133660 : dmac += 1;
789 5133660 : dmac_bad += 1;
790 5133660 : n_left -= 1;
791 5133660 : continue;
792 : }
793 :
794 5732 : n_bad = clib_min (4, n_left);
795 :
796 : /* If >= 4 left, compare 4 together */
797 5732 : if (n_bad == 4)
798 : {
799 5248 : bad |= eth_input_sec_dmac_check_x4 (hwaddr, dmac, dmac_bad);
800 5248 : adv = 4;
801 5248 : n_bad = 0;
802 : }
803 :
804 : /* handle individually */
805 6462 : while (n_bad > 0)
806 : {
807 730 : bad |= eth_input_sec_dmac_check_x1 (hwaddr, dmac + adv,
808 : dmac_bad + adv);
809 730 : adv += 1;
810 730 : n_bad -= 1;
811 : }
812 :
813 5732 : dmac += adv;
814 5732 : dmac_bad += adv;
815 5732 : n_left -= adv;
816 : }
817 :
818 178142 : if (!bad) /* can stop looping if everything matched */
819 177602 : break;
820 : }
821 : }
822 283479 : }
823 :
824 : /* process frame of buffers, store ethertype into array and update
825 : buffer metadata fields depending on interface being l2 or l3 assuming that
826 : packets are untagged. For tagged packets those fields are updated later.
827 : Optionally store Destionation MAC address and tag data into arrays
828 : for further processing */
829 :
830 : STATIC_ASSERT (VLIB_FRAME_SIZE % 8 == 0,
831 : "VLIB_FRAME_SIZE must be power of 8");
832 : static_always_inline void
833 1265850 : eth_input_process_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
834 : vnet_hw_interface_t * hi,
835 : u32 * buffer_indices, u32 n_packets, int main_is_l3,
836 : int ip4_cksum_ok, int dmac_check)
837 : {
838 1265850 : ethernet_main_t *em = ðernet_main;
839 : u16 nexts[VLIB_FRAME_SIZE], *next;
840 1265850 : u16 etypes[VLIB_FRAME_SIZE], *etype = etypes;
841 1265850 : u64 dmacs[VLIB_FRAME_SIZE], *dmac = dmacs;
842 : u8 dmacs_bad[VLIB_FRAME_SIZE];
843 1265850 : u64 tags[VLIB_FRAME_SIZE], *tag = tags;
844 : u16 slowpath_indices[VLIB_FRAME_SIZE];
845 : u16 n_slowpath, i;
846 : u16 next_ip4, next_ip6, next_mpls, next_l2;
847 1265850 : u16 et_ip4 = clib_host_to_net_u16 (ETHERNET_TYPE_IP4);
848 1265850 : u16 et_ip6 = clib_host_to_net_u16 (ETHERNET_TYPE_IP6);
849 1265850 : u16 et_mpls = clib_host_to_net_u16 (ETHERNET_TYPE_MPLS);
850 1265850 : u16 et_vlan = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN);
851 1265850 : u16 et_dot1ad = clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD);
852 1265850 : i32 n_left = n_packets;
853 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
854 1265850 : vlib_buffer_t **b = bufs;
855 1265850 : ethernet_interface_t *ei = ethernet_get_interface (em, hi->hw_if_index);
856 :
857 1265850 : vlib_get_buffers (vm, buffer_indices, b, n_left);
858 :
859 12048900 : while (n_left >= 20)
860 : {
861 10783000 : vlib_buffer_t **ph = b + 16, **pd = b + 8;
862 :
863 10783000 : vlib_prefetch_buffer_header (ph[0], LOAD);
864 10783000 : vlib_prefetch_buffer_data (pd[0], LOAD);
865 10783000 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
866 :
867 10783000 : vlib_prefetch_buffer_header (ph[1], LOAD);
868 10783000 : vlib_prefetch_buffer_data (pd[1], LOAD);
869 10783000 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
870 :
871 10783000 : vlib_prefetch_buffer_header (ph[2], LOAD);
872 10783000 : vlib_prefetch_buffer_data (pd[2], LOAD);
873 10783000 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
874 :
875 10783000 : vlib_prefetch_buffer_header (ph[3], LOAD);
876 10783000 : vlib_prefetch_buffer_data (pd[3], LOAD);
877 10783000 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
878 :
879 10783000 : eth_input_adv_and_flags_x4 (b, main_is_l3);
880 :
881 : /* next */
882 10783000 : b += 4;
883 10783000 : n_left -= 4;
884 10783000 : etype += 4;
885 10783000 : tag += 4;
886 10783000 : dmac += 4;
887 : }
888 5381250 : while (n_left >= 4)
889 : {
890 4115400 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
891 4115400 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
892 4115400 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
893 4115400 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
894 4115400 : eth_input_adv_and_flags_x4 (b, main_is_l3);
895 :
896 : /* next */
897 4115400 : b += 4;
898 4115400 : n_left -= 4;
899 4115400 : etype += 4;
900 4115400 : tag += 4;
901 4115400 : dmac += 4;
902 : }
903 3122150 : while (n_left)
904 : {
905 1856300 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
906 1856300 : eth_input_adv_and_flags_x1 (b, main_is_l3);
907 :
908 : /* next */
909 1856300 : b += 1;
910 1856300 : n_left -= 1;
911 1856300 : etype += 1;
912 1856300 : tag += 1;
913 1856300 : dmac += 1;
914 : }
915 :
916 1265850 : if (dmac_check)
917 : {
918 283479 : if (ei && vec_len (ei->secondary_addrs))
919 282948 : eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets,
920 : ei, 1 /* have_sec_dmac */ );
921 : else
922 531 : eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets,
923 : ei, 0 /* have_sec_dmac */ );
924 : }
925 :
926 1265850 : next_ip4 = em->l3_next.input_next_ip4;
927 1265850 : next_ip6 = em->l3_next.input_next_ip6;
928 1265850 : next_mpls = em->l3_next.input_next_mpls;
929 1265850 : next_l2 = em->l2_next;
930 :
931 1265850 : if (next_ip4 == ETHERNET_INPUT_NEXT_IP4_INPUT && ip4_cksum_ok)
932 0 : next_ip4 = ETHERNET_INPUT_NEXT_IP4_INPUT_NCS;
933 :
934 : #ifdef CLIB_HAVE_VEC256
935 1265850 : u16x16 et16_ip4 = u16x16_splat (et_ip4);
936 1265850 : u16x16 et16_ip6 = u16x16_splat (et_ip6);
937 1265850 : u16x16 et16_mpls = u16x16_splat (et_mpls);
938 1265850 : u16x16 et16_vlan = u16x16_splat (et_vlan);
939 1265850 : u16x16 et16_dot1ad = u16x16_splat (et_dot1ad);
940 1265850 : u16x16 next16_ip4 = u16x16_splat (next_ip4);
941 1265850 : u16x16 next16_ip6 = u16x16_splat (next_ip6);
942 1265850 : u16x16 next16_mpls = u16x16_splat (next_mpls);
943 1265850 : u16x16 next16_l2 = u16x16_splat (next_l2);
944 1265850 : u16x16 zero = { 0 };
945 1265850 : u16x16 stairs = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
946 : #endif
947 :
948 1265850 : etype = etypes;
949 1265850 : n_left = n_packets;
950 1265850 : next = nexts;
951 1265850 : n_slowpath = 0;
952 1265850 : i = 0;
953 :
954 : /* fastpath - in l3 mode hadles ip4, ip6 and mpls packets, other packets
955 : are considered as slowpath, in l2 mode all untagged packets are
956 : considered as fastpath */
957 10930200 : while (n_left > 0)
958 : {
959 : #ifdef CLIB_HAVE_VEC256
960 9664390 : if (n_left >= 16)
961 : {
962 3452380 : u16x16 r = zero;
963 3452380 : u16x16 e16 = u16x16_load_unaligned (etype);
964 3452380 : if (main_is_l3)
965 : {
966 779334 : r += (e16 == et16_ip4) & next16_ip4;
967 779334 : r += (e16 == et16_ip6) & next16_ip6;
968 779334 : r += (e16 == et16_mpls) & next16_mpls;
969 : }
970 : else
971 2673050 : r = ((e16 != et16_vlan) & (e16 != et16_dot1ad)) & next16_l2;
972 3452380 : u16x16_store_unaligned (r, next);
973 :
974 3452380 : if (!u16x16_is_all_zero (r == zero))
975 : {
976 291 : if (u16x16_is_all_zero (r))
977 : {
978 291 : u16x16_store_unaligned (u16x16_splat (i) + stairs,
979 291 : slowpath_indices + n_slowpath);
980 291 : n_slowpath += 16;
981 : }
982 : else
983 : {
984 0 : for (int j = 0; j < 16; j++)
985 0 : if (next[j] == 0)
986 0 : slowpath_indices[n_slowpath++] = i + j;
987 : }
988 : }
989 :
990 3452380 : etype += 16;
991 3452380 : next += 16;
992 3452380 : n_left -= 16;
993 3452380 : i += 16;
994 3452380 : continue;
995 : }
996 : #endif
997 6212010 : if (main_is_l3 && etype[0] == et_ip4)
998 645259 : next[0] = next_ip4;
999 5566750 : else if (main_is_l3 && etype[0] == et_ip6)
1000 561613 : next[0] = next_ip6;
1001 5005140 : else if (main_is_l3 && etype[0] == et_mpls)
1002 125 : next[0] = next_mpls;
1003 5005010 : else if (main_is_l3 == 0 &&
1004 5002640 : etype[0] != et_vlan && etype[0] != et_dot1ad)
1005 5002600 : next[0] = next_l2;
1006 : else
1007 : {
1008 2411 : next[0] = 0;
1009 2411 : slowpath_indices[n_slowpath++] = i;
1010 : }
1011 :
1012 6212010 : etype += 1;
1013 6212010 : next += 1;
1014 6212010 : n_left -= 1;
1015 6212010 : i += 1;
1016 : }
1017 :
1018 1265850 : if (n_slowpath)
1019 : {
1020 2257 : vnet_main_t *vnm = vnet_get_main ();
1021 2257 : n_left = n_slowpath;
1022 2257 : u16 *si = slowpath_indices;
1023 2257 : u32 last_unknown_etype = ~0;
1024 2257 : u32 last_unknown_next = ~0;
1025 2257 : eth_input_tag_lookup_t dot1ad_lookup, dot1q_lookup = {
1026 : .mask = -1LL,
1027 2257 : .tag = tags[si[0]] ^ -1LL,
1028 : .sw_if_index = ~0
1029 : };
1030 :
1031 2257 : clib_memcpy_fast (&dot1ad_lookup, &dot1q_lookup, sizeof (dot1q_lookup));
1032 :
1033 9324 : while (n_left)
1034 : {
1035 7067 : i = si[0];
1036 7067 : u16 etype = etypes[i];
1037 :
1038 7067 : if (etype == et_vlan)
1039 : {
1040 4304 : vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
1041 4304 : eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
1042 4304 : &dot1q_lookup, dmacs_bad[i], 0,
1043 : main_is_l3, dmac_check);
1044 :
1045 : }
1046 2763 : else if (etype == et_dot1ad)
1047 : {
1048 686 : vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
1049 686 : eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
1050 686 : &dot1ad_lookup, dmacs_bad[i], 1,
1051 : main_is_l3, dmac_check);
1052 : }
1053 : else
1054 : {
1055 : /* untagged packet with not well known etyertype */
1056 2077 : if (last_unknown_etype != etype)
1057 : {
1058 2075 : last_unknown_etype = etype;
1059 2075 : etype = clib_host_to_net_u16 (etype);
1060 2075 : last_unknown_next = eth_input_next_by_type (etype);
1061 : }
1062 2077 : if (dmac_check && main_is_l3 && dmacs_bad[i])
1063 4 : {
1064 4 : vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
1065 4 : b->error = node->errors[ETHERNET_ERROR_L3_MAC_MISMATCH];
1066 4 : nexts[i] = ETHERNET_INPUT_NEXT_PUNT;
1067 : }
1068 : else
1069 2073 : nexts[i] = last_unknown_next;
1070 : }
1071 :
1072 : /* next */
1073 7067 : n_left--;
1074 7067 : si++;
1075 : }
1076 :
1077 2257 : eth_input_update_if_counters (vm, vnm, &dot1q_lookup);
1078 2257 : eth_input_update_if_counters (vm, vnm, &dot1ad_lookup);
1079 : }
1080 :
1081 1265850 : vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_packets);
1082 1265850 : }
1083 :
1084 : static_always_inline void
1085 1265850 : eth_input_single_int (vlib_main_t * vm, vlib_node_runtime_t * node,
1086 : vnet_hw_interface_t * hi, u32 * from, u32 n_pkts,
1087 : int ip4_cksum_ok)
1088 : {
1089 1265850 : ethernet_main_t *em = ðernet_main;
1090 : ethernet_interface_t *ei;
1091 1265850 : ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
1092 1265850 : main_intf_t *intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1093 1265850 : subint_config_t *subint0 = &intf0->untagged_subint;
1094 :
1095 1265850 : int main_is_l3 = (subint0->flags & SUBINT_CONFIG_L2) == 0;
1096 1265850 : int int_is_l3 = ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3;
1097 :
1098 1265850 : if (main_is_l3)
1099 : {
1100 283463 : if (int_is_l3 || /* DMAC filter already done by NIC */
1101 283463 : ((hi->l2_if_count != 0) && (hi->l3_if_count == 0)))
1102 : { /* All L2 usage - DMAC check not needed */
1103 32 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1104 : /*is_l3 */ 1, ip4_cksum_ok, 0);
1105 : }
1106 : else
1107 : { /* DMAC check needed for L3 */
1108 283431 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1109 : /*is_l3 */ 1, ip4_cksum_ok, 1);
1110 : }
1111 283463 : return;
1112 : }
1113 : else
1114 : {
1115 982383 : if (hi->l3_if_count == 0)
1116 : { /* All L2 usage - DMAC check not needed */
1117 982336 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1118 : /*is_l3 */ 0, ip4_cksum_ok, 0);
1119 : }
1120 : else
1121 : { /* DMAC check needed for L3 */
1122 47 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1123 : /*is_l3 */ 0, ip4_cksum_ok, 1);
1124 : }
1125 982383 : return;
1126 : }
1127 : }
1128 :
1129 : static_always_inline void
1130 1874010 : ethernet_input_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
1131 : vlib_frame_t * from_frame)
1132 : {
1133 1874010 : vnet_main_t *vnm = vnet_get_main ();
1134 : u32 *from, n_left;
1135 1874010 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
1136 : {
1137 15578 : from = vlib_frame_vector_args (from_frame);
1138 15578 : n_left = from_frame->n_vectors;
1139 :
1140 540146 : while (n_left)
1141 : {
1142 : ethernet_input_trace_t *t0;
1143 524570 : vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
1144 :
1145 524586 : if (b0->flags & VLIB_BUFFER_IS_TRACED)
1146 : {
1147 523534 : t0 = vlib_add_trace (vm, node, b0,
1148 : sizeof (ethernet_input_trace_t));
1149 523645 : clib_memcpy_fast (t0->packet_data, b0->data + b0->current_data,
1150 : sizeof (t0->packet_data));
1151 523560 : t0->frame_flags = from_frame->flags;
1152 523551 : clib_memcpy_fast (&t0->frame_data,
1153 523560 : vlib_frame_scalar_args (from_frame),
1154 : sizeof (ethernet_input_frame_t));
1155 : }
1156 524568 : from += 1;
1157 524568 : n_left -= 1;
1158 : }
1159 : }
1160 :
1161 : /* rx pcap capture if enabled */
1162 1874010 : if (PREDICT_FALSE (vnm->pcap.pcap_rx_enable))
1163 : {
1164 : u32 bi0;
1165 9 : vnet_pcap_t *pp = &vnm->pcap;
1166 :
1167 9 : from = vlib_frame_vector_args (from_frame);
1168 9 : n_left = from_frame->n_vectors;
1169 188 : while (n_left > 0)
1170 : {
1171 : vlib_buffer_t *b0;
1172 179 : bi0 = from[0];
1173 179 : from++;
1174 179 : n_left--;
1175 179 : b0 = vlib_get_buffer (vm, bi0);
1176 179 : if (vnet_is_packet_pcaped (pp, b0, ~0))
1177 87 : pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt);
1178 : }
1179 : }
1180 1874010 : }
1181 :
1182 : static_always_inline void
1183 608165 : ethernet_input_inline (vlib_main_t * vm,
1184 : vlib_node_runtime_t * node,
1185 : u32 * from, u32 n_packets,
1186 : ethernet_input_variant_t variant)
1187 : {
1188 608165 : vnet_main_t *vnm = vnet_get_main ();
1189 608165 : ethernet_main_t *em = ðernet_main;
1190 : vlib_node_runtime_t *error_node;
1191 : u32 n_left_from, next_index, *to_next;
1192 : u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
1193 608165 : u32 thread_index = vm->thread_index;
1194 608165 : u32 cached_sw_if_index = ~0;
1195 608165 : u32 cached_is_l2 = 0; /* shut up gcc */
1196 608165 : vnet_hw_interface_t *hi = NULL; /* used for main interface only */
1197 608165 : ethernet_interface_t *ei = NULL;
1198 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
1199 608165 : vlib_buffer_t **b = bufs;
1200 :
1201 608165 : if (variant != ETHERNET_INPUT_VARIANT_ETHERNET)
1202 0 : error_node = vlib_node_get_runtime (vm, ethernet_input_node.index);
1203 : else
1204 608165 : error_node = node;
1205 :
1206 608165 : n_left_from = n_packets;
1207 :
1208 608165 : next_index = node->cached_next_index;
1209 608165 : stats_sw_if_index = node->runtime_data[0];
1210 608165 : stats_n_packets = stats_n_bytes = 0;
1211 608165 : vlib_get_buffers (vm, from, bufs, n_left_from);
1212 :
1213 1216330 : while (n_left_from > 0)
1214 : {
1215 : u32 n_left_to_next;
1216 :
1217 608168 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1218 :
1219 6432840 : while (n_left_from >= 4 && n_left_to_next >= 2)
1220 : {
1221 : u32 bi0, bi1;
1222 : vlib_buffer_t *b0, *b1;
1223 : u8 next0, next1, error0, error1;
1224 : u16 type0, orig_type0, type1, orig_type1;
1225 : u16 outer_id0, inner_id0, outer_id1, inner_id1;
1226 : u32 match_flags0, match_flags1;
1227 : u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1,
1228 : new_sw_if_index1, len1;
1229 : vnet_hw_interface_t *hi0, *hi1;
1230 : main_intf_t *main_intf0, *main_intf1;
1231 : vlan_intf_t *vlan_intf0, *vlan_intf1;
1232 : qinq_intf_t *qinq_intf0, *qinq_intf1;
1233 : u32 is_l20, is_l21;
1234 : ethernet_header_t *e0, *e1;
1235 : u64 dmacs[2];
1236 : u8 dmacs_bad[2];
1237 :
1238 : /* Prefetch next iteration. */
1239 : {
1240 5824670 : vlib_prefetch_buffer_header (b[2], STORE);
1241 5824670 : vlib_prefetch_buffer_header (b[3], STORE);
1242 :
1243 5824670 : CLIB_PREFETCH (b[2]->data, sizeof (ethernet_header_t), LOAD);
1244 5824670 : CLIB_PREFETCH (b[3]->data, sizeof (ethernet_header_t), LOAD);
1245 : }
1246 :
1247 5824670 : bi0 = from[0];
1248 5824670 : bi1 = from[1];
1249 5824670 : to_next[0] = bi0;
1250 5824670 : to_next[1] = bi1;
1251 5824670 : from += 2;
1252 5824670 : to_next += 2;
1253 5824670 : n_left_to_next -= 2;
1254 5824670 : n_left_from -= 2;
1255 :
1256 5824670 : b0 = b[0];
1257 5824670 : b1 = b[1];
1258 5824670 : b += 2;
1259 :
1260 5824670 : error0 = error1 = ETHERNET_ERROR_NONE;
1261 5824670 : e0 = vlib_buffer_get_current (b0);
1262 5824670 : type0 = clib_net_to_host_u16 (e0->type);
1263 5824670 : e1 = vlib_buffer_get_current (b1);
1264 5824670 : type1 = clib_net_to_host_u16 (e1->type);
1265 :
1266 : /* Set the L2 header offset for all packets */
1267 5824670 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1268 5824670 : vnet_buffer (b1)->l2_hdr_offset = b1->current_data;
1269 5824670 : b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1270 5824670 : b1->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1271 :
1272 : /* Speed-path for the untagged case */
1273 5824670 : if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET
1274 : && !ethernet_frame_is_any_tagged_x2 (type0,
1275 : type1)))
1276 : {
1277 : main_intf_t *intf0;
1278 : subint_config_t *subint0;
1279 : u32 sw_if_index0, sw_if_index1;
1280 :
1281 5824420 : sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1282 5824420 : sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1283 5824420 : is_l20 = cached_is_l2;
1284 :
1285 : /* This is probably wholly unnecessary */
1286 5824420 : if (PREDICT_FALSE (sw_if_index0 != sw_if_index1))
1287 89986 : goto slowpath;
1288 :
1289 : /* Now sw_if_index0 == sw_if_index1 */
1290 5734430 : if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1291 : {
1292 508201 : cached_sw_if_index = sw_if_index0;
1293 508201 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1294 508201 : ei = ethernet_get_interface (em, hi->hw_if_index);
1295 508201 : intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1296 508201 : subint0 = &intf0->untagged_subint;
1297 508201 : cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1298 : }
1299 :
1300 5734430 : if (PREDICT_TRUE (is_l20 != 0))
1301 : {
1302 3774900 : vnet_buffer (b0)->l3_hdr_offset =
1303 3774900 : vnet_buffer (b0)->l2_hdr_offset +
1304 : sizeof (ethernet_header_t);
1305 3774900 : vnet_buffer (b1)->l3_hdr_offset =
1306 3774900 : vnet_buffer (b1)->l2_hdr_offset +
1307 : sizeof (ethernet_header_t);
1308 3774900 : b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1309 3774900 : b1->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1310 3774900 : next0 = em->l2_next;
1311 3774900 : vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1312 3774900 : next1 = em->l2_next;
1313 3774900 : vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t);
1314 : }
1315 : else
1316 : {
1317 1959530 : if (ei && (ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3))
1318 0 : goto skip_dmac_check01;
1319 :
1320 1959530 : dmacs[0] = *(u64 *) e0;
1321 1959530 : dmacs[1] = *(u64 *) e1;
1322 :
1323 1959530 : if (ei && vec_len (ei->secondary_addrs))
1324 1959510 : ethernet_input_inline_dmac_check (hi, dmacs,
1325 : dmacs_bad,
1326 : 2 /* n_packets */ ,
1327 : ei,
1328 : 1 /* have_sec_dmac */ );
1329 : else
1330 21 : ethernet_input_inline_dmac_check (hi, dmacs,
1331 : dmacs_bad,
1332 : 2 /* n_packets */ ,
1333 : ei,
1334 : 0 /* have_sec_dmac */ );
1335 :
1336 1959530 : if (dmacs_bad[0])
1337 21 : error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1338 1959530 : if (dmacs_bad[1])
1339 21 : error1 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1340 :
1341 1959510 : skip_dmac_check01:
1342 1959530 : vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1343 1959530 : determine_next_node (em, variant, 0, type0, b0,
1344 : &error0, &next0);
1345 1959530 : vlib_buffer_advance (b1, sizeof (ethernet_header_t));
1346 1959530 : determine_next_node (em, variant, 0, type1, b1,
1347 : &error1, &next1);
1348 : }
1349 5734430 : goto ship_it01;
1350 : }
1351 :
1352 : /* Slow-path for the tagged case */
1353 254 : slowpath:
1354 90240 : parse_header (variant,
1355 : b0,
1356 : &type0,
1357 : &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1358 :
1359 90240 : parse_header (variant,
1360 : b1,
1361 : &type1,
1362 : &orig_type1, &outer_id1, &inner_id1, &match_flags1);
1363 :
1364 90240 : old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1365 90240 : old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1366 :
1367 90240 : eth_vlan_table_lookups (em,
1368 : vnm,
1369 : old_sw_if_index0,
1370 : orig_type0,
1371 : outer_id0,
1372 : inner_id0,
1373 : &hi0,
1374 : &main_intf0, &vlan_intf0, &qinq_intf0);
1375 :
1376 90240 : eth_vlan_table_lookups (em,
1377 : vnm,
1378 : old_sw_if_index1,
1379 : orig_type1,
1380 : outer_id1,
1381 : inner_id1,
1382 : &hi1,
1383 : &main_intf1, &vlan_intf1, &qinq_intf1);
1384 :
1385 90240 : identify_subint (em,
1386 : hi0,
1387 : b0,
1388 : match_flags0,
1389 : main_intf0,
1390 : vlan_intf0,
1391 : qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1392 :
1393 90240 : identify_subint (em,
1394 : hi1,
1395 : b1,
1396 : match_flags1,
1397 : main_intf1,
1398 : vlan_intf1,
1399 : qinq_intf1, &new_sw_if_index1, &error1, &is_l21);
1400 :
1401 : // Save RX sw_if_index for later nodes
1402 90240 : vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1403 90240 : error0 !=
1404 90240 : ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1405 90240 : vnet_buffer (b1)->sw_if_index[VLIB_RX] =
1406 90240 : error1 !=
1407 90240 : ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1;
1408 :
1409 : // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1)
1410 90240 : if (((new_sw_if_index0 != ~0)
1411 90240 : && (new_sw_if_index0 != old_sw_if_index0))
1412 89986 : || ((new_sw_if_index1 != ~0)
1413 89986 : && (new_sw_if_index1 != old_sw_if_index1)))
1414 : {
1415 :
1416 254 : len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1417 254 : - vnet_buffer (b0)->l2_hdr_offset;
1418 254 : len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data
1419 254 : - vnet_buffer (b1)->l2_hdr_offset;
1420 :
1421 254 : stats_n_packets += 2;
1422 254 : stats_n_bytes += len0 + len1;
1423 :
1424 254 : if (PREDICT_FALSE
1425 : (!(new_sw_if_index0 == stats_sw_if_index
1426 : && new_sw_if_index1 == stats_sw_if_index)))
1427 : {
1428 2 : stats_n_packets -= 2;
1429 2 : stats_n_bytes -= len0 + len1;
1430 :
1431 2 : if (new_sw_if_index0 != old_sw_if_index0
1432 2 : && new_sw_if_index0 != ~0)
1433 2 : vlib_increment_combined_counter (vnm->
1434 : interface_main.combined_sw_if_counters
1435 : +
1436 : VNET_INTERFACE_COUNTER_RX,
1437 : thread_index,
1438 : new_sw_if_index0, 1,
1439 : len0);
1440 2 : if (new_sw_if_index1 != old_sw_if_index1
1441 2 : && new_sw_if_index1 != ~0)
1442 2 : vlib_increment_combined_counter (vnm->
1443 : interface_main.combined_sw_if_counters
1444 : +
1445 : VNET_INTERFACE_COUNTER_RX,
1446 : thread_index,
1447 : new_sw_if_index1, 1,
1448 : len1);
1449 :
1450 2 : if (new_sw_if_index0 == new_sw_if_index1)
1451 : {
1452 2 : if (stats_n_packets > 0)
1453 : {
1454 0 : vlib_increment_combined_counter
1455 : (vnm->interface_main.combined_sw_if_counters
1456 : + VNET_INTERFACE_COUNTER_RX,
1457 : thread_index,
1458 : stats_sw_if_index,
1459 : stats_n_packets, stats_n_bytes);
1460 0 : stats_n_packets = stats_n_bytes = 0;
1461 : }
1462 2 : stats_sw_if_index = new_sw_if_index0;
1463 : }
1464 : }
1465 : }
1466 :
1467 90240 : if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1468 0 : is_l20 = is_l21 = 0;
1469 :
1470 90240 : determine_next_node (em, variant, is_l20, type0, b0, &error0,
1471 : &next0);
1472 90240 : determine_next_node (em, variant, is_l21, type1, b1, &error1,
1473 : &next1);
1474 :
1475 5824670 : ship_it01:
1476 5824670 : b0->error = error_node->errors[error0];
1477 5824670 : b1->error = error_node->errors[error1];
1478 :
1479 : // verify speculative enqueue
1480 5824670 : vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1481 : n_left_to_next, bi0, bi1, next0,
1482 : next1);
1483 : }
1484 :
1485 1741190 : while (n_left_from > 0 && n_left_to_next > 0)
1486 : {
1487 : u32 bi0;
1488 : vlib_buffer_t *b0;
1489 : u8 error0, next0;
1490 : u16 type0, orig_type0;
1491 : u16 outer_id0, inner_id0;
1492 : u32 match_flags0;
1493 : u32 old_sw_if_index0, new_sw_if_index0, len0;
1494 : vnet_hw_interface_t *hi0;
1495 : main_intf_t *main_intf0;
1496 : vlan_intf_t *vlan_intf0;
1497 : qinq_intf_t *qinq_intf0;
1498 : ethernet_header_t *e0;
1499 : u32 is_l20;
1500 : u64 dmacs[2];
1501 : u8 dmacs_bad[2];
1502 :
1503 : // Prefetch next iteration
1504 1133020 : if (n_left_from > 1)
1505 : {
1506 524860 : vlib_prefetch_buffer_header (b[1], STORE);
1507 524860 : clib_prefetch_load (b[1]->data);
1508 : }
1509 :
1510 1133020 : bi0 = from[0];
1511 1133020 : to_next[0] = bi0;
1512 1133020 : from += 1;
1513 1133020 : to_next += 1;
1514 1133020 : n_left_from -= 1;
1515 1133020 : n_left_to_next -= 1;
1516 :
1517 1133020 : b0 = b[0];
1518 1133020 : b += 1;
1519 :
1520 1133020 : error0 = ETHERNET_ERROR_NONE;
1521 1133020 : e0 = vlib_buffer_get_current (b0);
1522 1133020 : type0 = clib_net_to_host_u16 (e0->type);
1523 :
1524 : /* Set the L2 header offset for all packets */
1525 1133020 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1526 1133020 : b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1527 :
1528 : /* Speed-path for the untagged case */
1529 1133020 : if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET
1530 : && !ethernet_frame_is_tagged (type0)))
1531 : {
1532 : main_intf_t *intf0;
1533 : subint_config_t *subint0;
1534 : u32 sw_if_index0;
1535 :
1536 1133020 : sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1537 1133020 : is_l20 = cached_is_l2;
1538 :
1539 1133020 : if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1540 : {
1541 262803 : cached_sw_if_index = sw_if_index0;
1542 262803 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1543 262803 : ei = ethernet_get_interface (em, hi->hw_if_index);
1544 262803 : intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1545 262803 : subint0 = &intf0->untagged_subint;
1546 262803 : cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1547 : }
1548 :
1549 :
1550 1133020 : if (PREDICT_TRUE (is_l20 != 0))
1551 : {
1552 738737 : vnet_buffer (b0)->l3_hdr_offset =
1553 738737 : vnet_buffer (b0)->l2_hdr_offset +
1554 : sizeof (ethernet_header_t);
1555 738737 : b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1556 738737 : next0 = em->l2_next;
1557 738737 : vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1558 : }
1559 : else
1560 : {
1561 394282 : if (ei && ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3)
1562 0 : goto skip_dmac_check0;
1563 :
1564 394282 : dmacs[0] = *(u64 *) e0;
1565 :
1566 394282 : if (ei)
1567 : {
1568 394282 : if (vec_len (ei->secondary_addrs))
1569 394166 : ethernet_input_inline_dmac_check (
1570 : hi, dmacs, dmacs_bad, 1 /* n_packets */, ei,
1571 : 1 /* have_sec_dmac */);
1572 : else
1573 116 : ethernet_input_inline_dmac_check (
1574 : hi, dmacs, dmacs_bad, 1 /* n_packets */, ei,
1575 : 0 /* have_sec_dmac */);
1576 :
1577 394282 : if (dmacs_bad[0])
1578 5 : error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1579 : }
1580 :
1581 394277 : skip_dmac_check0:
1582 394282 : vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1583 394282 : determine_next_node (em, variant, 0, type0, b0,
1584 : &error0, &next0);
1585 : }
1586 1133020 : goto ship_it0;
1587 : }
1588 :
1589 : /* Slow-path for the tagged case */
1590 6 : parse_header (variant,
1591 : b0,
1592 : &type0,
1593 : &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1594 :
1595 6 : old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1596 :
1597 6 : eth_vlan_table_lookups (em,
1598 : vnm,
1599 : old_sw_if_index0,
1600 : orig_type0,
1601 : outer_id0,
1602 : inner_id0,
1603 : &hi0,
1604 : &main_intf0, &vlan_intf0, &qinq_intf0);
1605 :
1606 6 : identify_subint (em,
1607 : hi0,
1608 : b0,
1609 : match_flags0,
1610 : main_intf0,
1611 : vlan_intf0,
1612 : qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1613 :
1614 : // Save RX sw_if_index for later nodes
1615 6 : vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1616 6 : error0 !=
1617 6 : ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1618 :
1619 : // Increment subinterface stats
1620 : // Note that interface-level counters have already been incremented
1621 : // prior to calling this function. Thus only subinterface counters
1622 : // are incremented here.
1623 : //
1624 : // Interface level counters include packets received on the main
1625 : // interface and all subinterfaces. Subinterface level counters
1626 : // include only those packets received on that subinterface
1627 : // Increment stats if the subint is valid and it is not the main intf
1628 6 : if ((new_sw_if_index0 != ~0)
1629 6 : && (new_sw_if_index0 != old_sw_if_index0))
1630 : {
1631 :
1632 6 : len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1633 6 : - vnet_buffer (b0)->l2_hdr_offset;
1634 :
1635 6 : stats_n_packets += 1;
1636 6 : stats_n_bytes += len0;
1637 :
1638 : // Batch stat increments from the same subinterface so counters
1639 : // don't need to be incremented for every packet.
1640 6 : if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index))
1641 : {
1642 0 : stats_n_packets -= 1;
1643 0 : stats_n_bytes -= len0;
1644 :
1645 0 : if (new_sw_if_index0 != ~0)
1646 0 : vlib_increment_combined_counter
1647 : (vnm->interface_main.combined_sw_if_counters
1648 : + VNET_INTERFACE_COUNTER_RX,
1649 : thread_index, new_sw_if_index0, 1, len0);
1650 0 : if (stats_n_packets > 0)
1651 : {
1652 0 : vlib_increment_combined_counter
1653 : (vnm->interface_main.combined_sw_if_counters
1654 : + VNET_INTERFACE_COUNTER_RX,
1655 : thread_index,
1656 : stats_sw_if_index, stats_n_packets, stats_n_bytes);
1657 0 : stats_n_packets = stats_n_bytes = 0;
1658 : }
1659 0 : stats_sw_if_index = new_sw_if_index0;
1660 : }
1661 : }
1662 :
1663 6 : if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1664 0 : is_l20 = 0;
1665 :
1666 6 : determine_next_node (em, variant, is_l20, type0, b0, &error0,
1667 : &next0);
1668 :
1669 1133020 : ship_it0:
1670 1133020 : b0->error = error_node->errors[error0];
1671 :
1672 : // verify speculative enqueue
1673 1133020 : vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1674 : to_next, n_left_to_next,
1675 : bi0, next0);
1676 : }
1677 :
1678 608168 : vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1679 : }
1680 :
1681 : // Increment any remaining batched stats
1682 608165 : if (stats_n_packets > 0)
1683 : {
1684 4 : vlib_increment_combined_counter
1685 : (vnm->interface_main.combined_sw_if_counters
1686 : + VNET_INTERFACE_COUNTER_RX,
1687 : thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
1688 4 : node->runtime_data[0] = stats_sw_if_index;
1689 : }
1690 608165 : }
1691 :
1692 1876310 : VLIB_NODE_FN (ethernet_input_node) (vlib_main_t * vm,
1693 : vlib_node_runtime_t * node,
1694 : vlib_frame_t * frame)
1695 : {
1696 1874010 : vnet_main_t *vnm = vnet_get_main ();
1697 1874010 : u32 *from = vlib_frame_vector_args (frame);
1698 1874010 : u32 n_packets = frame->n_vectors;
1699 :
1700 1874010 : ethernet_input_trace (vm, node, frame);
1701 :
1702 1874010 : if (frame->flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
1703 : {
1704 1265850 : ethernet_input_frame_t *ef = vlib_frame_scalar_args (frame);
1705 1265850 : int ip4_cksum_ok = (frame->flags & ETH_INPUT_FRAME_F_IP4_CKSUM_OK) != 0;
1706 1265850 : vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, ef->hw_if_index);
1707 1265850 : eth_input_single_int (vm, node, hi, from, n_packets, ip4_cksum_ok);
1708 : }
1709 : else
1710 608165 : ethernet_input_inline (vm, node, from, n_packets,
1711 : ETHERNET_INPUT_VARIANT_ETHERNET);
1712 1874010 : return n_packets;
1713 : }
1714 :
1715 2300 : VLIB_NODE_FN (ethernet_input_type_node) (vlib_main_t * vm,
1716 : vlib_node_runtime_t * node,
1717 : vlib_frame_t * from_frame)
1718 : {
1719 0 : u32 *from = vlib_frame_vector_args (from_frame);
1720 0 : u32 n_packets = from_frame->n_vectors;
1721 0 : ethernet_input_trace (vm, node, from_frame);
1722 0 : ethernet_input_inline (vm, node, from, n_packets,
1723 : ETHERNET_INPUT_VARIANT_ETHERNET_TYPE);
1724 0 : return n_packets;
1725 : }
1726 :
1727 2300 : VLIB_NODE_FN (ethernet_input_not_l2_node) (vlib_main_t * vm,
1728 : vlib_node_runtime_t * node,
1729 : vlib_frame_t * from_frame)
1730 : {
1731 0 : u32 *from = vlib_frame_vector_args (from_frame);
1732 0 : u32 n_packets = from_frame->n_vectors;
1733 0 : ethernet_input_trace (vm, node, from_frame);
1734 0 : ethernet_input_inline (vm, node, from, n_packets,
1735 : ETHERNET_INPUT_VARIANT_NOT_L2);
1736 0 : return n_packets;
1737 : }
1738 :
1739 :
1740 : // Return the subinterface config struct for the given sw_if_index
1741 : // Also return via parameter the appropriate match flags for the
1742 : // configured number of tags.
1743 : // On error (unsupported or not ethernet) return 0.
1744 : static subint_config_t *
1745 28865 : ethernet_sw_interface_get_config (vnet_main_t * vnm,
1746 : u32 sw_if_index,
1747 : u32 * flags, u32 * unsupported)
1748 : {
1749 28865 : ethernet_main_t *em = ðernet_main;
1750 : vnet_hw_interface_t *hi;
1751 : vnet_sw_interface_t *si;
1752 : main_intf_t *main_intf;
1753 : vlan_table_t *vlan_table;
1754 : qinq_table_t *qinq_table;
1755 28865 : subint_config_t *subint = 0;
1756 :
1757 28865 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1758 :
1759 28865 : if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index))
1760 : {
1761 3094 : *unsupported = 0;
1762 3094 : goto done; // non-ethernet interface
1763 : }
1764 :
1765 : // ensure there's an entry for the main intf (shouldn't really be necessary)
1766 25771 : vec_validate (em->main_intfs, hi->hw_if_index);
1767 25771 : main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1768 :
1769 : // Locate the subint for the given ethernet config
1770 25771 : si = vnet_get_sw_interface (vnm, sw_if_index);
1771 :
1772 25771 : if (si->type == VNET_SW_INTERFACE_TYPE_P2P)
1773 : {
1774 2107 : p2p_ethernet_main_t *p2pm = &p2p_main;
1775 : u32 p2pe_sw_if_index =
1776 2107 : p2p_ethernet_lookup (hi->hw_if_index, si->p2p.client_mac);
1777 2107 : if (p2pe_sw_if_index == ~0)
1778 : {
1779 1023 : pool_get (p2pm->p2p_subif_pool, subint);
1780 1023 : si->p2p.pool_index = subint - p2pm->p2p_subif_pool;
1781 : }
1782 : else
1783 1084 : subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index);
1784 2107 : *flags = SUBINT_CONFIG_P2P;
1785 : }
1786 23664 : else if (si->type == VNET_SW_INTERFACE_TYPE_PIPE)
1787 : {
1788 : pipe_t *pipe;
1789 :
1790 10 : pipe = pipe_get (sw_if_index);
1791 10 : subint = &pipe->subint;
1792 10 : *flags = SUBINT_CONFIG_P2P;
1793 : }
1794 23654 : else if (si->sub.eth.flags.default_sub)
1795 : {
1796 0 : subint = &main_intf->default_subint;
1797 0 : *flags = SUBINT_CONFIG_MATCH_1_TAG |
1798 : SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG;
1799 : }
1800 23654 : else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0))
1801 : {
1802 : // if no flags are set then this is a main interface
1803 : // so treat as untagged
1804 23296 : subint = &main_intf->untagged_subint;
1805 23296 : *flags = SUBINT_CONFIG_MATCH_0_TAG;
1806 : }
1807 : else
1808 : {
1809 : // one or two tags
1810 : // first get the vlan table
1811 358 : if (si->sub.eth.flags.dot1ad)
1812 : {
1813 51 : if (main_intf->dot1ad_vlans == 0)
1814 : {
1815 : // Allocate a vlan table from the pool
1816 19 : pool_get (em->vlan_pool, vlan_table);
1817 19 : main_intf->dot1ad_vlans = vlan_table - em->vlan_pool;
1818 : }
1819 : else
1820 : {
1821 : // Get ptr to existing vlan table
1822 32 : vlan_table =
1823 32 : vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans);
1824 : }
1825 : }
1826 : else
1827 : { // dot1q
1828 307 : if (main_intf->dot1q_vlans == 0)
1829 : {
1830 : // Allocate a vlan table from the pool
1831 34 : pool_get (em->vlan_pool, vlan_table);
1832 34 : main_intf->dot1q_vlans = vlan_table - em->vlan_pool;
1833 : }
1834 : else
1835 : {
1836 : // Get ptr to existing vlan table
1837 273 : vlan_table =
1838 273 : vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans);
1839 : }
1840 : }
1841 :
1842 358 : if (si->sub.eth.flags.one_tag)
1843 : {
1844 614 : *flags = si->sub.eth.flags.exact_match ?
1845 307 : SUBINT_CONFIG_MATCH_1_TAG :
1846 : (SUBINT_CONFIG_MATCH_1_TAG |
1847 : SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG);
1848 :
1849 307 : if (si->sub.eth.flags.outer_vlan_id_any)
1850 : {
1851 : // not implemented yet
1852 0 : *unsupported = 1;
1853 0 : goto done;
1854 : }
1855 : else
1856 : {
1857 : // a single vlan, a common case
1858 307 : subint =
1859 307 : &vlan_table->vlans[si->sub.eth.
1860 : outer_vlan_id].single_tag_subint;
1861 : }
1862 :
1863 : }
1864 : else
1865 : {
1866 : // Two tags
1867 102 : *flags = si->sub.eth.flags.exact_match ?
1868 51 : SUBINT_CONFIG_MATCH_2_TAG :
1869 : (SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG);
1870 :
1871 51 : if (si->sub.eth.flags.outer_vlan_id_any
1872 0 : && si->sub.eth.flags.inner_vlan_id_any)
1873 : {
1874 : // not implemented yet
1875 0 : *unsupported = 1;
1876 0 : goto done;
1877 : }
1878 :
1879 51 : if (si->sub.eth.flags.inner_vlan_id_any)
1880 : {
1881 : // a specific outer and "any" inner
1882 : // don't need a qinq table for this
1883 0 : subint =
1884 0 : &vlan_table->vlans[si->sub.eth.
1885 : outer_vlan_id].inner_any_subint;
1886 0 : if (si->sub.eth.flags.exact_match)
1887 : {
1888 0 : *flags = SUBINT_CONFIG_MATCH_2_TAG;
1889 : }
1890 : else
1891 : {
1892 0 : *flags = SUBINT_CONFIG_MATCH_2_TAG |
1893 : SUBINT_CONFIG_MATCH_3_TAG;
1894 : }
1895 : }
1896 : else
1897 : {
1898 : // a specific outer + specific innner vlan id, a common case
1899 :
1900 : // get the qinq table
1901 51 : if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0)
1902 : {
1903 : // Allocate a qinq table from the pool
1904 19 : pool_get (em->qinq_pool, qinq_table);
1905 19 : vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs =
1906 19 : qinq_table - em->qinq_pool;
1907 : }
1908 : else
1909 : {
1910 : // Get ptr to existing qinq table
1911 32 : qinq_table =
1912 32 : vec_elt_at_index (em->qinq_pool,
1913 : vlan_table->vlans[si->sub.
1914 : eth.outer_vlan_id].
1915 : qinqs);
1916 : }
1917 51 : subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint;
1918 : }
1919 : }
1920 : }
1921 :
1922 28865 : done:
1923 28865 : return subint;
1924 : }
1925 :
1926 : static clib_error_t *
1927 13514 : ethernet_sw_interface_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
1928 : {
1929 : subint_config_t *subint;
1930 : u32 placeholder_flags;
1931 : u32 placeholder_unsup;
1932 13514 : clib_error_t *error = 0;
1933 :
1934 : // Find the config for this subinterface
1935 : subint =
1936 13514 : ethernet_sw_interface_get_config (vnm, sw_if_index, &placeholder_flags,
1937 : &placeholder_unsup);
1938 :
1939 13514 : if (subint == 0)
1940 : {
1941 : // not implemented yet or not ethernet
1942 1214 : goto done;
1943 : }
1944 :
1945 12300 : subint->sw_if_index =
1946 12300 : ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0);
1947 :
1948 13514 : done:
1949 13514 : return error;
1950 : }
1951 :
1952 2881 : VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ethernet_sw_interface_up_down);
1953 :
1954 :
1955 : #ifndef CLIB_MARCH_VARIANT
1956 : // Set the L2/L3 mode for the subinterface
1957 : void
1958 3553 : ethernet_sw_interface_set_l2_mode (vnet_main_t * vnm, u32 sw_if_index, u32 l2)
1959 : {
1960 : subint_config_t *subint;
1961 : u32 placeholder_flags;
1962 : u32 placeholder_unsup;
1963 : int is_port;
1964 3553 : vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index);
1965 :
1966 3553 : is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB);
1967 :
1968 : // Find the config for this subinterface
1969 : subint =
1970 3553 : ethernet_sw_interface_get_config (vnm, sw_if_index, &placeholder_flags,
1971 : &placeholder_unsup);
1972 :
1973 3553 : if (subint == 0)
1974 : {
1975 : // unimplemented or not ethernet
1976 143 : goto done;
1977 : }
1978 :
1979 : // Double check that the config we found is for our interface (or the interface is down)
1980 3410 : ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1981 :
1982 3410 : if (l2)
1983 : {
1984 1257 : subint->flags |= SUBINT_CONFIG_L2;
1985 1257 : if (is_port)
1986 1203 : subint->flags |=
1987 : SUBINT_CONFIG_MATCH_0_TAG | SUBINT_CONFIG_MATCH_1_TAG
1988 : | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG;
1989 : }
1990 : else
1991 : {
1992 2153 : subint->flags &= ~SUBINT_CONFIG_L2;
1993 2153 : if (is_port)
1994 2117 : subint->flags &=
1995 : ~(SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG
1996 : | SUBINT_CONFIG_MATCH_3_TAG);
1997 : }
1998 :
1999 36 : done:
2000 3553 : return;
2001 : }
2002 :
2003 : /*
2004 : * Set the L2/L3 mode for the subinterface regardless of port
2005 : */
2006 : void
2007 0 : ethernet_sw_interface_set_l2_mode_noport (vnet_main_t * vnm,
2008 : u32 sw_if_index, u32 l2)
2009 : {
2010 : subint_config_t *subint;
2011 : u32 placeholder_flags;
2012 : u32 placeholder_unsup;
2013 :
2014 : /* Find the config for this subinterface */
2015 : subint =
2016 0 : ethernet_sw_interface_get_config (vnm, sw_if_index, &placeholder_flags,
2017 : &placeholder_unsup);
2018 :
2019 0 : if (subint == 0)
2020 : {
2021 : /* unimplemented or not ethernet */
2022 0 : goto done;
2023 : }
2024 :
2025 : /*
2026 : * Double check that the config we found is for our interface (or the
2027 : * interface is down)
2028 : */
2029 0 : ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
2030 :
2031 0 : if (l2)
2032 : {
2033 0 : subint->flags |= SUBINT_CONFIG_L2;
2034 : }
2035 : else
2036 : {
2037 0 : subint->flags &= ~SUBINT_CONFIG_L2;
2038 : }
2039 :
2040 0 : done:
2041 0 : return;
2042 : }
2043 : #endif
2044 :
2045 : static clib_error_t *
2046 11798 : ethernet_sw_interface_add_del (vnet_main_t * vnm,
2047 : u32 sw_if_index, u32 is_create)
2048 : {
2049 11798 : clib_error_t *error = 0;
2050 : subint_config_t *subint;
2051 : u32 match_flags;
2052 11798 : u32 unsupported = 0;
2053 :
2054 : // Find the config for this subinterface
2055 : subint =
2056 11798 : ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags,
2057 : &unsupported);
2058 :
2059 11798 : if (subint == 0)
2060 : {
2061 : // not implemented yet or not ethernet
2062 1737 : if (unsupported)
2063 : {
2064 : // this is the NYI case
2065 0 : error = clib_error_return (0, "not implemented yet");
2066 : }
2067 1737 : goto done;
2068 : }
2069 :
2070 10061 : if (!is_create)
2071 : {
2072 3713 : subint->flags = 0;
2073 3713 : return error;
2074 : }
2075 :
2076 : // Initialize the subint
2077 6348 : if (subint->flags & SUBINT_CONFIG_VALID)
2078 : {
2079 : // Error vlan already in use
2080 0 : error = clib_error_return (0, "vlan is already in use");
2081 : }
2082 : else
2083 : {
2084 : // Note that config is L3 by default
2085 6348 : subint->flags = SUBINT_CONFIG_VALID | match_flags;
2086 6348 : subint->sw_if_index = ~0; // because interfaces are initially down
2087 : }
2088 :
2089 8085 : done:
2090 8085 : return error;
2091 : }
2092 :
2093 3459 : VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ethernet_sw_interface_add_del);
2094 :
2095 : static char *ethernet_error_strings[] = {
2096 : #define ethernet_error(n,c,s) s,
2097 : #include "error.def"
2098 : #undef ethernet_error
2099 : };
2100 :
2101 : /* *INDENT-OFF* */
2102 183788 : VLIB_REGISTER_NODE (ethernet_input_node) = {
2103 : .name = "ethernet-input",
2104 : /* Takes a vector of packets. */
2105 : .vector_size = sizeof (u32),
2106 : .scalar_size = sizeof (ethernet_input_frame_t),
2107 : .n_errors = ETHERNET_N_ERROR,
2108 : .error_strings = ethernet_error_strings,
2109 : .n_next_nodes = ETHERNET_INPUT_N_NEXT,
2110 : .next_nodes = {
2111 : #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
2112 : foreach_ethernet_input_next
2113 : #undef _
2114 : },
2115 : .format_buffer = format_ethernet_header_with_length,
2116 : .format_trace = format_ethernet_input_trace,
2117 : .unformat_buffer = unformat_ethernet_header,
2118 : };
2119 :
2120 183788 : VLIB_REGISTER_NODE (ethernet_input_type_node) = {
2121 : .name = "ethernet-input-type",
2122 : /* Takes a vector of packets. */
2123 : .vector_size = sizeof (u32),
2124 : .n_next_nodes = ETHERNET_INPUT_N_NEXT,
2125 : .next_nodes = {
2126 : #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
2127 : foreach_ethernet_input_next
2128 : #undef _
2129 : },
2130 : };
2131 :
2132 183788 : VLIB_REGISTER_NODE (ethernet_input_not_l2_node) = {
2133 : .name = "ethernet-input-not-l2",
2134 : /* Takes a vector of packets. */
2135 : .vector_size = sizeof (u32),
2136 : .n_next_nodes = ETHERNET_INPUT_N_NEXT,
2137 : .next_nodes = {
2138 : #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
2139 : foreach_ethernet_input_next
2140 : #undef _
2141 : },
2142 : };
2143 : /* *INDENT-ON* */
2144 :
2145 : #ifndef CLIB_MARCH_VARIANT
2146 : void
2147 0 : ethernet_set_rx_redirect (vnet_main_t * vnm,
2148 : vnet_hw_interface_t * hi, u32 enable)
2149 : {
2150 : // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets
2151 : // don't go directly to ip4-input)
2152 0 : vnet_hw_interface_rx_redirect_to_node
2153 : (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0);
2154 0 : }
2155 :
2156 :
2157 : /*
2158 : * Initialization and registration for the next_by_ethernet structure
2159 : */
2160 :
2161 : clib_error_t *
2162 1150 : next_by_ethertype_init (next_by_ethertype_t * l3_next)
2163 : {
2164 1150 : l3_next->input_next_by_type = sparse_vec_new
2165 : ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]),
2166 : /* bits in index */ BITS (((ethernet_header_t *) 0)->type));
2167 :
2168 1150 : vec_validate (l3_next->sparse_index_by_input_next_index,
2169 : ETHERNET_INPUT_NEXT_DROP);
2170 1150 : vec_validate (l3_next->sparse_index_by_input_next_index,
2171 : ETHERNET_INPUT_NEXT_PUNT);
2172 1150 : l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] =
2173 : SPARSE_VEC_INVALID_INDEX;
2174 1150 : l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] =
2175 : SPARSE_VEC_INVALID_INDEX;
2176 :
2177 : /*
2178 : * Make sure we don't wipe out an ethernet registration by mistake
2179 : * Can happen if init function ordering constraints are missing.
2180 : */
2181 : if (CLIB_DEBUG > 0)
2182 : {
2183 1150 : ethernet_main_t *em = ðernet_main;
2184 1150 : ASSERT (em->next_by_ethertype_register_called == 0);
2185 : }
2186 :
2187 1150 : return 0;
2188 : }
2189 :
2190 : // Add an ethertype -> next index mapping to the structure
2191 : clib_error_t *
2192 13812 : next_by_ethertype_register (next_by_ethertype_t * l3_next,
2193 : u32 ethertype, u32 next_index)
2194 : {
2195 : u32 i;
2196 : u16 *n;
2197 13812 : ethernet_main_t *em = ðernet_main;
2198 :
2199 : if (CLIB_DEBUG > 0)
2200 : {
2201 13812 : ethernet_main_t *em = ðernet_main;
2202 13812 : em->next_by_ethertype_register_called = 1;
2203 : }
2204 :
2205 : /* Setup ethernet type -> next index sparse vector mapping. */
2206 13812 : n = sparse_vec_validate (l3_next->input_next_by_type, ethertype);
2207 13812 : n[0] = next_index;
2208 :
2209 : /* Rebuild next index -> sparse index inverse mapping when sparse vector
2210 : is updated. */
2211 13812 : vec_validate (l3_next->sparse_index_by_input_next_index, next_index);
2212 76020 : for (i = 1; i < vec_len (l3_next->input_next_by_type); i++)
2213 62208 : l3_next->
2214 62208 : sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i;
2215 :
2216 : // do not allow the cached next index's to be updated if L3
2217 : // redirect is enabled, as it will have overwritten them
2218 13812 : if (!em->redirect_l3)
2219 : {
2220 : // Cache common ethertypes directly
2221 13812 : if (ethertype == ETHERNET_TYPE_IP4)
2222 : {
2223 1725 : l3_next->input_next_ip4 = next_index;
2224 : }
2225 12087 : else if (ethertype == ETHERNET_TYPE_IP6)
2226 : {
2227 1725 : l3_next->input_next_ip6 = next_index;
2228 : }
2229 10362 : else if (ethertype == ETHERNET_TYPE_MPLS)
2230 : {
2231 1725 : l3_next->input_next_mpls = next_index;
2232 : }
2233 : }
2234 13812 : return 0;
2235 : }
2236 :
2237 : void
2238 8680 : ethernet_setup_node (vlib_main_t *vm, u32 node_index)
2239 : {
2240 8680 : vlib_node_t *n = vlib_get_node (vm, node_index);
2241 8680 : pg_node_t *pn = pg_get_node (node_index);
2242 :
2243 8680 : n->format_buffer = format_ethernet_header_with_length;
2244 8680 : n->unformat_buffer = unformat_ethernet_header;
2245 8680 : pn->unformat_edit = unformat_pg_ethernet_header;
2246 8680 : }
2247 :
2248 : void
2249 575 : ethernet_input_init (vlib_main_t * vm, ethernet_main_t * em)
2250 : {
2251 : __attribute__ ((unused)) vlan_table_t *invalid_vlan_table;
2252 : __attribute__ ((unused)) qinq_table_t *invalid_qinq_table;
2253 :
2254 575 : ethernet_setup_node (vm, ethernet_input_node.index);
2255 575 : ethernet_setup_node (vm, ethernet_input_type_node.index);
2256 575 : ethernet_setup_node (vm, ethernet_input_not_l2_node.index);
2257 :
2258 575 : next_by_ethertype_init (&em->l3_next);
2259 :
2260 : // Initialize pools and vector for vlan parsing
2261 575 : vec_validate (em->main_intfs, 10); // 10 main interfaces
2262 575 : pool_alloc (em->vlan_pool, 10);
2263 575 : pool_alloc (em->qinq_pool, 1);
2264 :
2265 : // The first vlan pool will always be reserved for an invalid table
2266 575 : pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0
2267 : // The first qinq pool will always be reserved for an invalid table
2268 575 : pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0
2269 575 : }
2270 :
2271 : void
2272 4604 : ethernet_register_input_type (vlib_main_t * vm,
2273 : ethernet_type_t type, u32 node_index)
2274 : {
2275 4604 : ethernet_main_t *em = ðernet_main;
2276 : ethernet_type_info_t *ti;
2277 : u32 i;
2278 :
2279 : {
2280 4604 : clib_error_t *error = vlib_call_init_function (vm, ethernet_init);
2281 4604 : if (error)
2282 0 : clib_error_report (error);
2283 : }
2284 :
2285 4604 : ti = ethernet_get_type_info (em, type);
2286 4604 : if (ti == 0)
2287 : {
2288 0 : clib_warning ("type_info NULL for type %d", type);
2289 0 : return;
2290 : }
2291 4604 : ti->node_index = node_index;
2292 9208 : ti->next_index = vlib_node_add_next (vm,
2293 4604 : ethernet_input_node.index, node_index);
2294 4604 : i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2295 4604 : ASSERT (i == ti->next_index);
2296 :
2297 4604 : i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2298 4604 : ASSERT (i == ti->next_index);
2299 :
2300 : // Add the L3 node for this ethertype to the next nodes structure
2301 4604 : next_by_ethertype_register (&em->l3_next, type, ti->next_index);
2302 :
2303 : // Call the registration functions for other nodes that want a mapping
2304 4604 : l2bvi_register_input_type (vm, type, node_index);
2305 : }
2306 :
2307 : void
2308 575 : ethernet_register_l2_input (vlib_main_t * vm, u32 node_index)
2309 : {
2310 575 : ethernet_main_t *em = ðernet_main;
2311 : u32 i;
2312 :
2313 575 : em->l2_next =
2314 575 : vlib_node_add_next (vm, ethernet_input_node.index, node_index);
2315 :
2316 : /*
2317 : * Even if we never use these arcs, we have to align the next indices...
2318 : */
2319 575 : i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2320 :
2321 575 : ASSERT (i == em->l2_next);
2322 :
2323 575 : i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2324 575 : ASSERT (i == em->l2_next);
2325 575 : }
2326 :
2327 : // Register a next node for L3 redirect, and enable L3 redirect
2328 : void
2329 0 : ethernet_register_l3_redirect (vlib_main_t * vm, u32 node_index)
2330 : {
2331 0 : ethernet_main_t *em = ðernet_main;
2332 : u32 i;
2333 :
2334 0 : em->redirect_l3 = 1;
2335 0 : em->redirect_l3_next = vlib_node_add_next (vm,
2336 0 : ethernet_input_node.index,
2337 : node_index);
2338 : /*
2339 : * Change the cached next nodes to the redirect node
2340 : */
2341 0 : em->l3_next.input_next_ip4 = em->redirect_l3_next;
2342 0 : em->l3_next.input_next_ip6 = em->redirect_l3_next;
2343 0 : em->l3_next.input_next_mpls = em->redirect_l3_next;
2344 :
2345 : /*
2346 : * Even if we never use these arcs, we have to align the next indices...
2347 : */
2348 0 : i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2349 :
2350 0 : ASSERT (i == em->redirect_l3_next);
2351 :
2352 0 : i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2353 :
2354 0 : ASSERT (i == em->redirect_l3_next);
2355 0 : }
2356 : #endif
2357 :
2358 : /*
2359 : * fd.io coding-style-patch-verification: ON
2360 : *
2361 : * Local Variables:
2362 : * eval: (c-set-style "gnu")
2363 : * End:
2364 : */
|
