Line data Source code
1 : /*
2 : * Copyright (c) 2018 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * ethernet_node.c: ethernet packet processing
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #include <vlib/vlib.h>
41 : #include <vnet/pg/pg.h>
42 : #include <vnet/ethernet/ethernet.h>
43 : #include <vnet/ethernet/p2p_ethernet.h>
44 : #include <vnet/devices/pipe/pipe.h>
45 : #include <vppinfra/sparse_vec.h>
46 : #include <vnet/l2/l2_bvi.h>
47 : #include <vnet/classify/pcap_classify.h>
48 :
49 : #define foreach_ethernet_input_next \
50 : _ (PUNT, "error-punt") \
51 : _ (DROP, "error-drop") \
52 : _ (LLC, "llc-input") \
53 : _ (IP4_INPUT, "ip4-input") \
54 : _ (IP4_INPUT_NCS, "ip4-input-no-checksum")
55 :
56 : typedef enum
57 : {
58 : #define _(s,n) ETHERNET_INPUT_NEXT_##s,
59 : foreach_ethernet_input_next
60 : #undef _
61 : ETHERNET_INPUT_N_NEXT,
62 : } ethernet_input_next_t;
63 :
64 : typedef struct
65 : {
66 : u8 packet_data[32];
67 : u16 frame_flags;
68 : ethernet_input_frame_t frame_data;
69 : } ethernet_input_trace_t;
70 :
71 : static u8 *
72 331018 : format_ethernet_input_trace (u8 * s, va_list * va)
73 : {
74 331018 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
75 331018 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
76 331018 : ethernet_input_trace_t *t = va_arg (*va, ethernet_input_trace_t *);
77 331018 : u32 indent = format_get_indent (s);
78 :
79 331018 : if (t->frame_flags)
80 : {
81 328960 : s = format (s, "frame: flags 0x%x", t->frame_flags);
82 328960 : if (t->frame_flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
83 328960 : s = format (s, ", hw-if-index %u, sw-if-index %u",
84 : t->frame_data.hw_if_index, t->frame_data.sw_if_index);
85 328960 : s = format (s, "\n%U", format_white_space, indent);
86 : }
87 331018 : s = format (s, "%U", format_ethernet_header, t->packet_data);
88 :
89 331018 : return s;
90 : }
91 :
92 : extern vlib_node_registration_t ethernet_input_node;
93 :
94 : typedef enum
95 : {
96 : ETHERNET_INPUT_VARIANT_ETHERNET,
97 : ETHERNET_INPUT_VARIANT_ETHERNET_TYPE,
98 : ETHERNET_INPUT_VARIANT_NOT_L2,
99 : } ethernet_input_variant_t;
100 :
101 :
102 : // Parse the ethernet header to extract vlan tags and innermost ethertype
103 : static_always_inline void
104 180792 : parse_header (ethernet_input_variant_t variant,
105 : vlib_buffer_t * b0,
106 : u16 * type,
107 : u16 * orig_type,
108 : u16 * outer_id, u16 * inner_id, u32 * match_flags)
109 : {
110 : u8 vlan_count;
111 :
112 180792 : if (variant == ETHERNET_INPUT_VARIANT_ETHERNET
113 0 : || variant == ETHERNET_INPUT_VARIANT_NOT_L2)
114 180792 : {
115 : ethernet_header_t *e0;
116 :
117 180792 : e0 = vlib_buffer_get_current (b0);
118 :
119 180792 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
120 180792 : b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
121 :
122 180792 : vlib_buffer_advance (b0, sizeof (e0[0]));
123 :
124 180792 : *type = clib_net_to_host_u16 (e0->type);
125 : }
126 0 : else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE)
127 : {
128 : // here when prior node was LLC/SNAP processing
129 : u16 *e0;
130 :
131 0 : e0 = vlib_buffer_get_current (b0);
132 :
133 0 : vlib_buffer_advance (b0, sizeof (e0[0]));
134 :
135 0 : *type = clib_net_to_host_u16 (e0[0]);
136 : }
137 :
138 : // save for distinguishing between dot1q and dot1ad later
139 180792 : *orig_type = *type;
140 :
141 : // default the tags to 0 (used if there is no corresponding tag)
142 180792 : *outer_id = 0;
143 180792 : *inner_id = 0;
144 :
145 180792 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_0_TAG;
146 180792 : vlan_count = 0;
147 :
148 : // check for vlan encaps
149 180792 : if (ethernet_frame_is_tagged (*type))
150 : {
151 : ethernet_vlan_header_t *h0;
152 : u16 tag;
153 :
154 514 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_1_TAG;
155 :
156 514 : h0 = vlib_buffer_get_current (b0);
157 :
158 514 : tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
159 :
160 514 : *outer_id = tag & 0xfff;
161 514 : if (0 == *outer_id)
162 0 : *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG;
163 :
164 514 : *type = clib_net_to_host_u16 (h0->type);
165 :
166 514 : vlib_buffer_advance (b0, sizeof (h0[0]));
167 514 : vlan_count = 1;
168 :
169 514 : if (*type == ETHERNET_TYPE_VLAN)
170 : {
171 : // Double tagged packet
172 0 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_2_TAG;
173 :
174 0 : h0 = vlib_buffer_get_current (b0);
175 :
176 0 : tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
177 :
178 0 : *inner_id = tag & 0xfff;
179 :
180 0 : *type = clib_net_to_host_u16 (h0->type);
181 :
182 0 : vlib_buffer_advance (b0, sizeof (h0[0]));
183 0 : vlan_count = 2;
184 0 : if (*type == ETHERNET_TYPE_VLAN)
185 : {
186 : // More than double tagged packet
187 0 : *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_3_TAG;
188 :
189 0 : vlib_buffer_advance (b0, sizeof (h0[0]));
190 0 : vlan_count = 3; // "unknown" number, aka, 3-or-more
191 : }
192 : }
193 : }
194 180792 : ethernet_buffer_set_vlan_count (b0, vlan_count);
195 180792 : }
196 :
197 : static_always_inline void
198 : ethernet_input_inline_dmac_check (vnet_hw_interface_t * hi,
199 : u64 * dmacs, u8 * dmacs_bad,
200 : u32 n_packets, ethernet_interface_t * ei,
201 : u8 have_sec_dmac);
202 :
203 : // Determine the subinterface for this packet, given the result of the
204 : // vlan table lookups and vlan header parsing. Check the most specific
205 : // matches first.
206 : static_always_inline void
207 180792 : identify_subint (ethernet_main_t * em,
208 : vnet_hw_interface_t * hi,
209 : vlib_buffer_t * b0,
210 : u32 match_flags,
211 : main_intf_t * main_intf,
212 : vlan_intf_t * vlan_intf,
213 : qinq_intf_t * qinq_intf,
214 : u32 * new_sw_if_index, u8 * error0, u32 * is_l2)
215 : {
216 : u32 matched;
217 180792 : ethernet_interface_t *ei = ethernet_get_interface (em, hi->hw_if_index);
218 :
219 180792 : matched = eth_identify_subint (hi, match_flags, main_intf, vlan_intf,
220 : qinq_intf, new_sw_if_index, error0, is_l2);
221 :
222 180792 : if (matched)
223 : {
224 : // Perform L3 my-mac filter
225 : // A unicast packet arriving on an L3 interface must have a dmac
226 : // matching the interface mac. If interface has STATUS_L3 bit set
227 : // mac filter is already done.
228 180792 : if ((!*is_l2) && ei &&
229 27136 : (!(ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3)))
230 : {
231 : u64 dmacs[2];
232 : u8 dmacs_bad[2];
233 : ethernet_header_t *e0;
234 :
235 27136 : e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset);
236 27136 : dmacs[0] = *(u64 *) e0;
237 :
238 27136 : if (vec_len (ei->secondary_addrs))
239 27136 : ethernet_input_inline_dmac_check (hi, dmacs, dmacs_bad,
240 : 1 /* n_packets */, ei,
241 : 1 /* have_sec_dmac */);
242 : else
243 0 : ethernet_input_inline_dmac_check (hi, dmacs, dmacs_bad,
244 : 1 /* n_packets */, ei,
245 : 0 /* have_sec_dmac */);
246 27136 : if (dmacs_bad[0])
247 0 : *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
248 : }
249 :
250 : // Check for down subinterface
251 180792 : *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN;
252 : }
253 180792 : }
254 :
255 : static_always_inline void
256 4684800 : determine_next_node (ethernet_main_t * em,
257 : ethernet_input_variant_t variant,
258 : u32 is_l20,
259 : u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0)
260 : {
261 4684800 : vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
262 4684800 : b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
263 :
264 4684800 : if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE))
265 : {
266 : // some error occurred
267 2 : *next0 = ETHERNET_INPUT_NEXT_DROP;
268 : }
269 4684800 : else if (is_l20)
270 : {
271 : // record the L2 len and reset the buffer so the L2 header is preserved
272 153656 : u32 eth_start = vnet_buffer (b0)->l2_hdr_offset;
273 153656 : vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start;
274 153656 : *next0 = em->l2_next;
275 153656 : ASSERT (vnet_buffer (b0)->l2.l2_len ==
276 : ethernet_buffer_header_size (b0));
277 153656 : vlib_buffer_advance (b0, -(vnet_buffer (b0)->l2.l2_len));
278 :
279 : // check for common IP/MPLS ethertypes
280 : }
281 4531140 : else if (type0 == ETHERNET_TYPE_IP4)
282 : {
283 2614800 : *next0 = em->l3_next.input_next_ip4;
284 : }
285 1916340 : else if (type0 == ETHERNET_TYPE_IP6)
286 : {
287 1916210 : *next0 = em->l3_next.input_next_ip6;
288 : }
289 124 : else if (type0 == ETHERNET_TYPE_MPLS)
290 : {
291 0 : *next0 = em->l3_next.input_next_mpls;
292 :
293 : }
294 124 : else if (em->redirect_l3)
295 : {
296 : // L3 Redirect is on, the cached common next nodes will be
297 : // pointing to the redirect node, catch the uncommon types here
298 0 : *next0 = em->redirect_l3_next;
299 : }
300 : else
301 : {
302 : // uncommon ethertype, check table
303 : u32 i0;
304 124 : i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0);
305 124 : *next0 = vec_elt (em->l3_next.input_next_by_type, i0);
306 124 : *error0 =
307 : i0 ==
308 : SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0;
309 :
310 : // The table is not populated with LLC values, so check that now.
311 : // If variant is variant_ethernet then we came from LLC processing. Don't
312 : // go back there; drop instead using by keeping the drop/bad table result.
313 124 : if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET))
314 : {
315 0 : *next0 = ETHERNET_INPUT_NEXT_LLC;
316 : }
317 : }
318 4684800 : }
319 :
320 :
321 : /* following vector code relies on following assumptions */
322 : STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_data, 0);
323 : STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_length, 2);
324 : STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, flags, 4);
325 : STATIC_ASSERT (STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l2_hdr_offset) ==
326 : STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l3_hdr_offset) - 2,
327 : "l3_hdr_offset must follow l2_hdr_offset");
328 :
329 : static_always_inline void
330 11989700 : eth_input_adv_and_flags_x4 (vlib_buffer_t ** b, int is_l3)
331 : {
332 11989700 : i16 adv = sizeof (ethernet_header_t);
333 11989700 : u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
334 : VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
335 :
336 : #ifdef CLIB_HAVE_VEC256
337 : /* to reduce number of small loads/stores we are loading first 64 bits
338 : of each buffer metadata into 256-bit register so we can advance
339 : current_data, current_length and flags.
340 : Observed saving of this code is ~2 clocks per packet */
341 : u64x4 r, radv;
342 :
343 : /* vector if signed 16 bit integers used in signed vector add operation
344 : to advnce current_data and current_length */
345 11989700 : u32x8 flags4 = { 0, flags, 0, flags, 0, flags, 0, flags };
346 11989700 : i16x16 adv4 = {
347 : adv, -adv, 0, 0, adv, -adv, 0, 0,
348 : adv, -adv, 0, 0, adv, -adv, 0, 0
349 : };
350 :
351 : /* load 4 x 64 bits */
352 11989700 : r = u64x4_gather (b[0], b[1], b[2], b[3]);
353 :
354 : /* set flags */
355 11989700 : r |= (u64x4) flags4;
356 :
357 : /* advance buffer */
358 11989700 : radv = (u64x4) ((i16x16) r + adv4);
359 :
360 : /* write 4 x 64 bits */
361 11989700 : u64x4_scatter (is_l3 ? radv : r, b[0], b[1], b[2], b[3]);
362 :
363 : /* use old current_data as l2_hdr_offset and new current_data as
364 : l3_hdr_offset */
365 11989700 : r = (u64x4) u16x16_blend (r, radv << 16, 0xaa);
366 :
367 : /* store both l2_hdr_offset and l3_hdr_offset in single store operation */
368 11989700 : u32x8_scatter_one ((u32x8) r, 0, &vnet_buffer (b[0])->l2_hdr_offset);
369 11989700 : u32x8_scatter_one ((u32x8) r, 2, &vnet_buffer (b[1])->l2_hdr_offset);
370 11989700 : u32x8_scatter_one ((u32x8) r, 4, &vnet_buffer (b[2])->l2_hdr_offset);
371 11989700 : u32x8_scatter_one ((u32x8) r, 6, &vnet_buffer (b[3])->l2_hdr_offset);
372 :
373 11989700 : if (is_l3)
374 : {
375 1374660 : ASSERT (b[0]->current_data == vnet_buffer (b[0])->l3_hdr_offset);
376 1374660 : ASSERT (b[1]->current_data == vnet_buffer (b[1])->l3_hdr_offset);
377 1374660 : ASSERT (b[2]->current_data == vnet_buffer (b[2])->l3_hdr_offset);
378 1374660 : ASSERT (b[3]->current_data == vnet_buffer (b[3])->l3_hdr_offset);
379 :
380 1374660 : ASSERT (b[0]->current_data - vnet_buffer (b[0])->l2_hdr_offset == adv);
381 1374660 : ASSERT (b[1]->current_data - vnet_buffer (b[1])->l2_hdr_offset == adv);
382 1374660 : ASSERT (b[2]->current_data - vnet_buffer (b[2])->l2_hdr_offset == adv);
383 1374660 : ASSERT (b[3]->current_data - vnet_buffer (b[3])->l2_hdr_offset == adv);
384 : }
385 : else
386 : {
387 10615000 : ASSERT (b[0]->current_data == vnet_buffer (b[0])->l2_hdr_offset);
388 10615000 : ASSERT (b[1]->current_data == vnet_buffer (b[1])->l2_hdr_offset);
389 10615000 : ASSERT (b[2]->current_data == vnet_buffer (b[2])->l2_hdr_offset);
390 10615000 : ASSERT (b[3]->current_data == vnet_buffer (b[3])->l2_hdr_offset);
391 :
392 10615000 : ASSERT (b[0]->current_data - vnet_buffer (b[0])->l3_hdr_offset == -adv);
393 10615000 : ASSERT (b[1]->current_data - vnet_buffer (b[1])->l3_hdr_offset == -adv);
394 10615000 : ASSERT (b[2]->current_data - vnet_buffer (b[2])->l3_hdr_offset == -adv);
395 10615000 : ASSERT (b[3]->current_data - vnet_buffer (b[3])->l3_hdr_offset == -adv);
396 : }
397 :
398 : #else
399 0 : vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
400 0 : vnet_buffer (b[1])->l2_hdr_offset = b[1]->current_data;
401 0 : vnet_buffer (b[2])->l2_hdr_offset = b[2]->current_data;
402 0 : vnet_buffer (b[3])->l2_hdr_offset = b[3]->current_data;
403 0 : vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
404 0 : vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data + adv;
405 0 : vnet_buffer (b[2])->l3_hdr_offset = b[2]->current_data + adv;
406 0 : vnet_buffer (b[3])->l3_hdr_offset = b[3]->current_data + adv;
407 :
408 0 : if (is_l3)
409 : {
410 0 : vlib_buffer_advance (b[0], adv);
411 0 : vlib_buffer_advance (b[1], adv);
412 0 : vlib_buffer_advance (b[2], adv);
413 0 : vlib_buffer_advance (b[3], adv);
414 : }
415 :
416 0 : b[0]->flags |= flags;
417 0 : b[1]->flags |= flags;
418 0 : b[2]->flags |= flags;
419 0 : b[3]->flags |= flags;
420 : #endif
421 :
422 11989700 : if (!is_l3)
423 : {
424 10615000 : vnet_buffer (b[0])->l2.l2_len = adv;
425 10615000 : vnet_buffer (b[1])->l2.l2_len = adv;
426 10615000 : vnet_buffer (b[2])->l2.l2_len = adv;
427 10615000 : vnet_buffer (b[3])->l2.l2_len = adv;
428 : }
429 11989700 : }
430 :
431 : static_always_inline void
432 1387260 : eth_input_adv_and_flags_x1 (vlib_buffer_t ** b, int is_l3)
433 : {
434 1387260 : i16 adv = sizeof (ethernet_header_t);
435 1387260 : u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
436 : VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
437 :
438 1387260 : vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
439 1387260 : vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
440 :
441 1387260 : if (is_l3)
442 139820 : vlib_buffer_advance (b[0], adv);
443 1387260 : b[0]->flags |= flags;
444 1387260 : if (!is_l3)
445 1247440 : vnet_buffer (b[0])->l2.l2_len = adv;
446 1387260 : }
447 :
448 :
449 : static_always_inline void
450 49346000 : eth_input_get_etype_and_tags (vlib_buffer_t ** b, u16 * etype, u64 * tags,
451 : u64 * dmacs, int offset, int dmac_check)
452 : {
453 : ethernet_header_t *e;
454 49346000 : e = vlib_buffer_get_current (b[offset]);
455 : #ifdef CLIB_HAVE_VEC128
456 49346000 : u64x2 r = u64x2_load_unaligned (((u8 *) & e->type) - 6);
457 49346000 : etype[offset] = ((u16x8) r)[3];
458 49346000 : tags[offset] = r[1];
459 : #else
460 : etype[offset] = e->type;
461 : tags[offset] = *(u64 *) (e + 1);
462 : #endif
463 :
464 49346000 : if (dmac_check)
465 5638620 : dmacs[offset] = *(u64 *) e;
466 49346000 : }
467 :
468 : static_always_inline u16
469 2034 : eth_input_next_by_type (u16 etype)
470 : {
471 2034 : ethernet_main_t *em = ðernet_main;
472 :
473 4060 : return (etype < 0x600) ? ETHERNET_INPUT_NEXT_LLC :
474 2026 : vec_elt (em->l3_next.input_next_by_type,
475 : sparse_vec_index (em->l3_next.input_next_by_type, etype));
476 : }
477 :
478 : typedef struct
479 : {
480 : u64 tag, mask;
481 : u32 sw_if_index;
482 : u16 type, len, next;
483 : i16 adv;
484 : u8 err, n_tags;
485 : u64 n_packets, n_bytes;
486 : } eth_input_tag_lookup_t;
487 :
488 : static_always_inline void
489 4623 : eth_input_update_if_counters (vlib_main_t * vm, vnet_main_t * vnm,
490 : eth_input_tag_lookup_t * l)
491 : {
492 4623 : if (l->n_packets == 0 || l->sw_if_index == ~0)
493 4376 : return;
494 :
495 247 : if (l->adv > 0)
496 158 : l->n_bytes += l->n_packets * l->len;
497 :
498 247 : vlib_increment_combined_counter
499 : (vnm->interface_main.combined_sw_if_counters +
500 : VNET_INTERFACE_COUNTER_RX, vm->thread_index, l->sw_if_index,
501 : l->n_packets, l->n_bytes);
502 : }
503 :
504 : static_always_inline void
505 4990 : eth_input_tag_lookup (vlib_main_t * vm, vnet_main_t * vnm,
506 : vlib_node_runtime_t * node, vnet_hw_interface_t * hi,
507 : u64 tag, u16 * next, vlib_buffer_t * b,
508 : eth_input_tag_lookup_t * l, u8 dmac_bad, int is_dot1ad,
509 : int main_is_l3, int check_dmac)
510 : {
511 4990 : ethernet_main_t *em = ðernet_main;
512 :
513 4990 : if ((tag ^ l->tag) & l->mask)
514 : {
515 251 : main_intf_t *mif = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
516 : vlan_intf_t *vif;
517 : qinq_intf_t *qif;
518 : vlan_table_t *vlan_table;
519 : qinq_table_t *qinq_table;
520 251 : u16 *t = (u16 *) & tag;
521 251 : u16 vlan1 = clib_net_to_host_u16 (t[0]) & 0xFFF;
522 251 : u16 vlan2 = clib_net_to_host_u16 (t[2]) & 0xFFF;
523 : u32 matched, is_l2, new_sw_if_index;
524 :
525 251 : vlan_table = vec_elt_at_index (em->vlan_pool, is_dot1ad ?
526 : mif->dot1ad_vlans : mif->dot1q_vlans);
527 251 : vif = &vlan_table->vlans[vlan1];
528 251 : qinq_table = vec_elt_at_index (em->qinq_pool, vif->qinqs);
529 251 : qif = &qinq_table->vlans[vlan2];
530 251 : l->err = ETHERNET_ERROR_NONE;
531 251 : l->type = clib_net_to_host_u16 (t[1]);
532 :
533 251 : if (l->type == ETHERNET_TYPE_VLAN)
534 : {
535 53 : l->type = clib_net_to_host_u16 (t[3]);
536 53 : l->n_tags = 2;
537 53 : matched = eth_identify_subint (hi, SUBINT_CONFIG_VALID |
538 : SUBINT_CONFIG_MATCH_2_TAG, mif, vif,
539 : qif, &new_sw_if_index, &l->err,
540 : &is_l2);
541 : }
542 : else
543 : {
544 198 : l->n_tags = 1;
545 198 : if (vlan1 == 0)
546 : {
547 4 : new_sw_if_index = hi->sw_if_index;
548 4 : l->err = ETHERNET_ERROR_NONE;
549 4 : matched = 1;
550 4 : is_l2 = main_is_l3 == 0;
551 : }
552 : else
553 194 : matched = eth_identify_subint (hi, SUBINT_CONFIG_VALID |
554 : SUBINT_CONFIG_MATCH_1_TAG, mif,
555 : vif, qif, &new_sw_if_index,
556 : &l->err, &is_l2);
557 : }
558 :
559 251 : if (l->sw_if_index != new_sw_if_index)
560 : {
561 247 : eth_input_update_if_counters (vm, vnm, l);
562 247 : l->n_packets = 0;
563 247 : l->n_bytes = 0;
564 247 : l->sw_if_index = new_sw_if_index;
565 : }
566 251 : l->tag = tag;
567 502 : l->mask = (l->n_tags == 2) ?
568 251 : clib_net_to_host_u64 (0xffffffffffffffff) :
569 198 : clib_net_to_host_u64 (0xffffffff00000000);
570 :
571 251 : if (matched && l->sw_if_index == ~0)
572 0 : l->err = ETHERNET_ERROR_DOWN;
573 :
574 251 : l->len = sizeof (ethernet_header_t) +
575 251 : l->n_tags * sizeof (ethernet_vlan_header_t);
576 251 : if (main_is_l3)
577 387 : l->adv = is_l2 ? -(int) sizeof (ethernet_header_t) :
578 162 : l->n_tags * sizeof (ethernet_vlan_header_t);
579 : else
580 26 : l->adv = is_l2 ? 0 : l->len;
581 :
582 251 : if (PREDICT_FALSE (l->err != ETHERNET_ERROR_NONE))
583 4 : l->next = ETHERNET_INPUT_NEXT_DROP;
584 247 : else if (is_l2)
585 89 : l->next = em->l2_next;
586 158 : else if (l->type == ETHERNET_TYPE_IP4)
587 57 : l->next = em->l3_next.input_next_ip4;
588 101 : else if (l->type == ETHERNET_TYPE_IP6)
589 73 : l->next = em->l3_next.input_next_ip6;
590 28 : else if (l->type == ETHERNET_TYPE_MPLS)
591 0 : l->next = em->l3_next.input_next_mpls;
592 28 : else if (em->redirect_l3)
593 0 : l->next = em->redirect_l3_next;
594 : else
595 : {
596 28 : l->next = eth_input_next_by_type (l->type);
597 28 : if (l->next == ETHERNET_INPUT_NEXT_PUNT)
598 0 : l->err = ETHERNET_ERROR_UNKNOWN_TYPE;
599 : }
600 : }
601 :
602 4990 : if (check_dmac && l->adv > 0 && dmac_bad)
603 : {
604 0 : l->err = ETHERNET_ERROR_L3_MAC_MISMATCH;
605 0 : next[0] = ETHERNET_INPUT_NEXT_PUNT;
606 : }
607 : else
608 4990 : next[0] = l->next;
609 :
610 4990 : vlib_buffer_advance (b, l->adv);
611 4990 : vnet_buffer (b)->l2.l2_len = l->len;
612 4990 : vnet_buffer (b)->l3_hdr_offset = vnet_buffer (b)->l2_hdr_offset + l->len;
613 :
614 4990 : if (l->err == ETHERNET_ERROR_NONE)
615 : {
616 4954 : vnet_buffer (b)->sw_if_index[VLIB_RX] = l->sw_if_index;
617 4954 : ethernet_buffer_set_vlan_count (b, l->n_tags);
618 : }
619 : else
620 36 : b->error = node->errors[l->err];
621 :
622 : /* update counters */
623 4990 : l->n_packets += 1;
624 4990 : l->n_bytes += vlib_buffer_length_in_chain (vm, b);
625 4990 : }
626 :
627 : #define DMAC_MASK clib_net_to_host_u64 (0xFFFFFFFFFFFF0000)
628 : #define DMAC_IGBIT clib_net_to_host_u64 (0x0100000000000000)
629 :
630 : #ifdef CLIB_HAVE_VEC256
631 : static_always_inline u32
632 1510560 : is_dmac_bad_x4 (u64 * dmacs, u64 hwaddr)
633 : {
634 1510560 : u64x4 r0 = u64x4_load_unaligned (dmacs) & u64x4_splat (DMAC_MASK);
635 1510560 : r0 = (r0 != u64x4_splat (hwaddr)) & ((r0 & u64x4_splat (DMAC_IGBIT)) == 0);
636 1510560 : return u8x32_msb_mask ((u8x32) (r0));
637 : }
638 : #endif
639 :
640 : static_always_inline u8
641 5001660 : is_dmac_bad (u64 dmac, u64 hwaddr)
642 : {
643 5001660 : u64 r0 = dmac & DMAC_MASK;
644 5001660 : return (r0 != hwaddr) && ((r0 & DMAC_IGBIT) == 0);
645 : }
646 :
647 : static_always_inline u8
648 714 : is_sec_dmac_bad (u64 dmac, u64 hwaddr)
649 : {
650 714 : return ((dmac & DMAC_MASK) != hwaddr);
651 : }
652 :
653 : #ifdef CLIB_HAVE_VEC256
654 : static_always_inline u32
655 5152 : is_sec_dmac_bad_x4 (u64 * dmacs, u64 hwaddr)
656 : {
657 5152 : u64x4 r0 = u64x4_load_unaligned (dmacs) & u64x4_splat (DMAC_MASK);
658 5152 : r0 = (r0 != u64x4_splat (hwaddr));
659 5152 : return u8x32_msb_mask ((u8x32) (r0));
660 : }
661 : #endif
662 :
663 : static_always_inline u8
664 714 : eth_input_sec_dmac_check_x1 (u64 hwaddr, u64 * dmac, u8 * dmac_bad)
665 : {
666 714 : dmac_bad[0] &= is_sec_dmac_bad (dmac[0], hwaddr);
667 714 : return dmac_bad[0];
668 : }
669 :
670 : static_always_inline u32
671 5152 : eth_input_sec_dmac_check_x4 (u64 hwaddr, u64 * dmac, u8 * dmac_bad)
672 : {
673 : #ifdef CLIB_HAVE_VEC256
674 5152 : *(u32 *) (dmac_bad + 0) &= is_sec_dmac_bad_x4 (dmac + 0, hwaddr);
675 : #else
676 0 : dmac_bad[0] &= is_sec_dmac_bad (dmac[0], hwaddr);
677 0 : dmac_bad[1] &= is_sec_dmac_bad (dmac[1], hwaddr);
678 0 : dmac_bad[2] &= is_sec_dmac_bad (dmac[2], hwaddr);
679 0 : dmac_bad[3] &= is_sec_dmac_bad (dmac[3], hwaddr);
680 : #endif
681 5152 : return *(u32 *) dmac_bad;
682 : }
683 :
684 : /*
685 : * DMAC check for ethernet_input_inline()
686 : *
687 : * dmacs and dmacs_bad are arrays that are 2 elements long
688 : * n_packets should be 1 or 2 for ethernet_input_inline()
689 : */
690 : static_always_inline void
691 2500830 : ethernet_input_inline_dmac_check (vnet_hw_interface_t * hi,
692 : u64 * dmacs, u8 * dmacs_bad,
693 : u32 n_packets, ethernet_interface_t * ei,
694 : u8 have_sec_dmac)
695 : {
696 2500830 : u64 hwaddr = ei->address.as_u64;
697 2500830 : u8 bad = 0;
698 :
699 2500830 : ASSERT (0 == ei->address.zero);
700 :
701 2500830 : dmacs_bad[0] = is_dmac_bad (dmacs[0], hwaddr);
702 2500830 : dmacs_bad[1] = ((n_packets > 1) & is_dmac_bad (dmacs[1], hwaddr));
703 :
704 2500830 : bad = dmacs_bad[0] | dmacs_bad[1];
705 :
706 2500830 : if (PREDICT_FALSE (bad && have_sec_dmac))
707 : {
708 : ethernet_interface_address_t *sec_addr;
709 :
710 3 : vec_foreach (sec_addr, ei->secondary_addrs)
711 : {
712 2 : ASSERT (0 == sec_addr->zero);
713 2 : hwaddr = sec_addr->as_u64;
714 :
715 2 : bad = (eth_input_sec_dmac_check_x1 (hwaddr, dmacs, dmacs_bad) |
716 2 : eth_input_sec_dmac_check_x1 (hwaddr, dmacs + 1,
717 : dmacs_bad + 1));
718 :
719 2 : if (!bad)
720 0 : return;
721 : }
722 : }
723 : }
724 :
725 : static_always_inline void
726 111605 : eth_input_process_frame_dmac_check (vnet_hw_interface_t * hi,
727 : u64 * dmacs, u8 * dmacs_bad,
728 : u32 n_packets, ethernet_interface_t * ei,
729 : u8 have_sec_dmac)
730 : {
731 111605 : u64 hwaddr = ei->address.as_u64;
732 111605 : u64 *dmac = dmacs;
733 111605 : u8 *dmac_bad = dmacs_bad;
734 111605 : u32 bad = 0;
735 111605 : i32 n_left = n_packets;
736 :
737 111605 : ASSERT (0 == ei->address.zero);
738 :
739 : #ifdef CLIB_HAVE_VEC256
740 866896 : while (n_left > 0)
741 : {
742 755292 : bad |= *(u32 *) (dmac_bad + 0) = is_dmac_bad_x4 (dmac + 0, hwaddr);
743 755289 : bad |= *(u32 *) (dmac_bad + 4) = is_dmac_bad_x4 (dmac + 4, hwaddr);
744 :
745 : /* next */
746 755291 : dmac += 8;
747 755291 : dmac_bad += 8;
748 755291 : n_left -= 8;
749 : }
750 : #else
751 0 : while (n_left > 0)
752 : {
753 0 : bad |= dmac_bad[0] = is_dmac_bad (dmac[0], hwaddr);
754 0 : bad |= dmac_bad[1] = is_dmac_bad (dmac[1], hwaddr);
755 0 : bad |= dmac_bad[2] = is_dmac_bad (dmac[2], hwaddr);
756 0 : bad |= dmac_bad[3] = is_dmac_bad (dmac[3], hwaddr);
757 :
758 : /* next */
759 0 : dmac += 4;
760 0 : dmac_bad += 4;
761 0 : n_left -= 4;
762 : }
763 : #endif
764 :
765 111604 : if (have_sec_dmac && bad)
766 : {
767 : ethernet_interface_address_t *addr;
768 :
769 74542 : vec_foreach (addr, ei->secondary_addrs)
770 : {
771 74425 : u64 hwaddr = addr->as_u64;
772 74425 : i32 n_left = n_packets;
773 74425 : u64 *dmac = dmacs;
774 74425 : u8 *dmac_bad = dmacs_bad;
775 :
776 74425 : ASSERT (0 == addr->zero);
777 :
778 74425 : bad = 0;
779 :
780 1543740 : while (n_left > 0)
781 : {
782 1469310 : int adv = 0;
783 : int n_bad;
784 :
785 : /* skip any that have already matched */
786 1469310 : if (!dmac_bad[0])
787 : {
788 1463680 : dmac += 1;
789 1463680 : dmac_bad += 1;
790 1463680 : n_left -= 1;
791 1463680 : continue;
792 : }
793 :
794 5628 : n_bad = clib_min (4, n_left);
795 :
796 : /* If >= 4 left, compare 4 together */
797 5628 : if (n_bad == 4)
798 : {
799 5152 : bad |= eth_input_sec_dmac_check_x4 (hwaddr, dmac, dmac_bad);
800 5152 : adv = 4;
801 5152 : n_bad = 0;
802 : }
803 :
804 : /* handle individually */
805 6338 : while (n_bad > 0)
806 : {
807 710 : bad |= eth_input_sec_dmac_check_x1 (hwaddr, dmac + adv,
808 : dmac_bad + adv);
809 710 : adv += 1;
810 710 : n_bad -= 1;
811 : }
812 :
813 5628 : dmac += adv;
814 5628 : dmac_bad += adv;
815 5628 : n_left -= adv;
816 : }
817 :
818 74425 : if (!bad) /* can stop looping if everything matched */
819 73893 : break;
820 : }
821 : }
822 111604 : }
823 :
824 : /* process frame of buffers, store ethertype into array and update
825 : buffer metadata fields depending on interface being l2 or l3 assuming that
826 : packets are untagged. For tagged packets those fields are updated later.
827 : Optionally store Destionation MAC address and tag data into arrays
828 : for further processing */
829 :
830 : STATIC_ASSERT (VLIB_FRAME_SIZE % 8 == 0,
831 : "VLIB_FRAME_SIZE must be power of 8");
832 : static_always_inline void
833 974110 : eth_input_process_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
834 : vnet_hw_interface_t * hi,
835 : u32 * buffer_indices, u32 n_packets, int main_is_l3,
836 : int ip4_cksum_ok, int dmac_check)
837 : {
838 974110 : ethernet_main_t *em = ðernet_main;
839 : u16 nexts[VLIB_FRAME_SIZE], *next;
840 974110 : u16 etypes[VLIB_FRAME_SIZE], *etype = etypes;
841 974110 : u64 dmacs[VLIB_FRAME_SIZE], *dmac = dmacs;
842 : u8 dmacs_bad[VLIB_FRAME_SIZE];
843 974110 : u64 tags[VLIB_FRAME_SIZE], *tag = tags;
844 : u16 slowpath_indices[VLIB_FRAME_SIZE];
845 : u16 n_slowpath, i;
846 : u16 next_ip4, next_ip6, next_mpls, next_l2;
847 974110 : u16 et_ip4 = clib_host_to_net_u16 (ETHERNET_TYPE_IP4);
848 974110 : u16 et_ip6 = clib_host_to_net_u16 (ETHERNET_TYPE_IP6);
849 974110 : u16 et_mpls = clib_host_to_net_u16 (ETHERNET_TYPE_MPLS);
850 974110 : u16 et_vlan = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN);
851 974110 : u16 et_dot1ad = clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD);
852 974110 : i32 n_left = n_packets;
853 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
854 974110 : vlib_buffer_t **b = bufs;
855 974110 : ethernet_interface_t *ei = ethernet_get_interface (em, hi->hw_if_index);
856 :
857 974110 : vlib_get_buffers (vm, buffer_indices, b, n_left);
858 :
859 9907680 : while (n_left >= 20)
860 : {
861 8933570 : vlib_buffer_t **ph = b + 16, **pd = b + 8;
862 :
863 8933570 : vlib_prefetch_buffer_header (ph[0], LOAD);
864 8933570 : vlib_prefetch_buffer_data (pd[0], LOAD);
865 8933570 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
866 :
867 8933570 : vlib_prefetch_buffer_header (ph[1], LOAD);
868 8933570 : vlib_prefetch_buffer_data (pd[1], LOAD);
869 8933570 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
870 :
871 8933570 : vlib_prefetch_buffer_header (ph[2], LOAD);
872 8933570 : vlib_prefetch_buffer_data (pd[2], LOAD);
873 8933570 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
874 :
875 8933570 : vlib_prefetch_buffer_header (ph[3], LOAD);
876 8933570 : vlib_prefetch_buffer_data (pd[3], LOAD);
877 8933560 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
878 :
879 8933560 : eth_input_adv_and_flags_x4 (b, main_is_l3);
880 :
881 : /* next */
882 8933570 : b += 4;
883 8933570 : n_left -= 4;
884 8933570 : etype += 4;
885 8933570 : tag += 4;
886 8933570 : dmac += 4;
887 : }
888 4030240 : while (n_left >= 4)
889 : {
890 3056130 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
891 3056130 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
892 3056130 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
893 3056130 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
894 3056130 : eth_input_adv_and_flags_x4 (b, main_is_l3);
895 :
896 : /* next */
897 3056130 : b += 4;
898 3056130 : n_left -= 4;
899 3056130 : etype += 4;
900 3056130 : tag += 4;
901 3056130 : dmac += 4;
902 : }
903 2361370 : while (n_left)
904 : {
905 1387260 : eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
906 1387260 : eth_input_adv_and_flags_x1 (b, main_is_l3);
907 :
908 : /* next */
909 1387260 : b += 1;
910 1387260 : n_left -= 1;
911 1387260 : etype += 1;
912 1387260 : tag += 1;
913 1387260 : dmac += 1;
914 : }
915 :
916 974110 : if (dmac_check)
917 : {
918 111605 : if (ei && vec_len (ei->secondary_addrs))
919 111147 : eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets,
920 : ei, 1 /* have_sec_dmac */ );
921 : else
922 458 : eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets,
923 : ei, 0 /* have_sec_dmac */ );
924 : }
925 :
926 974110 : next_ip4 = em->l3_next.input_next_ip4;
927 974110 : next_ip6 = em->l3_next.input_next_ip6;
928 974110 : next_mpls = em->l3_next.input_next_mpls;
929 974110 : next_l2 = em->l2_next;
930 :
931 974110 : if (next_ip4 == ETHERNET_INPUT_NEXT_IP4_INPUT && ip4_cksum_ok)
932 0 : next_ip4 = ETHERNET_INPUT_NEXT_IP4_INPUT_NCS;
933 :
934 : #ifdef CLIB_HAVE_VEC256
935 974110 : u16x16 et16_ip4 = u16x16_splat (et_ip4);
936 974110 : u16x16 et16_ip6 = u16x16_splat (et_ip6);
937 974110 : u16x16 et16_mpls = u16x16_splat (et_mpls);
938 974110 : u16x16 et16_vlan = u16x16_splat (et_vlan);
939 974110 : u16x16 et16_dot1ad = u16x16_splat (et_dot1ad);
940 974110 : u16x16 next16_ip4 = u16x16_splat (next_ip4);
941 974110 : u16x16 next16_ip6 = u16x16_splat (next_ip6);
942 974110 : u16x16 next16_mpls = u16x16_splat (next_mpls);
943 974110 : u16x16 next16_l2 = u16x16_splat (next_l2);
944 974110 : u16x16 zero = { 0 };
945 974110 : u16x16 stairs = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
946 : #endif
947 :
948 974110 : etype = etypes;
949 974110 : n_left = n_packets;
950 974110 : next = nexts;
951 974110 : n_slowpath = 0;
952 974110 : i = 0;
953 :
954 : /* fastpath - in l3 mode hadles ip4, ip6 and mpls packets, other packets
955 : are considered as slowpath, in l2 mode all untagged packets are
956 : considered as fastpath */
957 8247440 : while (n_left > 0)
958 : {
959 : #ifdef CLIB_HAVE_VEC256
960 7273330 : if (n_left >= 16)
961 : {
962 2804860 : u16x16 r = zero;
963 2804860 : u16x16 e16 = u16x16_load_unaligned (etype);
964 2804860 : if (main_is_l3)
965 : {
966 330388 : r += (e16 == et16_ip4) & next16_ip4;
967 330388 : r += (e16 == et16_ip6) & next16_ip6;
968 330388 : r += (e16 == et16_mpls) & next16_mpls;
969 : }
970 : else
971 2474480 : r = ((e16 != et16_vlan) & (e16 != et16_dot1ad)) & next16_l2;
972 2804860 : u16x16_store_unaligned (r, next);
973 :
974 2804860 : if (!u16x16_is_all_zero (r == zero))
975 : {
976 291 : if (u16x16_is_all_zero (r))
977 : {
978 291 : u16x16_store_unaligned (u16x16_splat (i) + stairs,
979 291 : slowpath_indices + n_slowpath);
980 291 : n_slowpath += 16;
981 : }
982 : else
983 : {
984 0 : for (int j = 0; j < 16; j++)
985 0 : if (next[j] == 0)
986 0 : slowpath_indices[n_slowpath++] = i + j;
987 : }
988 : }
989 :
990 2804860 : etype += 16;
991 2804860 : next += 16;
992 2804860 : n_left -= 16;
993 2804860 : i += 16;
994 2804860 : continue;
995 : }
996 : #endif
997 4468460 : if (main_is_l3 && etype[0] == et_ip4)
998 187020 : next[0] = next_ip4;
999 4281440 : else if (main_is_l3 && etype[0] == et_ip6)
1000 163103 : next[0] = next_ip6;
1001 4118340 : else if (main_is_l3 && etype[0] == et_mpls)
1002 125 : next[0] = next_mpls;
1003 4118210 : else if (main_is_l3 == 0 &&
1004 4115910 : etype[0] != et_vlan && etype[0] != et_dot1ad)
1005 4115870 : next[0] = next_l2;
1006 : else
1007 : {
1008 2342 : next[0] = 0;
1009 2342 : slowpath_indices[n_slowpath++] = i;
1010 : }
1011 :
1012 4468460 : etype += 1;
1013 4468460 : next += 1;
1014 4468460 : n_left -= 1;
1015 4468460 : i += 1;
1016 : }
1017 :
1018 974110 : if (n_slowpath)
1019 : {
1020 2188 : vnet_main_t *vnm = vnet_get_main ();
1021 2188 : n_left = n_slowpath;
1022 2188 : u16 *si = slowpath_indices;
1023 2188 : u32 last_unknown_etype = ~0;
1024 2188 : u32 last_unknown_next = ~0;
1025 2188 : eth_input_tag_lookup_t dot1ad_lookup, dot1q_lookup = {
1026 : .mask = -1LL,
1027 2188 : .tag = tags[si[0]] ^ -1LL,
1028 : .sw_if_index = ~0
1029 : };
1030 :
1031 2188 : clib_memcpy_fast (&dot1ad_lookup, &dot1q_lookup, sizeof (dot1q_lookup));
1032 :
1033 9186 : while (n_left)
1034 : {
1035 6998 : i = si[0];
1036 6998 : u16 etype = etypes[i];
1037 :
1038 6998 : if (etype == et_vlan)
1039 : {
1040 4304 : vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
1041 4304 : eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
1042 4304 : &dot1q_lookup, dmacs_bad[i], 0,
1043 : main_is_l3, dmac_check);
1044 :
1045 : }
1046 2694 : else if (etype == et_dot1ad)
1047 : {
1048 686 : vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
1049 686 : eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
1050 686 : &dot1ad_lookup, dmacs_bad[i], 1,
1051 : main_is_l3, dmac_check);
1052 : }
1053 : else
1054 : {
1055 : /* untagged packet with not well known etyertype */
1056 2008 : if (last_unknown_etype != etype)
1057 : {
1058 2006 : last_unknown_etype = etype;
1059 2006 : etype = clib_host_to_net_u16 (etype);
1060 2006 : last_unknown_next = eth_input_next_by_type (etype);
1061 : }
1062 2008 : if (dmac_check && main_is_l3 && dmacs_bad[i])
1063 4 : {
1064 4 : vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
1065 4 : b->error = node->errors[ETHERNET_ERROR_L3_MAC_MISMATCH];
1066 4 : nexts[i] = ETHERNET_INPUT_NEXT_PUNT;
1067 : }
1068 : else
1069 2004 : nexts[i] = last_unknown_next;
1070 : }
1071 :
1072 : /* next */
1073 6998 : n_left--;
1074 6998 : si++;
1075 : }
1076 :
1077 2188 : eth_input_update_if_counters (vm, vnm, &dot1q_lookup);
1078 2188 : eth_input_update_if_counters (vm, vnm, &dot1ad_lookup);
1079 : }
1080 :
1081 974110 : vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_packets);
1082 974109 : }
1083 :
1084 : static_always_inline void
1085 974110 : eth_input_single_int (vlib_main_t * vm, vlib_node_runtime_t * node,
1086 : vnet_hw_interface_t * hi, u32 * from, u32 n_pkts,
1087 : int ip4_cksum_ok)
1088 : {
1089 974110 : ethernet_main_t *em = ðernet_main;
1090 : ethernet_interface_t *ei;
1091 974110 : ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
1092 974110 : main_intf_t *intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1093 974110 : subint_config_t *subint0 = &intf0->untagged_subint;
1094 :
1095 974110 : int main_is_l3 = (subint0->flags & SUBINT_CONFIG_L2) == 0;
1096 974110 : int int_is_l3 = ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3;
1097 :
1098 974110 : if (main_is_l3)
1099 : {
1100 111589 : if (int_is_l3 || /* DMAC filter already done by NIC */
1101 111589 : ((hi->l2_if_count != 0) && (hi->l3_if_count == 0)))
1102 : { /* All L2 usage - DMAC check not needed */
1103 32 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1104 : /*is_l3 */ 1, ip4_cksum_ok, 0);
1105 : }
1106 : else
1107 : { /* DMAC check needed for L3 */
1108 111557 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1109 : /*is_l3 */ 1, ip4_cksum_ok, 1);
1110 : }
1111 111588 : return;
1112 : }
1113 : else
1114 : {
1115 862521 : if (hi->l3_if_count == 0)
1116 : { /* All L2 usage - DMAC check not needed */
1117 862473 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1118 : /*is_l3 */ 0, ip4_cksum_ok, 0);
1119 : }
1120 : else
1121 : { /* DMAC check needed for L3 */
1122 48 : eth_input_process_frame (vm, node, hi, from, n_pkts,
1123 : /*is_l3 */ 0, ip4_cksum_ok, 1);
1124 : }
1125 862521 : return;
1126 : }
1127 : }
1128 :
1129 : static_always_inline void
1130 1611110 : ethernet_input_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
1131 : vlib_frame_t * from_frame)
1132 : {
1133 1611110 : vnet_main_t *vnm = vnet_get_main ();
1134 : u32 *from, n_left;
1135 1611110 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
1136 : {
1137 14970 : from = vlib_frame_vector_args (from_frame);
1138 14970 : n_left = from_frame->n_vectors;
1139 :
1140 507997 : while (n_left)
1141 : {
1142 : ethernet_input_trace_t *t0;
1143 493023 : vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
1144 :
1145 493051 : if (b0->flags & VLIB_BUFFER_IS_TRACED)
1146 : {
1147 492044 : t0 = vlib_add_trace (vm, node, b0,
1148 : sizeof (ethernet_input_trace_t));
1149 492066 : clib_memcpy_fast (t0->packet_data, b0->data + b0->current_data,
1150 : sizeof (t0->packet_data));
1151 492039 : t0->frame_flags = from_frame->flags;
1152 492033 : clib_memcpy_fast (&t0->frame_data,
1153 492039 : vlib_frame_scalar_args (from_frame),
1154 : sizeof (ethernet_input_frame_t));
1155 : }
1156 493027 : from += 1;
1157 493027 : n_left -= 1;
1158 : }
1159 : }
1160 :
1161 : /* rx pcap capture if enabled */
1162 1611120 : if (PREDICT_FALSE (vnm->pcap.pcap_rx_enable))
1163 : {
1164 : u32 bi0;
1165 9 : vnet_pcap_t *pp = &vnm->pcap;
1166 :
1167 9 : from = vlib_frame_vector_args (from_frame);
1168 9 : n_left = from_frame->n_vectors;
1169 188 : while (n_left > 0)
1170 : {
1171 : vlib_buffer_t *b0;
1172 179 : bi0 = from[0];
1173 179 : from++;
1174 179 : n_left--;
1175 179 : b0 = vlib_get_buffer (vm, bi0);
1176 179 : if (vnet_is_packet_pcaped (pp, b0, ~0))
1177 87 : pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt);
1178 : }
1179 : }
1180 1611120 : }
1181 :
1182 : static_always_inline void
1183 637001 : ethernet_input_inline (vlib_main_t * vm,
1184 : vlib_node_runtime_t * node,
1185 : u32 * from, u32 n_packets,
1186 : ethernet_input_variant_t variant)
1187 : {
1188 637001 : vnet_main_t *vnm = vnet_get_main ();
1189 637001 : ethernet_main_t *em = ðernet_main;
1190 : vlib_node_runtime_t *error_node;
1191 : u32 n_left_from, next_index, *to_next;
1192 : u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
1193 637001 : u32 thread_index = vm->thread_index;
1194 637001 : u32 cached_sw_if_index = ~0;
1195 637001 : u32 cached_is_l2 = 0; /* shut up gcc */
1196 637001 : vnet_hw_interface_t *hi = NULL; /* used for main interface only */
1197 637001 : ethernet_interface_t *ei = NULL;
1198 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
1199 637001 : vlib_buffer_t **b = bufs;
1200 :
1201 637001 : if (variant != ETHERNET_INPUT_VARIANT_ETHERNET)
1202 0 : error_node = vlib_node_get_runtime (vm, ethernet_input_node.index);
1203 : else
1204 637001 : error_node = node;
1205 :
1206 637001 : n_left_from = n_packets;
1207 :
1208 637001 : next_index = node->cached_next_index;
1209 637001 : stats_sw_if_index = node->runtime_data[0];
1210 637001 : stats_n_packets = stats_n_bytes = 0;
1211 637001 : vlib_get_buffers (vm, from, bufs, n_left_from);
1212 :
1213 1274000 : while (n_left_from > 0)
1214 : {
1215 : u32 n_left_to_next;
1216 :
1217 637004 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1218 :
1219 6686720 : while (n_left_from >= 4 && n_left_to_next >= 2)
1220 : {
1221 : u32 bi0, bi1;
1222 : vlib_buffer_t *b0, *b1;
1223 : u8 next0, next1, error0, error1;
1224 : u16 type0, orig_type0, type1, orig_type1;
1225 : u16 outer_id0, inner_id0, outer_id1, inner_id1;
1226 : u32 match_flags0, match_flags1;
1227 : u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1,
1228 : new_sw_if_index1, len1;
1229 : vnet_hw_interface_t *hi0, *hi1;
1230 : main_intf_t *main_intf0, *main_intf1;
1231 : vlan_intf_t *vlan_intf0, *vlan_intf1;
1232 : qinq_intf_t *qinq_intf0, *qinq_intf1;
1233 : u32 is_l20, is_l21;
1234 : ethernet_header_t *e0, *e1;
1235 : u64 dmacs[2];
1236 : u8 dmacs_bad[2];
1237 :
1238 : /* Prefetch next iteration. */
1239 : {
1240 6049720 : vlib_prefetch_buffer_header (b[2], STORE);
1241 6049720 : vlib_prefetch_buffer_header (b[3], STORE);
1242 :
1243 6049720 : CLIB_PREFETCH (b[2]->data, sizeof (ethernet_header_t), LOAD);
1244 6049720 : CLIB_PREFETCH (b[3]->data, sizeof (ethernet_header_t), LOAD);
1245 : }
1246 :
1247 6049720 : bi0 = from[0];
1248 6049720 : bi1 = from[1];
1249 6049720 : to_next[0] = bi0;
1250 6049720 : to_next[1] = bi1;
1251 6049720 : from += 2;
1252 6049720 : to_next += 2;
1253 6049720 : n_left_to_next -= 2;
1254 6049720 : n_left_from -= 2;
1255 :
1256 6049720 : b0 = b[0];
1257 6049720 : b1 = b[1];
1258 6049720 : b += 2;
1259 :
1260 6049720 : error0 = error1 = ETHERNET_ERROR_NONE;
1261 6049720 : e0 = vlib_buffer_get_current (b0);
1262 6049720 : type0 = clib_net_to_host_u16 (e0->type);
1263 6049720 : e1 = vlib_buffer_get_current (b1);
1264 6049720 : type1 = clib_net_to_host_u16 (e1->type);
1265 :
1266 : /* Set the L2 header offset for all packets */
1267 6049720 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1268 6049720 : vnet_buffer (b1)->l2_hdr_offset = b1->current_data;
1269 6049720 : b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1270 6049720 : b1->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1271 :
1272 : /* Speed-path for the untagged case */
1273 6049720 : if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET
1274 : && !ethernet_frame_is_any_tagged_x2 (type0,
1275 : type1)))
1276 : {
1277 : main_intf_t *intf0;
1278 : subint_config_t *subint0;
1279 : u32 sw_if_index0, sw_if_index1;
1280 :
1281 6049460 : sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1282 6049460 : sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1283 6049460 : is_l20 = cached_is_l2;
1284 :
1285 : /* This is probably wholly unnecessary */
1286 6049460 : if (PREDICT_FALSE (sw_if_index0 != sw_if_index1))
1287 90139 : goto slowpath;
1288 :
1289 : /* Now sw_if_index0 == sw_if_index1 */
1290 5959320 : if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1291 : {
1292 520168 : cached_sw_if_index = sw_if_index0;
1293 520168 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1294 520168 : ei = ethernet_get_interface (em, hi->hw_if_index);
1295 520168 : intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1296 520168 : subint0 = &intf0->untagged_subint;
1297 520168 : cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1298 : }
1299 :
1300 5959320 : if (PREDICT_TRUE (is_l20 != 0))
1301 : {
1302 3929010 : vnet_buffer (b0)->l3_hdr_offset =
1303 3929010 : vnet_buffer (b0)->l2_hdr_offset +
1304 : sizeof (ethernet_header_t);
1305 3929010 : vnet_buffer (b1)->l3_hdr_offset =
1306 3929010 : vnet_buffer (b1)->l2_hdr_offset +
1307 : sizeof (ethernet_header_t);
1308 3929010 : b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1309 3929010 : b1->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1310 3929010 : next0 = em->l2_next;
1311 3929010 : vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1312 3929010 : next1 = em->l2_next;
1313 3929010 : vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t);
1314 : }
1315 : else
1316 : {
1317 2030310 : if (ei && (ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3))
1318 0 : goto skip_dmac_check01;
1319 :
1320 2030310 : dmacs[0] = *(u64 *) e0;
1321 2030310 : dmacs[1] = *(u64 *) e1;
1322 :
1323 2030310 : if (ei && vec_len (ei->secondary_addrs))
1324 2030310 : ethernet_input_inline_dmac_check (hi, dmacs,
1325 : dmacs_bad,
1326 : 2 /* n_packets */ ,
1327 : ei,
1328 : 1 /* have_sec_dmac */ );
1329 : else
1330 0 : ethernet_input_inline_dmac_check (hi, dmacs,
1331 : dmacs_bad,
1332 : 2 /* n_packets */ ,
1333 : ei,
1334 : 0 /* have_sec_dmac */ );
1335 :
1336 2030310 : if (dmacs_bad[0])
1337 0 : error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1338 2030310 : if (dmacs_bad[1])
1339 0 : error1 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1340 :
1341 2030310 : skip_dmac_check01:
1342 2030310 : vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1343 2030310 : determine_next_node (em, variant, 0, type0, b0,
1344 : &error0, &next0);
1345 2030310 : vlib_buffer_advance (b1, sizeof (ethernet_header_t));
1346 2030310 : determine_next_node (em, variant, 0, type1, b1,
1347 : &error1, &next1);
1348 : }
1349 5959320 : goto ship_it01;
1350 : }
1351 :
1352 : /* Slow-path for the tagged case */
1353 254 : slowpath:
1354 90393 : parse_header (variant,
1355 : b0,
1356 : &type0,
1357 : &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1358 :
1359 90393 : parse_header (variant,
1360 : b1,
1361 : &type1,
1362 : &orig_type1, &outer_id1, &inner_id1, &match_flags1);
1363 :
1364 90393 : old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1365 90393 : old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1366 :
1367 90393 : eth_vlan_table_lookups (em,
1368 : vnm,
1369 : old_sw_if_index0,
1370 : orig_type0,
1371 : outer_id0,
1372 : inner_id0,
1373 : &hi0,
1374 : &main_intf0, &vlan_intf0, &qinq_intf0);
1375 :
1376 90393 : eth_vlan_table_lookups (em,
1377 : vnm,
1378 : old_sw_if_index1,
1379 : orig_type1,
1380 : outer_id1,
1381 : inner_id1,
1382 : &hi1,
1383 : &main_intf1, &vlan_intf1, &qinq_intf1);
1384 :
1385 90393 : identify_subint (em,
1386 : hi0,
1387 : b0,
1388 : match_flags0,
1389 : main_intf0,
1390 : vlan_intf0,
1391 : qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1392 :
1393 90393 : identify_subint (em,
1394 : hi1,
1395 : b1,
1396 : match_flags1,
1397 : main_intf1,
1398 : vlan_intf1,
1399 : qinq_intf1, &new_sw_if_index1, &error1, &is_l21);
1400 :
1401 : // Save RX sw_if_index for later nodes
1402 90393 : vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1403 90393 : error0 !=
1404 90393 : ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1405 90393 : vnet_buffer (b1)->sw_if_index[VLIB_RX] =
1406 90393 : error1 !=
1407 90393 : ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1;
1408 :
1409 : // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1)
1410 90393 : if (((new_sw_if_index0 != ~0)
1411 90393 : && (new_sw_if_index0 != old_sw_if_index0))
1412 90139 : || ((new_sw_if_index1 != ~0)
1413 90139 : && (new_sw_if_index1 != old_sw_if_index1)))
1414 : {
1415 :
1416 254 : len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1417 254 : - vnet_buffer (b0)->l2_hdr_offset;
1418 254 : len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data
1419 254 : - vnet_buffer (b1)->l2_hdr_offset;
1420 :
1421 254 : stats_n_packets += 2;
1422 254 : stats_n_bytes += len0 + len1;
1423 :
1424 254 : if (PREDICT_FALSE
1425 : (!(new_sw_if_index0 == stats_sw_if_index
1426 : && new_sw_if_index1 == stats_sw_if_index)))
1427 : {
1428 2 : stats_n_packets -= 2;
1429 2 : stats_n_bytes -= len0 + len1;
1430 :
1431 2 : if (new_sw_if_index0 != old_sw_if_index0
1432 2 : && new_sw_if_index0 != ~0)
1433 2 : vlib_increment_combined_counter (vnm->
1434 : interface_main.combined_sw_if_counters
1435 : +
1436 : VNET_INTERFACE_COUNTER_RX,
1437 : thread_index,
1438 : new_sw_if_index0, 1,
1439 : len0);
1440 2 : if (new_sw_if_index1 != old_sw_if_index1
1441 2 : && new_sw_if_index1 != ~0)
1442 2 : vlib_increment_combined_counter (vnm->
1443 : interface_main.combined_sw_if_counters
1444 : +
1445 : VNET_INTERFACE_COUNTER_RX,
1446 : thread_index,
1447 : new_sw_if_index1, 1,
1448 : len1);
1449 :
1450 2 : if (new_sw_if_index0 == new_sw_if_index1)
1451 : {
1452 2 : if (stats_n_packets > 0)
1453 : {
1454 0 : vlib_increment_combined_counter
1455 : (vnm->interface_main.combined_sw_if_counters
1456 : + VNET_INTERFACE_COUNTER_RX,
1457 : thread_index,
1458 : stats_sw_if_index,
1459 : stats_n_packets, stats_n_bytes);
1460 0 : stats_n_packets = stats_n_bytes = 0;
1461 : }
1462 2 : stats_sw_if_index = new_sw_if_index0;
1463 : }
1464 : }
1465 : }
1466 :
1467 90393 : if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1468 0 : is_l20 = is_l21 = 0;
1469 :
1470 90393 : determine_next_node (em, variant, is_l20, type0, b0, &error0,
1471 : &next0);
1472 90393 : determine_next_node (em, variant, is_l21, type1, b1, &error1,
1473 : &next1);
1474 :
1475 6049720 : ship_it01:
1476 6049720 : b0->error = error_node->errors[error0];
1477 6049720 : b1->error = error_node->errors[error1];
1478 :
1479 : // verify speculative enqueue
1480 6049720 : vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1481 : n_left_to_next, bi0, bi1, next0,
1482 : next1);
1483 : }
1484 :
1485 1813980 : while (n_left_from > 0 && n_left_to_next > 0)
1486 : {
1487 : u32 bi0;
1488 : vlib_buffer_t *b0;
1489 : u8 error0, next0;
1490 : u16 type0, orig_type0;
1491 : u16 outer_id0, inner_id0;
1492 : u32 match_flags0;
1493 : u32 old_sw_if_index0, new_sw_if_index0, len0;
1494 : vnet_hw_interface_t *hi0;
1495 : main_intf_t *main_intf0;
1496 : vlan_intf_t *vlan_intf0;
1497 : qinq_intf_t *qinq_intf0;
1498 : ethernet_header_t *e0;
1499 : u32 is_l20;
1500 : u64 dmacs[2];
1501 : u8 dmacs_bad[2];
1502 :
1503 : // Prefetch next iteration
1504 1176980 : if (n_left_from > 1)
1505 : {
1506 539974 : vlib_prefetch_buffer_header (b[1], STORE);
1507 539974 : clib_prefetch_load (b[1]->data);
1508 : }
1509 :
1510 1176980 : bi0 = from[0];
1511 1176980 : to_next[0] = bi0;
1512 1176980 : from += 1;
1513 1176980 : to_next += 1;
1514 1176980 : n_left_from -= 1;
1515 1176980 : n_left_to_next -= 1;
1516 :
1517 1176980 : b0 = b[0];
1518 1176980 : b += 1;
1519 :
1520 1176980 : error0 = ETHERNET_ERROR_NONE;
1521 1176980 : e0 = vlib_buffer_get_current (b0);
1522 1176980 : type0 = clib_net_to_host_u16 (e0->type);
1523 :
1524 : /* Set the L2 header offset for all packets */
1525 1176980 : vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1526 1176980 : b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1527 :
1528 : /* Speed-path for the untagged case */
1529 1176980 : if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET
1530 : && !ethernet_frame_is_tagged (type0)))
1531 : {
1532 : main_intf_t *intf0;
1533 : subint_config_t *subint0;
1534 : u32 sw_if_index0;
1535 :
1536 1176970 : sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1537 1176970 : is_l20 = cached_is_l2;
1538 :
1539 1176970 : if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1540 : {
1541 280960 : cached_sw_if_index = sw_if_index0;
1542 280960 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1543 280960 : ei = ethernet_get_interface (em, hi->hw_if_index);
1544 280960 : intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1545 280960 : subint0 = &intf0->untagged_subint;
1546 280960 : cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1547 : }
1548 :
1549 :
1550 1176970 : if (PREDICT_TRUE (is_l20 != 0))
1551 : {
1552 733584 : vnet_buffer (b0)->l3_hdr_offset =
1553 733584 : vnet_buffer (b0)->l2_hdr_offset +
1554 : sizeof (ethernet_header_t);
1555 733584 : b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1556 733584 : next0 = em->l2_next;
1557 733584 : vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1558 : }
1559 : else
1560 : {
1561 443385 : if (ei && ei->flags & ETHERNET_INTERFACE_FLAG_STATUS_L3)
1562 0 : goto skip_dmac_check0;
1563 :
1564 443385 : dmacs[0] = *(u64 *) e0;
1565 :
1566 443385 : if (ei)
1567 : {
1568 443385 : if (vec_len (ei->secondary_addrs))
1569 443269 : ethernet_input_inline_dmac_check (
1570 : hi, dmacs, dmacs_bad, 1 /* n_packets */, ei,
1571 : 1 /* have_sec_dmac */);
1572 : else
1573 116 : ethernet_input_inline_dmac_check (
1574 : hi, dmacs, dmacs_bad, 1 /* n_packets */, ei,
1575 : 0 /* have_sec_dmac */);
1576 :
1577 443385 : if (dmacs_bad[0])
1578 2 : error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1579 : }
1580 :
1581 443383 : skip_dmac_check0:
1582 443385 : vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1583 443385 : determine_next_node (em, variant, 0, type0, b0,
1584 : &error0, &next0);
1585 : }
1586 1176970 : goto ship_it0;
1587 : }
1588 :
1589 : /* Slow-path for the tagged case */
1590 6 : parse_header (variant,
1591 : b0,
1592 : &type0,
1593 : &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1594 :
1595 6 : old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1596 :
1597 6 : eth_vlan_table_lookups (em,
1598 : vnm,
1599 : old_sw_if_index0,
1600 : orig_type0,
1601 : outer_id0,
1602 : inner_id0,
1603 : &hi0,
1604 : &main_intf0, &vlan_intf0, &qinq_intf0);
1605 :
1606 6 : identify_subint (em,
1607 : hi0,
1608 : b0,
1609 : match_flags0,
1610 : main_intf0,
1611 : vlan_intf0,
1612 : qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1613 :
1614 : // Save RX sw_if_index for later nodes
1615 6 : vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1616 6 : error0 !=
1617 6 : ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1618 :
1619 : // Increment subinterface stats
1620 : // Note that interface-level counters have already been incremented
1621 : // prior to calling this function. Thus only subinterface counters
1622 : // are incremented here.
1623 : //
1624 : // Interface level counters include packets received on the main
1625 : // interface and all subinterfaces. Subinterface level counters
1626 : // include only those packets received on that subinterface
1627 : // Increment stats if the subint is valid and it is not the main intf
1628 6 : if ((new_sw_if_index0 != ~0)
1629 6 : && (new_sw_if_index0 != old_sw_if_index0))
1630 : {
1631 :
1632 6 : len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1633 6 : - vnet_buffer (b0)->l2_hdr_offset;
1634 :
1635 6 : stats_n_packets += 1;
1636 6 : stats_n_bytes += len0;
1637 :
1638 : // Batch stat increments from the same subinterface so counters
1639 : // don't need to be incremented for every packet.
1640 6 : if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index))
1641 : {
1642 0 : stats_n_packets -= 1;
1643 0 : stats_n_bytes -= len0;
1644 :
1645 0 : if (new_sw_if_index0 != ~0)
1646 0 : vlib_increment_combined_counter
1647 : (vnm->interface_main.combined_sw_if_counters
1648 : + VNET_INTERFACE_COUNTER_RX,
1649 : thread_index, new_sw_if_index0, 1, len0);
1650 0 : if (stats_n_packets > 0)
1651 : {
1652 0 : vlib_increment_combined_counter
1653 : (vnm->interface_main.combined_sw_if_counters
1654 : + VNET_INTERFACE_COUNTER_RX,
1655 : thread_index,
1656 : stats_sw_if_index, stats_n_packets, stats_n_bytes);
1657 0 : stats_n_packets = stats_n_bytes = 0;
1658 : }
1659 0 : stats_sw_if_index = new_sw_if_index0;
1660 : }
1661 : }
1662 :
1663 6 : if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1664 0 : is_l20 = 0;
1665 :
1666 6 : determine_next_node (em, variant, is_l20, type0, b0, &error0,
1667 : &next0);
1668 :
1669 1176980 : ship_it0:
1670 1176980 : b0->error = error_node->errors[error0];
1671 :
1672 : // verify speculative enqueue
1673 1176980 : vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1674 : to_next, n_left_to_next,
1675 : bi0, next0);
1676 : }
1677 :
1678 637004 : vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1679 : }
1680 :
1681 : // Increment any remaining batched stats
1682 637001 : if (stats_n_packets > 0)
1683 : {
1684 4 : vlib_increment_combined_counter
1685 : (vnm->interface_main.combined_sw_if_counters
1686 : + VNET_INTERFACE_COUNTER_RX,
1687 : thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
1688 4 : node->runtime_data[0] = stats_sw_if_index;
1689 : }
1690 637001 : }
1691 :
1692 1613346 : VLIB_NODE_FN (ethernet_input_node) (vlib_main_t * vm,
1693 : vlib_node_runtime_t * node,
1694 : vlib_frame_t * frame)
1695 : {
1696 1611110 : vnet_main_t *vnm = vnet_get_main ();
1697 1611110 : u32 *from = vlib_frame_vector_args (frame);
1698 1611110 : u32 n_packets = frame->n_vectors;
1699 :
1700 1611110 : ethernet_input_trace (vm, node, frame);
1701 :
1702 1611110 : if (frame->flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
1703 : {
1704 974110 : ethernet_input_frame_t *ef = vlib_frame_scalar_args (frame);
1705 974110 : int ip4_cksum_ok = (frame->flags & ETH_INPUT_FRAME_F_IP4_CKSUM_OK) != 0;
1706 974110 : vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, ef->hw_if_index);
1707 974110 : eth_input_single_int (vm, node, hi, from, n_packets, ip4_cksum_ok);
1708 : }
1709 : else
1710 637001 : ethernet_input_inline (vm, node, from, n_packets,
1711 : ETHERNET_INPUT_VARIANT_ETHERNET);
1712 1611110 : return n_packets;
1713 : }
1714 :
1715 2236 : VLIB_NODE_FN (ethernet_input_type_node) (vlib_main_t * vm,
1716 : vlib_node_runtime_t * node,
1717 : vlib_frame_t * from_frame)
1718 : {
1719 0 : u32 *from = vlib_frame_vector_args (from_frame);
1720 0 : u32 n_packets = from_frame->n_vectors;
1721 0 : ethernet_input_trace (vm, node, from_frame);
1722 0 : ethernet_input_inline (vm, node, from, n_packets,
1723 : ETHERNET_INPUT_VARIANT_ETHERNET_TYPE);
1724 0 : return n_packets;
1725 : }
1726 :
1727 2236 : VLIB_NODE_FN (ethernet_input_not_l2_node) (vlib_main_t * vm,
1728 : vlib_node_runtime_t * node,
1729 : vlib_frame_t * from_frame)
1730 : {
1731 0 : u32 *from = vlib_frame_vector_args (from_frame);
1732 0 : u32 n_packets = from_frame->n_vectors;
1733 0 : ethernet_input_trace (vm, node, from_frame);
1734 0 : ethernet_input_inline (vm, node, from, n_packets,
1735 : ETHERNET_INPUT_VARIANT_NOT_L2);
1736 0 : return n_packets;
1737 : }
1738 :
1739 :
1740 : // Return the subinterface config struct for the given sw_if_index
1741 : // Also return via parameter the appropriate match flags for the
1742 : // configured number of tags.
1743 : // On error (unsupported or not ethernet) return 0.
1744 : static subint_config_t *
1745 28372 : ethernet_sw_interface_get_config (vnet_main_t * vnm,
1746 : u32 sw_if_index,
1747 : u32 * flags, u32 * unsupported)
1748 : {
1749 28372 : ethernet_main_t *em = ðernet_main;
1750 : vnet_hw_interface_t *hi;
1751 : vnet_sw_interface_t *si;
1752 : main_intf_t *main_intf;
1753 : vlan_table_t *vlan_table;
1754 : qinq_table_t *qinq_table;
1755 28372 : subint_config_t *subint = 0;
1756 :
1757 28372 : hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1758 :
1759 28372 : if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index))
1760 : {
1761 3078 : *unsupported = 0;
1762 3078 : goto done; // non-ethernet interface
1763 : }
1764 :
1765 : // ensure there's an entry for the main intf (shouldn't really be necessary)
1766 25294 : vec_validate (em->main_intfs, hi->hw_if_index);
1767 25294 : main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1768 :
1769 : // Locate the subint for the given ethernet config
1770 25294 : si = vnet_get_sw_interface (vnm, sw_if_index);
1771 :
1772 25294 : if (si->type == VNET_SW_INTERFACE_TYPE_P2P)
1773 : {
1774 2107 : p2p_ethernet_main_t *p2pm = &p2p_main;
1775 : u32 p2pe_sw_if_index =
1776 2107 : p2p_ethernet_lookup (hi->hw_if_index, si->p2p.client_mac);
1777 2107 : if (p2pe_sw_if_index == ~0)
1778 : {
1779 1023 : pool_get (p2pm->p2p_subif_pool, subint);
1780 1023 : si->p2p.pool_index = subint - p2pm->p2p_subif_pool;
1781 : }
1782 : else
1783 1084 : subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index);
1784 2107 : *flags = SUBINT_CONFIG_P2P;
1785 : }
1786 23187 : else if (si->type == VNET_SW_INTERFACE_TYPE_PIPE)
1787 : {
1788 : pipe_t *pipe;
1789 :
1790 10 : pipe = pipe_get (sw_if_index);
1791 10 : subint = &pipe->subint;
1792 10 : *flags = SUBINT_CONFIG_P2P;
1793 : }
1794 23177 : else if (si->sub.eth.flags.default_sub)
1795 : {
1796 0 : subint = &main_intf->default_subint;
1797 0 : *flags = SUBINT_CONFIG_MATCH_1_TAG |
1798 : SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG;
1799 : }
1800 23177 : else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0))
1801 : {
1802 : // if no flags are set then this is a main interface
1803 : // so treat as untagged
1804 22823 : subint = &main_intf->untagged_subint;
1805 22823 : *flags = SUBINT_CONFIG_MATCH_0_TAG;
1806 : }
1807 : else
1808 : {
1809 : // one or two tags
1810 : // first get the vlan table
1811 354 : if (si->sub.eth.flags.dot1ad)
1812 : {
1813 47 : if (main_intf->dot1ad_vlans == 0)
1814 : {
1815 : // Allocate a vlan table from the pool
1816 18 : pool_get (em->vlan_pool, vlan_table);
1817 18 : main_intf->dot1ad_vlans = vlan_table - em->vlan_pool;
1818 : }
1819 : else
1820 : {
1821 : // Get ptr to existing vlan table
1822 29 : vlan_table =
1823 29 : vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans);
1824 : }
1825 : }
1826 : else
1827 : { // dot1q
1828 307 : if (main_intf->dot1q_vlans == 0)
1829 : {
1830 : // Allocate a vlan table from the pool
1831 34 : pool_get (em->vlan_pool, vlan_table);
1832 34 : main_intf->dot1q_vlans = vlan_table - em->vlan_pool;
1833 : }
1834 : else
1835 : {
1836 : // Get ptr to existing vlan table
1837 273 : vlan_table =
1838 273 : vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans);
1839 : }
1840 : }
1841 :
1842 354 : if (si->sub.eth.flags.one_tag)
1843 : {
1844 614 : *flags = si->sub.eth.flags.exact_match ?
1845 307 : SUBINT_CONFIG_MATCH_1_TAG :
1846 : (SUBINT_CONFIG_MATCH_1_TAG |
1847 : SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG);
1848 :
1849 307 : if (si->sub.eth.flags.outer_vlan_id_any)
1850 : {
1851 : // not implemented yet
1852 0 : *unsupported = 1;
1853 0 : goto done;
1854 : }
1855 : else
1856 : {
1857 : // a single vlan, a common case
1858 307 : subint =
1859 307 : &vlan_table->vlans[si->sub.eth.
1860 : outer_vlan_id].single_tag_subint;
1861 : }
1862 :
1863 : }
1864 : else
1865 : {
1866 : // Two tags
1867 94 : *flags = si->sub.eth.flags.exact_match ?
1868 47 : SUBINT_CONFIG_MATCH_2_TAG :
1869 : (SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG);
1870 :
1871 47 : if (si->sub.eth.flags.outer_vlan_id_any
1872 0 : && si->sub.eth.flags.inner_vlan_id_any)
1873 : {
1874 : // not implemented yet
1875 0 : *unsupported = 1;
1876 0 : goto done;
1877 : }
1878 :
1879 47 : if (si->sub.eth.flags.inner_vlan_id_any)
1880 : {
1881 : // a specific outer and "any" inner
1882 : // don't need a qinq table for this
1883 0 : subint =
1884 0 : &vlan_table->vlans[si->sub.eth.
1885 : outer_vlan_id].inner_any_subint;
1886 0 : if (si->sub.eth.flags.exact_match)
1887 : {
1888 0 : *flags = SUBINT_CONFIG_MATCH_2_TAG;
1889 : }
1890 : else
1891 : {
1892 0 : *flags = SUBINT_CONFIG_MATCH_2_TAG |
1893 : SUBINT_CONFIG_MATCH_3_TAG;
1894 : }
1895 : }
1896 : else
1897 : {
1898 : // a specific outer + specific innner vlan id, a common case
1899 :
1900 : // get the qinq table
1901 47 : if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0)
1902 : {
1903 : // Allocate a qinq table from the pool
1904 18 : pool_get (em->qinq_pool, qinq_table);
1905 18 : vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs =
1906 18 : qinq_table - em->qinq_pool;
1907 : }
1908 : else
1909 : {
1910 : // Get ptr to existing qinq table
1911 29 : qinq_table =
1912 29 : vec_elt_at_index (em->qinq_pool,
1913 : vlan_table->vlans[si->sub.
1914 : eth.outer_vlan_id].
1915 : qinqs);
1916 : }
1917 47 : subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint;
1918 : }
1919 : }
1920 : }
1921 :
1922 28372 : done:
1923 28372 : return subint;
1924 : }
1925 :
1926 : static clib_error_t *
1927 13268 : ethernet_sw_interface_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
1928 : {
1929 : subint_config_t *subint;
1930 : u32 placeholder_flags;
1931 : u32 placeholder_unsup;
1932 13268 : clib_error_t *error = 0;
1933 :
1934 : // Find the config for this subinterface
1935 : subint =
1936 13268 : ethernet_sw_interface_get_config (vnm, sw_if_index, &placeholder_flags,
1937 : &placeholder_unsup);
1938 :
1939 13268 : if (subint == 0)
1940 : {
1941 : // not implemented yet or not ethernet
1942 1214 : goto done;
1943 : }
1944 :
1945 12054 : subint->sw_if_index =
1946 12054 : ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0);
1947 :
1948 13268 : done:
1949 13268 : return error;
1950 : }
1951 :
1952 2801 : VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ethernet_sw_interface_up_down);
1953 :
1954 :
1955 : #ifndef CLIB_MARCH_VARIANT
1956 : // Set the L2/L3 mode for the subinterface
1957 : void
1958 3507 : ethernet_sw_interface_set_l2_mode (vnet_main_t * vnm, u32 sw_if_index, u32 l2)
1959 : {
1960 : subint_config_t *subint;
1961 : u32 placeholder_flags;
1962 : u32 placeholder_unsup;
1963 : int is_port;
1964 3507 : vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index);
1965 :
1966 3507 : is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB);
1967 :
1968 : // Find the config for this subinterface
1969 : subint =
1970 3507 : ethernet_sw_interface_get_config (vnm, sw_if_index, &placeholder_flags,
1971 : &placeholder_unsup);
1972 :
1973 3507 : if (subint == 0)
1974 : {
1975 : // unimplemented or not ethernet
1976 143 : goto done;
1977 : }
1978 :
1979 : // Double check that the config we found is for our interface (or the interface is down)
1980 3364 : ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1981 :
1982 3364 : if (l2)
1983 : {
1984 1235 : subint->flags |= SUBINT_CONFIG_L2;
1985 1235 : if (is_port)
1986 1181 : subint->flags |=
1987 : SUBINT_CONFIG_MATCH_0_TAG | SUBINT_CONFIG_MATCH_1_TAG
1988 : | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG;
1989 : }
1990 : else
1991 : {
1992 2129 : subint->flags &= ~SUBINT_CONFIG_L2;
1993 2129 : if (is_port)
1994 2093 : subint->flags &=
1995 : ~(SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG
1996 : | SUBINT_CONFIG_MATCH_3_TAG);
1997 : }
1998 :
1999 36 : done:
2000 3507 : return;
2001 : }
2002 :
2003 : /*
2004 : * Set the L2/L3 mode for the subinterface regardless of port
2005 : */
2006 : void
2007 0 : ethernet_sw_interface_set_l2_mode_noport (vnet_main_t * vnm,
2008 : u32 sw_if_index, u32 l2)
2009 : {
2010 : subint_config_t *subint;
2011 : u32 placeholder_flags;
2012 : u32 placeholder_unsup;
2013 :
2014 : /* Find the config for this subinterface */
2015 : subint =
2016 0 : ethernet_sw_interface_get_config (vnm, sw_if_index, &placeholder_flags,
2017 : &placeholder_unsup);
2018 :
2019 0 : if (subint == 0)
2020 : {
2021 : /* unimplemented or not ethernet */
2022 0 : goto done;
2023 : }
2024 :
2025 : /*
2026 : * Double check that the config we found is for our interface (or the
2027 : * interface is down)
2028 : */
2029 0 : ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
2030 :
2031 0 : if (l2)
2032 : {
2033 0 : subint->flags |= SUBINT_CONFIG_L2;
2034 : }
2035 : else
2036 : {
2037 0 : subint->flags &= ~SUBINT_CONFIG_L2;
2038 : }
2039 :
2040 0 : done:
2041 0 : return;
2042 : }
2043 : #endif
2044 :
2045 : static clib_error_t *
2046 11597 : ethernet_sw_interface_add_del (vnet_main_t * vnm,
2047 : u32 sw_if_index, u32 is_create)
2048 : {
2049 11597 : clib_error_t *error = 0;
2050 : subint_config_t *subint;
2051 : u32 match_flags;
2052 11597 : u32 unsupported = 0;
2053 :
2054 : // Find the config for this subinterface
2055 : subint =
2056 11597 : ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags,
2057 : &unsupported);
2058 :
2059 11597 : if (subint == 0)
2060 : {
2061 : // not implemented yet or not ethernet
2062 1721 : if (unsupported)
2063 : {
2064 : // this is the NYI case
2065 0 : error = clib_error_return (0, "not implemented yet");
2066 : }
2067 1721 : goto done;
2068 : }
2069 :
2070 9876 : if (!is_create)
2071 : {
2072 3641 : subint->flags = 0;
2073 3641 : return error;
2074 : }
2075 :
2076 : // Initialize the subint
2077 6235 : if (subint->flags & SUBINT_CONFIG_VALID)
2078 : {
2079 : // Error vlan already in use
2080 0 : error = clib_error_return (0, "vlan is already in use");
2081 : }
2082 : else
2083 : {
2084 : // Note that config is L3 by default
2085 6235 : subint->flags = SUBINT_CONFIG_VALID | match_flags;
2086 6235 : subint->sw_if_index = ~0; // because interfaces are initially down
2087 : }
2088 :
2089 7956 : done:
2090 7956 : return error;
2091 : }
2092 :
2093 3363 : VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ethernet_sw_interface_add_del);
2094 :
2095 : static char *ethernet_error_strings[] = {
2096 : #define ethernet_error(n,c,s) s,
2097 : #include "error.def"
2098 : #undef ethernet_error
2099 : };
2100 :
2101 : /* *INDENT-OFF* */
2102 178120 : VLIB_REGISTER_NODE (ethernet_input_node) = {
2103 : .name = "ethernet-input",
2104 : /* Takes a vector of packets. */
2105 : .vector_size = sizeof (u32),
2106 : .scalar_size = sizeof (ethernet_input_frame_t),
2107 : .n_errors = ETHERNET_N_ERROR,
2108 : .error_strings = ethernet_error_strings,
2109 : .n_next_nodes = ETHERNET_INPUT_N_NEXT,
2110 : .next_nodes = {
2111 : #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
2112 : foreach_ethernet_input_next
2113 : #undef _
2114 : },
2115 : .format_buffer = format_ethernet_header_with_length,
2116 : .format_trace = format_ethernet_input_trace,
2117 : .unformat_buffer = unformat_ethernet_header,
2118 : };
2119 :
2120 178120 : VLIB_REGISTER_NODE (ethernet_input_type_node) = {
2121 : .name = "ethernet-input-type",
2122 : /* Takes a vector of packets. */
2123 : .vector_size = sizeof (u32),
2124 : .n_next_nodes = ETHERNET_INPUT_N_NEXT,
2125 : .next_nodes = {
2126 : #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
2127 : foreach_ethernet_input_next
2128 : #undef _
2129 : },
2130 : };
2131 :
2132 178120 : VLIB_REGISTER_NODE (ethernet_input_not_l2_node) = {
2133 : .name = "ethernet-input-not-l2",
2134 : /* Takes a vector of packets. */
2135 : .vector_size = sizeof (u32),
2136 : .n_next_nodes = ETHERNET_INPUT_N_NEXT,
2137 : .next_nodes = {
2138 : #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
2139 : foreach_ethernet_input_next
2140 : #undef _
2141 : },
2142 : };
2143 : /* *INDENT-ON* */
2144 :
2145 : #ifndef CLIB_MARCH_VARIANT
2146 : void
2147 0 : ethernet_set_rx_redirect (vnet_main_t * vnm,
2148 : vnet_hw_interface_t * hi, u32 enable)
2149 : {
2150 : // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets
2151 : // don't go directly to ip4-input)
2152 0 : vnet_hw_interface_rx_redirect_to_node
2153 : (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0);
2154 0 : }
2155 :
2156 :
2157 : /*
2158 : * Initialization and registration for the next_by_ethernet structure
2159 : */
2160 :
2161 : clib_error_t *
2162 1118 : next_by_ethertype_init (next_by_ethertype_t * l3_next)
2163 : {
2164 1118 : l3_next->input_next_by_type = sparse_vec_new
2165 : ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]),
2166 : /* bits in index */ BITS (((ethernet_header_t *) 0)->type));
2167 :
2168 1118 : vec_validate (l3_next->sparse_index_by_input_next_index,
2169 : ETHERNET_INPUT_NEXT_DROP);
2170 1118 : vec_validate (l3_next->sparse_index_by_input_next_index,
2171 : ETHERNET_INPUT_NEXT_PUNT);
2172 1118 : l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] =
2173 : SPARSE_VEC_INVALID_INDEX;
2174 1118 : l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] =
2175 : SPARSE_VEC_INVALID_INDEX;
2176 :
2177 : /*
2178 : * Make sure we don't wipe out an ethernet registration by mistake
2179 : * Can happen if init function ordering constraints are missing.
2180 : */
2181 : if (CLIB_DEBUG > 0)
2182 : {
2183 1118 : ethernet_main_t *em = ðernet_main;
2184 1118 : ASSERT (em->next_by_ethertype_register_called == 0);
2185 : }
2186 :
2187 1118 : return 0;
2188 : }
2189 :
2190 : // Add an ethertype -> next index mapping to the structure
2191 : clib_error_t *
2192 11751 : next_by_ethertype_register (next_by_ethertype_t * l3_next,
2193 : u32 ethertype, u32 next_index)
2194 : {
2195 : u32 i;
2196 : u16 *n;
2197 11751 : ethernet_main_t *em = ðernet_main;
2198 :
2199 : if (CLIB_DEBUG > 0)
2200 : {
2201 11751 : ethernet_main_t *em = ðernet_main;
2202 11751 : em->next_by_ethertype_register_called = 1;
2203 : }
2204 :
2205 : /* Setup ethernet type -> next index sparse vector mapping. */
2206 11751 : n = sparse_vec_validate (l3_next->input_next_by_type, ethertype);
2207 11751 : n[0] = next_index;
2208 :
2209 : /* Rebuild next index -> sparse index inverse mapping when sparse vector
2210 : is updated. */
2211 11751 : vec_validate (l3_next->sparse_index_by_input_next_index, next_index);
2212 58803 : for (i = 1; i < vec_len (l3_next->input_next_by_type); i++)
2213 47052 : l3_next->
2214 47052 : sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i;
2215 :
2216 : // do not allow the cached next index's to be updated if L3
2217 : // redirect is enabled, as it will have overwritten them
2218 11751 : if (!em->redirect_l3)
2219 : {
2220 : // Cache common ethertypes directly
2221 11751 : if (ethertype == ETHERNET_TYPE_IP4)
2222 : {
2223 1677 : l3_next->input_next_ip4 = next_index;
2224 : }
2225 10074 : else if (ethertype == ETHERNET_TYPE_IP6)
2226 : {
2227 1677 : l3_next->input_next_ip6 = next_index;
2228 : }
2229 8397 : else if (ethertype == ETHERNET_TYPE_MPLS)
2230 : {
2231 1677 : l3_next->input_next_mpls = next_index;
2232 : }
2233 : }
2234 11751 : return 0;
2235 : }
2236 :
2237 : void
2238 8472 : ethernet_setup_node (vlib_main_t *vm, u32 node_index)
2239 : {
2240 8472 : vlib_node_t *n = vlib_get_node (vm, node_index);
2241 8472 : pg_node_t *pn = pg_get_node (node_index);
2242 :
2243 8472 : n->format_buffer = format_ethernet_header_with_length;
2244 8472 : n->unformat_buffer = unformat_ethernet_header;
2245 8472 : pn->unformat_edit = unformat_pg_ethernet_header;
2246 8472 : }
2247 :
2248 : void
2249 559 : ethernet_input_init (vlib_main_t * vm, ethernet_main_t * em)
2250 : {
2251 : __attribute__ ((unused)) vlan_table_t *invalid_vlan_table;
2252 : __attribute__ ((unused)) qinq_table_t *invalid_qinq_table;
2253 :
2254 559 : ethernet_setup_node (vm, ethernet_input_node.index);
2255 559 : ethernet_setup_node (vm, ethernet_input_type_node.index);
2256 559 : ethernet_setup_node (vm, ethernet_input_not_l2_node.index);
2257 :
2258 559 : next_by_ethertype_init (&em->l3_next);
2259 :
2260 : // Initialize pools and vector for vlan parsing
2261 559 : vec_validate (em->main_intfs, 10); // 10 main interfaces
2262 559 : pool_alloc (em->vlan_pool, 10);
2263 559 : pool_alloc (em->qinq_pool, 1);
2264 :
2265 : // The first vlan pool will always be reserved for an invalid table
2266 559 : pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0
2267 : // The first qinq pool will always be reserved for an invalid table
2268 559 : pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0
2269 559 : }
2270 :
2271 : void
2272 3917 : ethernet_register_input_type (vlib_main_t * vm,
2273 : ethernet_type_t type, u32 node_index)
2274 : {
2275 3917 : ethernet_main_t *em = ðernet_main;
2276 : ethernet_type_info_t *ti;
2277 : u32 i;
2278 :
2279 : {
2280 3917 : clib_error_t *error = vlib_call_init_function (vm, ethernet_init);
2281 3917 : if (error)
2282 0 : clib_error_report (error);
2283 : }
2284 :
2285 3917 : ti = ethernet_get_type_info (em, type);
2286 3917 : if (ti == 0)
2287 : {
2288 0 : clib_warning ("type_info NULL for type %d", type);
2289 0 : return;
2290 : }
2291 3917 : ti->node_index = node_index;
2292 7834 : ti->next_index = vlib_node_add_next (vm,
2293 3917 : ethernet_input_node.index, node_index);
2294 3917 : i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2295 3917 : ASSERT (i == ti->next_index);
2296 :
2297 3917 : i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2298 3917 : ASSERT (i == ti->next_index);
2299 :
2300 : // Add the L3 node for this ethertype to the next nodes structure
2301 3917 : next_by_ethertype_register (&em->l3_next, type, ti->next_index);
2302 :
2303 : // Call the registration functions for other nodes that want a mapping
2304 3917 : l2bvi_register_input_type (vm, type, node_index);
2305 : }
2306 :
2307 : void
2308 559 : ethernet_register_l2_input (vlib_main_t * vm, u32 node_index)
2309 : {
2310 559 : ethernet_main_t *em = ðernet_main;
2311 : u32 i;
2312 :
2313 559 : em->l2_next =
2314 559 : vlib_node_add_next (vm, ethernet_input_node.index, node_index);
2315 :
2316 : /*
2317 : * Even if we never use these arcs, we have to align the next indices...
2318 : */
2319 559 : i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2320 :
2321 559 : ASSERT (i == em->l2_next);
2322 :
2323 559 : i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2324 559 : ASSERT (i == em->l2_next);
2325 559 : }
2326 :
2327 : // Register a next node for L3 redirect, and enable L3 redirect
2328 : void
2329 0 : ethernet_register_l3_redirect (vlib_main_t * vm, u32 node_index)
2330 : {
2331 0 : ethernet_main_t *em = ðernet_main;
2332 : u32 i;
2333 :
2334 0 : em->redirect_l3 = 1;
2335 0 : em->redirect_l3_next = vlib_node_add_next (vm,
2336 0 : ethernet_input_node.index,
2337 : node_index);
2338 : /*
2339 : * Change the cached next nodes to the redirect node
2340 : */
2341 0 : em->l3_next.input_next_ip4 = em->redirect_l3_next;
2342 0 : em->l3_next.input_next_ip6 = em->redirect_l3_next;
2343 0 : em->l3_next.input_next_mpls = em->redirect_l3_next;
2344 :
2345 : /*
2346 : * Even if we never use these arcs, we have to align the next indices...
2347 : */
2348 0 : i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2349 :
2350 0 : ASSERT (i == em->redirect_l3_next);
2351 :
2352 0 : i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2353 :
2354 0 : ASSERT (i == em->redirect_l3_next);
2355 0 : }
2356 : #endif
2357 :
2358 : /*
2359 : * fd.io coding-style-patch-verification: ON
2360 : *
2361 : * Local Variables:
2362 : * eval: (c-set-style "gnu")
2363 : * End:
2364 : */
|
