Line data Source code
1 : /*
2 : * Copyright (c) 2017 SUSE LLC.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #include <vlib/vlib.h>
17 :
18 : #include <geneve/geneve.h>
19 :
20 : typedef struct
21 : {
22 : u32 next_index;
23 : u32 tunnel_index;
24 : u32 error;
25 : u32 vni_rsvd;
26 : } geneve_rx_trace_t;
27 :
28 : static u8 *
29 25 : format_geneve_rx_trace (u8 * s, va_list * args)
30 : {
31 25 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
32 25 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
33 25 : geneve_rx_trace_t *t = va_arg (*args, geneve_rx_trace_t *);
34 :
35 25 : if (t->tunnel_index != ~0)
36 : {
37 : s =
38 15 : format (s,
39 : "GENEVE decap from geneve_tunnel%d vni %d next %d error %d",
40 : t->tunnel_index, t->vni_rsvd, t->next_index, t->error);
41 : }
42 : else
43 : {
44 10 : s = format (s, "GENEVE decap error - tunnel for vni %d does not exist",
45 : t->vni_rsvd);
46 : }
47 25 : return s;
48 : }
49 :
50 : always_inline u32
51 14 : validate_geneve_fib (vlib_buffer_t * b, geneve_tunnel_t * t, u32 is_ip4)
52 : {
53 : u32 fib_index, sw_if_index;
54 :
55 14 : sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
56 :
57 14 : if (is_ip4)
58 14 : fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
59 14 : vec_elt (ip4_main.fib_index_by_sw_if_index, sw_if_index) :
60 0 : vnet_buffer (b)->sw_if_index[VLIB_TX];
61 : else
62 0 : fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
63 0 : vec_elt (ip6_main.fib_index_by_sw_if_index, sw_if_index) :
64 0 : vnet_buffer (b)->sw_if_index[VLIB_TX];
65 :
66 14 : return (fib_index == t->encap_fib_index);
67 : }
68 :
69 : always_inline uword
70 5 : geneve_input (vlib_main_t * vm,
71 : vlib_node_runtime_t * node,
72 : vlib_frame_t * from_frame, u32 is_ip4)
73 : {
74 : u32 n_left_from, next_index, *from, *to_next;
75 5 : geneve_main_t *vxm = &geneve_main;
76 5 : vnet_main_t *vnm = vxm->vnet_main;
77 5 : vnet_interface_main_t *im = &vnm->interface_main;
78 5 : u32 last_tunnel_index = ~0;
79 : geneve4_tunnel_key_t last_key4;
80 : geneve6_tunnel_key_t last_key6;
81 5 : u32 pkts_decapsulated = 0;
82 5 : u32 thread_index = vm->thread_index;
83 : u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
84 5 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
85 :
86 5 : if (is_ip4)
87 5 : last_key4.as_u64 = ~0;
88 : else
89 0 : clib_memset (&last_key6, 0xff, sizeof (last_key6));
90 :
91 5 : from = vlib_frame_vector_args (from_frame);
92 5 : n_left_from = from_frame->n_vectors;
93 5 : vlib_get_buffers (vm, from, bufs, n_left_from);
94 :
95 5 : next_index = node->cached_next_index;
96 5 : stats_sw_if_index = node->runtime_data[0];
97 5 : stats_n_packets = stats_n_bytes = 0;
98 :
99 10 : while (n_left_from > 0)
100 : {
101 : u32 n_left_to_next;
102 :
103 5 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
104 14 : while (n_left_from >= 4 && n_left_to_next >= 2)
105 : {
106 : u32 bi0, bi1;
107 : vlib_buffer_t *b0, *b1;
108 : u32 next0, next1;
109 : ip4_header_t *ip4_0, *ip4_1;
110 : ip6_header_t *ip6_0, *ip6_1;
111 : geneve_header_t *geneve0, *geneve1;
112 : uword *p0, *p1;
113 : u32 tunnel_index0, tunnel_index1;
114 9 : geneve_tunnel_t *t0, *t1, *mt0 = NULL, *mt1 = NULL;
115 : geneve4_tunnel_key_t key4_0, key4_1;
116 : geneve6_tunnel_key_t key6_0, key6_1;
117 : u32 error0, error1;
118 : u32 sw_if_index0, sw_if_index1, len0, len1;
119 :
120 : /* Prefetch next iteration. */
121 : {
122 9 : vlib_prefetch_buffer_header (b[2], LOAD);
123 9 : vlib_prefetch_buffer_header (b[3], LOAD);
124 :
125 9 : CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
126 9 : CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
127 : }
128 :
129 9 : bi0 = from[0];
130 9 : bi1 = from[1];
131 9 : to_next[0] = bi0;
132 9 : to_next[1] = bi1;
133 9 : from += 2;
134 9 : to_next += 2;
135 9 : n_left_to_next -= 2;
136 9 : n_left_from -= 2;
137 :
138 9 : b0 = b[0];
139 9 : b1 = b[1];
140 9 : b += 2;
141 :
142 : /* udp leaves current_data pointing at the geneve header */
143 9 : geneve0 = vlib_buffer_get_current (b0);
144 9 : geneve1 = vlib_buffer_get_current (b1);
145 :
146 9 : vnet_geneve_hdr_1word_ntoh (geneve0);
147 9 : vnet_geneve_hdr_1word_ntoh (geneve1);
148 :
149 9 : if (is_ip4)
150 : {
151 9 : vlib_buffer_advance
152 : (b0, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
153 9 : vlib_buffer_advance
154 : (b1, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
155 9 : ip4_0 = vlib_buffer_get_current (b0);
156 9 : ip4_1 = vlib_buffer_get_current (b1);
157 : }
158 : else
159 : {
160 0 : vlib_buffer_advance
161 : (b0, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
162 0 : vlib_buffer_advance
163 : (b1, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
164 0 : ip6_0 = vlib_buffer_get_current (b0);
165 0 : ip6_1 = vlib_buffer_get_current (b1);
166 : }
167 :
168 : /* pop (ip, udp, geneve) */
169 9 : if (is_ip4)
170 : {
171 9 : vlib_buffer_advance (b0,
172 : sizeof (*ip4_0) + sizeof (udp_header_t) +
173 9 : GENEVE_BASE_HEADER_LENGTH +
174 9 : vnet_get_geneve_options_len (geneve0));
175 9 : vlib_buffer_advance (b1,
176 : sizeof (*ip4_1) + sizeof (udp_header_t) +
177 9 : GENEVE_BASE_HEADER_LENGTH +
178 9 : vnet_get_geneve_options_len (geneve1));
179 : }
180 : else
181 : {
182 0 : vlib_buffer_advance (b0,
183 : sizeof (*ip6_0) + sizeof (udp_header_t) +
184 0 : GENEVE_BASE_HEADER_LENGTH +
185 0 : vnet_get_geneve_options_len (geneve0));
186 0 : vlib_buffer_advance (b1,
187 : sizeof (*ip6_1) + sizeof (udp_header_t) +
188 0 : GENEVE_BASE_HEADER_LENGTH +
189 0 : vnet_get_geneve_options_len (geneve1));
190 : }
191 :
192 9 : tunnel_index0 = ~0;
193 9 : error0 = 0;
194 :
195 9 : tunnel_index1 = ~0;
196 9 : error1 = 0;
197 :
198 9 : if (PREDICT_FALSE
199 : (vnet_get_geneve_version (geneve0) != GENEVE_VERSION))
200 : {
201 0 : error0 = GENEVE_ERROR_BAD_FLAGS;
202 0 : next0 = GENEVE_INPUT_NEXT_DROP;
203 0 : goto trace0;
204 : }
205 : #if SUPPORT_OPTIONS_HEADER==1
206 : if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve0) == 1))
207 : {
208 : error0 = GENEVE_ERROR_BAD_FLAGS;
209 : next0 = GENEVE_INPUT_NEXT_DROP;
210 : goto trace0;
211 : }
212 : #endif
213 9 : if (is_ip4)
214 : {
215 9 : key4_0.remote = ip4_0->src_address.as_u32;
216 9 : key4_0.vni = vnet_get_geneve_vni_network_order (geneve0);
217 :
218 : /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
219 9 : if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
220 : {
221 9 : p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
222 9 : if (PREDICT_FALSE (p0 == NULL))
223 : {
224 4 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
225 4 : next0 = GENEVE_INPUT_NEXT_DROP;
226 4 : goto trace0;
227 : }
228 5 : last_key4.as_u64 = key4_0.as_u64;
229 5 : tunnel_index0 = last_tunnel_index = p0[0];
230 : }
231 : else
232 0 : tunnel_index0 = last_tunnel_index;
233 5 : t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
234 :
235 : /* Validate GENEVE tunnel encap-fib index against packet */
236 5 : if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
237 : {
238 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
239 0 : next0 = GENEVE_INPUT_NEXT_DROP;
240 0 : goto trace0;
241 : }
242 :
243 : /* Validate GENEVE tunnel SIP against packet DIP */
244 5 : if (PREDICT_TRUE
245 : (ip4_0->dst_address.as_u32 == t0->local.ip4.as_u32))
246 0 : goto next0; /* valid packet */
247 5 : if (PREDICT_FALSE
248 : (ip4_address_is_multicast (&ip4_0->dst_address)))
249 : {
250 5 : key4_0.remote = ip4_0->dst_address.as_u32;
251 5 : key4_0.vni = vnet_get_geneve_vni_network_order (geneve0);
252 : /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
253 5 : p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
254 5 : if (PREDICT_TRUE (p0 != NULL))
255 : {
256 5 : mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
257 5 : goto next0; /* valid packet */
258 : }
259 : }
260 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
261 0 : next0 = GENEVE_INPUT_NEXT_DROP;
262 0 : goto trace0;
263 :
264 : }
265 : else /* !is_ip4 */
266 : {
267 0 : key6_0.remote.as_u64[0] = ip6_0->src_address.as_u64[0];
268 0 : key6_0.remote.as_u64[1] = ip6_0->src_address.as_u64[1];
269 0 : key6_0.vni = vnet_get_geneve_vni_network_order (geneve0);
270 :
271 : /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
272 0 : if (PREDICT_FALSE
273 : (memcmp (&key6_0, &last_key6, sizeof (last_key6)) != 0))
274 : {
275 0 : p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
276 0 : if (PREDICT_FALSE (p0 == NULL))
277 : {
278 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
279 0 : next0 = GENEVE_INPUT_NEXT_DROP;
280 0 : goto trace0;
281 : }
282 0 : clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0));
283 0 : tunnel_index0 = last_tunnel_index = p0[0];
284 : }
285 : else
286 0 : tunnel_index0 = last_tunnel_index;
287 0 : t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
288 :
289 : /* Validate GENEVE tunnel encap-fib index against packet */
290 0 : if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
291 : {
292 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
293 0 : next0 = GENEVE_INPUT_NEXT_DROP;
294 0 : goto trace0;
295 : }
296 :
297 : /* Validate GENEVE tunnel SIP against packet DIP */
298 0 : if (PREDICT_TRUE (ip6_address_is_equal (&ip6_0->dst_address,
299 : &t0->local.ip6)))
300 0 : goto next0; /* valid packet */
301 0 : if (PREDICT_FALSE
302 : (ip6_address_is_multicast (&ip6_0->dst_address)))
303 : {
304 0 : key6_0.remote.as_u64[0] = ip6_0->dst_address.as_u64[0];
305 0 : key6_0.remote.as_u64[1] = ip6_0->dst_address.as_u64[1];
306 0 : key6_0.vni = vnet_get_geneve_vni_network_order (geneve0);
307 0 : p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
308 0 : if (PREDICT_TRUE (p0 != NULL))
309 : {
310 0 : mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
311 0 : goto next0; /* valid packet */
312 : }
313 : }
314 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
315 0 : next0 = GENEVE_INPUT_NEXT_DROP;
316 0 : goto trace0;
317 : }
318 :
319 5 : next0:
320 5 : next0 = t0->decap_next_index;
321 5 : sw_if_index0 = t0->sw_if_index;
322 5 : len0 = vlib_buffer_length_in_chain (vm, b0);
323 :
324 : /* Required to make the l2 tag push / pop code work on l2 subifs */
325 5 : if (PREDICT_TRUE (next0 == GENEVE_INPUT_NEXT_L2_INPUT))
326 5 : vnet_update_l2_len (b0);
327 :
328 : /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
329 5 : vnet_buffer (b0)->sw_if_index[VLIB_RX] = sw_if_index0;
330 5 : sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
331 :
332 5 : pkts_decapsulated++;
333 5 : stats_n_packets += 1;
334 5 : stats_n_bytes += len0;
335 :
336 : /* Batch stats increment on the same geneve tunnel so counter
337 : is not incremented per packet */
338 5 : if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
339 : {
340 1 : stats_n_packets -= 1;
341 1 : stats_n_bytes -= len0;
342 1 : if (stats_n_packets)
343 0 : vlib_increment_combined_counter
344 : (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
345 : thread_index, stats_sw_if_index,
346 : stats_n_packets, stats_n_bytes);
347 1 : stats_n_packets = 1;
348 1 : stats_n_bytes = len0;
349 1 : stats_sw_if_index = sw_if_index0;
350 : }
351 :
352 4 : trace0:
353 9 : b0->error = error0 ? node->errors[error0] : 0;
354 :
355 9 : if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
356 : {
357 : geneve_rx_trace_t *tr
358 9 : = vlib_add_trace (vm, node, b0, sizeof (*tr));
359 9 : tr->next_index = next0;
360 9 : tr->error = error0;
361 9 : tr->tunnel_index = tunnel_index0;
362 9 : tr->vni_rsvd = vnet_get_geneve_vni (geneve0);
363 : }
364 :
365 9 : if (PREDICT_FALSE
366 : (vnet_get_geneve_version (geneve1) != GENEVE_VERSION))
367 : {
368 0 : error1 = GENEVE_ERROR_BAD_FLAGS;
369 0 : next1 = GENEVE_INPUT_NEXT_DROP;
370 0 : goto trace1;
371 : }
372 : #if SUPPORT_OPTIONS_HEADER==1
373 : if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve1) == 1))
374 : {
375 : error1 = GENEVE_ERROR_BAD_FLAGS;
376 : next1 = GENEVE_INPUT_NEXT_DROP;
377 : goto trace1;
378 : }
379 : #endif
380 9 : if (is_ip4)
381 : {
382 9 : key4_1.remote = ip4_1->src_address.as_u32;
383 9 : key4_1.vni = vnet_get_geneve_vni_network_order (geneve1);
384 :
385 : /* Make sure unicast GENEVE tunnel exist by packet SIP and VNI */
386 9 : if (PREDICT_FALSE (key4_1.as_u64 != last_key4.as_u64))
387 : {
388 9 : p1 = hash_get (vxm->geneve4_tunnel_by_key, key4_1.as_u64);
389 9 : if (PREDICT_FALSE (p1 == NULL))
390 : {
391 4 : error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
392 4 : next1 = GENEVE_INPUT_NEXT_DROP;
393 4 : goto trace1;
394 : }
395 5 : last_key4.as_u64 = key4_1.as_u64;
396 5 : tunnel_index1 = last_tunnel_index = p1[0];
397 : }
398 : else
399 0 : tunnel_index1 = last_tunnel_index;
400 5 : t1 = pool_elt_at_index (vxm->tunnels, tunnel_index1);
401 :
402 : /* Validate GENEVE tunnel encap-fib index against packet */
403 5 : if (PREDICT_FALSE (validate_geneve_fib (b1, t1, is_ip4) == 0))
404 : {
405 0 : error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
406 0 : next1 = GENEVE_INPUT_NEXT_DROP;
407 0 : goto trace1;
408 : }
409 :
410 : /* Validate GENEVE tunnel SIP against packet DIP */
411 5 : if (PREDICT_TRUE
412 : (ip4_1->dst_address.as_u32 == t1->local.ip4.as_u32))
413 0 : goto next1; /* valid packet */
414 5 : if (PREDICT_FALSE
415 : (ip4_address_is_multicast (&ip4_1->dst_address)))
416 : {
417 5 : key4_1.remote = ip4_1->dst_address.as_u32;
418 5 : key4_1.vni = vnet_get_geneve_vni_network_order (geneve1);
419 : /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
420 5 : p1 = hash_get (vxm->geneve4_tunnel_by_key, key4_1.as_u64);
421 5 : if (PREDICT_TRUE (p1 != NULL))
422 : {
423 5 : mt1 = pool_elt_at_index (vxm->tunnels, p1[0]);
424 5 : goto next1; /* valid packet */
425 : }
426 : }
427 0 : error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
428 0 : next1 = GENEVE_INPUT_NEXT_DROP;
429 0 : goto trace1;
430 :
431 : }
432 : else /* !is_ip4 */
433 : {
434 0 : key6_1.remote.as_u64[0] = ip6_1->src_address.as_u64[0];
435 0 : key6_1.remote.as_u64[1] = ip6_1->src_address.as_u64[1];
436 0 : key6_1.vni = vnet_get_geneve_vni_network_order (geneve1);
437 :
438 : /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
439 0 : if (PREDICT_FALSE
440 : (memcmp (&key6_1, &last_key6, sizeof (last_key6)) != 0))
441 : {
442 0 : p1 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_1);
443 :
444 0 : if (PREDICT_FALSE (p1 == NULL))
445 : {
446 0 : error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
447 0 : next1 = GENEVE_INPUT_NEXT_DROP;
448 0 : goto trace1;
449 : }
450 :
451 0 : clib_memcpy_fast (&last_key6, &key6_1, sizeof (key6_1));
452 0 : tunnel_index1 = last_tunnel_index = p1[0];
453 : }
454 : else
455 0 : tunnel_index1 = last_tunnel_index;
456 0 : t1 = pool_elt_at_index (vxm->tunnels, tunnel_index1);
457 :
458 : /* Validate GENEVE tunnel encap-fib index against packet */
459 0 : if (PREDICT_FALSE (validate_geneve_fib (b1, t1, is_ip4) == 0))
460 : {
461 0 : error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
462 0 : next1 = GENEVE_INPUT_NEXT_DROP;
463 0 : goto trace1;
464 : }
465 :
466 : /* Validate GENEVE tunnel SIP against packet DIP */
467 0 : if (PREDICT_TRUE (ip6_address_is_equal (&ip6_1->dst_address,
468 : &t1->local.ip6)))
469 0 : goto next1; /* valid packet */
470 0 : if (PREDICT_FALSE
471 : (ip6_address_is_multicast (&ip6_1->dst_address)))
472 : {
473 0 : key6_1.remote.as_u64[0] = ip6_1->dst_address.as_u64[0];
474 0 : key6_1.remote.as_u64[1] = ip6_1->dst_address.as_u64[1];
475 0 : key6_1.vni = vnet_get_geneve_vni_network_order (geneve1);
476 0 : p1 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_1);
477 0 : if (PREDICT_TRUE (p1 != NULL))
478 : {
479 0 : mt1 = pool_elt_at_index (vxm->tunnels, p1[0]);
480 0 : goto next1; /* valid packet */
481 : }
482 : }
483 0 : error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
484 0 : next1 = GENEVE_INPUT_NEXT_DROP;
485 0 : goto trace1;
486 : }
487 :
488 5 : next1:
489 5 : next1 = t1->decap_next_index;
490 5 : sw_if_index1 = t1->sw_if_index;
491 5 : len1 = vlib_buffer_length_in_chain (vm, b1);
492 :
493 : /* Required to make the l2 tag push / pop code work on l2 subifs */
494 5 : if (PREDICT_TRUE (next1 == GENEVE_INPUT_NEXT_L2_INPUT))
495 5 : vnet_update_l2_len (b1);
496 :
497 : /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
498 5 : vnet_buffer (b1)->sw_if_index[VLIB_RX] = sw_if_index1;
499 5 : sw_if_index1 = (mt1) ? mt1->sw_if_index : sw_if_index1;
500 :
501 5 : pkts_decapsulated++;
502 5 : stats_n_packets += 1;
503 5 : stats_n_bytes += len1;
504 :
505 : /* Batch stats increment on the same geneve tunnel so counter
506 : is not incremented per packet */
507 5 : if (PREDICT_FALSE (sw_if_index1 != stats_sw_if_index))
508 : {
509 0 : stats_n_packets -= 1;
510 0 : stats_n_bytes -= len1;
511 0 : if (stats_n_packets)
512 0 : vlib_increment_combined_counter
513 : (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
514 : thread_index, stats_sw_if_index,
515 : stats_n_packets, stats_n_bytes);
516 0 : stats_n_packets = 1;
517 0 : stats_n_bytes = len1;
518 0 : stats_sw_if_index = sw_if_index1;
519 : }
520 :
521 5 : trace1:
522 9 : b1->error = error1 ? node->errors[error1] : 0;
523 :
524 9 : if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
525 : {
526 : geneve_rx_trace_t *tr
527 9 : = vlib_add_trace (vm, node, b1, sizeof (*tr));
528 9 : tr->next_index = next1;
529 9 : tr->error = error1;
530 9 : tr->tunnel_index = tunnel_index1;
531 9 : tr->vni_rsvd = vnet_get_geneve_vni (geneve1);
532 : }
533 :
534 9 : vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
535 : to_next, n_left_to_next,
536 : bi0, bi1, next0, next1);
537 : }
538 :
539 11 : while (n_left_from > 0 && n_left_to_next > 0)
540 : {
541 : u32 bi0;
542 : vlib_buffer_t *b0;
543 : u32 next0;
544 : ip4_header_t *ip4_0;
545 : ip6_header_t *ip6_0;
546 : geneve_header_t *geneve0;
547 : uword *p0;
548 : u32 tunnel_index0;
549 6 : geneve_tunnel_t *t0, *mt0 = NULL;
550 : geneve4_tunnel_key_t key4_0;
551 : geneve6_tunnel_key_t key6_0;
552 : u32 error0;
553 : u32 sw_if_index0, len0;
554 :
555 6 : bi0 = from[0];
556 6 : to_next[0] = bi0;
557 6 : from += 1;
558 6 : to_next += 1;
559 6 : n_left_from -= 1;
560 6 : n_left_to_next -= 1;
561 :
562 6 : b0 = b[0];
563 6 : b += 1;
564 :
565 : /* udp leaves current_data pointing at the geneve header */
566 6 : geneve0 = vlib_buffer_get_current (b0);
567 6 : vnet_geneve_hdr_1word_ntoh (geneve0);
568 :
569 6 : if (is_ip4)
570 : {
571 6 : vlib_buffer_advance
572 : (b0, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
573 6 : ip4_0 = vlib_buffer_get_current (b0);
574 : }
575 : else
576 : {
577 0 : vlib_buffer_advance
578 : (b0, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
579 0 : ip6_0 = vlib_buffer_get_current (b0);
580 : }
581 :
582 : /* pop (ip, udp, geneve) */
583 6 : if (is_ip4)
584 : {
585 6 : vlib_buffer_advance
586 : (b0,
587 : sizeof (*ip4_0) + sizeof (udp_header_t) +
588 6 : GENEVE_BASE_HEADER_LENGTH +
589 6 : vnet_get_geneve_options_len (geneve0));
590 : }
591 : else
592 : {
593 0 : vlib_buffer_advance
594 : (b0,
595 : sizeof (*ip6_0) + sizeof (udp_header_t) +
596 0 : GENEVE_BASE_HEADER_LENGTH +
597 0 : vnet_get_geneve_options_len (geneve0));
598 : }
599 :
600 6 : tunnel_index0 = ~0;
601 6 : error0 = 0;
602 :
603 6 : if (PREDICT_FALSE
604 : (vnet_get_geneve_version (geneve0) != GENEVE_VERSION))
605 : {
606 0 : error0 = GENEVE_ERROR_BAD_FLAGS;
607 0 : next0 = GENEVE_INPUT_NEXT_DROP;
608 0 : goto trace00;
609 : }
610 : #if SUPPORT_OPTIONS_HEADER==1
611 : if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve0) == 1))
612 : {
613 : error0 = GENEVE_ERROR_BAD_FLAGS;
614 : next0 = GENEVE_INPUT_NEXT_DROP;
615 : goto trace00;
616 : }
617 : #endif
618 6 : if (is_ip4)
619 : {
620 6 : key4_0.remote = ip4_0->src_address.as_u32;
621 6 : key4_0.vni = vnet_get_geneve_vni_network_order (geneve0);
622 :
623 : /* Make sure unicast GENEVE tunnel exist by packet SIP and VNI */
624 6 : if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
625 : {
626 6 : p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
627 6 : if (PREDICT_FALSE (p0 == NULL))
628 : {
629 2 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
630 2 : next0 = GENEVE_INPUT_NEXT_DROP;
631 2 : goto trace00;
632 : }
633 4 : last_key4.as_u64 = key4_0.as_u64;
634 4 : tunnel_index0 = last_tunnel_index = p0[0];
635 : }
636 : else
637 0 : tunnel_index0 = last_tunnel_index;
638 4 : t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
639 :
640 : /* Validate GENEVE tunnel encap-fib index agaist packet */
641 4 : if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
642 : {
643 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
644 0 : next0 = GENEVE_INPUT_NEXT_DROP;
645 0 : goto trace00;
646 : }
647 :
648 : /* Validate GENEVE tunnel SIP against packet DIP */
649 4 : if (PREDICT_TRUE
650 : (ip4_0->dst_address.as_u32 == t0->local.ip4.as_u32))
651 4 : goto next00; /* valid packet */
652 0 : if (PREDICT_FALSE
653 : (ip4_address_is_multicast (&ip4_0->dst_address)))
654 : {
655 0 : key4_0.remote = ip4_0->dst_address.as_u32;
656 0 : key4_0.vni = vnet_get_geneve_vni_network_order (geneve0);
657 : /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
658 0 : p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
659 0 : if (PREDICT_TRUE (p0 != NULL))
660 : {
661 0 : mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
662 0 : goto next00; /* valid packet */
663 : }
664 : }
665 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
666 0 : next0 = GENEVE_INPUT_NEXT_DROP;
667 0 : goto trace00;
668 :
669 : }
670 : else /* !is_ip4 */
671 : {
672 0 : key6_0.remote.as_u64[0] = ip6_0->src_address.as_u64[0];
673 0 : key6_0.remote.as_u64[1] = ip6_0->src_address.as_u64[1];
674 0 : key6_0.vni = vnet_get_geneve_vni_network_order (geneve0);
675 :
676 : /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
677 0 : if (PREDICT_FALSE
678 : (memcmp (&key6_0, &last_key6, sizeof (last_key6)) != 0))
679 : {
680 0 : p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
681 0 : if (PREDICT_FALSE (p0 == NULL))
682 : {
683 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
684 0 : next0 = GENEVE_INPUT_NEXT_DROP;
685 0 : goto trace00;
686 : }
687 0 : clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0));
688 0 : tunnel_index0 = last_tunnel_index = p0[0];
689 : }
690 : else
691 0 : tunnel_index0 = last_tunnel_index;
692 0 : t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
693 :
694 : /* Validate GENEVE tunnel encap-fib index agaist packet */
695 0 : if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
696 : {
697 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
698 0 : next0 = GENEVE_INPUT_NEXT_DROP;
699 0 : goto trace00;
700 : }
701 :
702 : /* Validate GENEVE tunnel SIP against packet DIP */
703 0 : if (PREDICT_TRUE (ip6_address_is_equal (&ip6_0->dst_address,
704 : &t0->local.ip6)))
705 0 : goto next00; /* valid packet */
706 0 : if (PREDICT_FALSE
707 : (ip6_address_is_multicast (&ip6_0->dst_address)))
708 : {
709 0 : key6_0.remote.as_u64[0] = ip6_0->dst_address.as_u64[0];
710 0 : key6_0.remote.as_u64[1] = ip6_0->dst_address.as_u64[1];
711 0 : key6_0.vni = vnet_get_geneve_vni_network_order (geneve0);
712 0 : p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
713 0 : if (PREDICT_TRUE (p0 != NULL))
714 : {
715 0 : mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
716 0 : goto next00; /* valid packet */
717 : }
718 : }
719 0 : error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
720 0 : next0 = GENEVE_INPUT_NEXT_DROP;
721 0 : goto trace00;
722 : }
723 :
724 4 : next00:
725 4 : next0 = t0->decap_next_index;
726 4 : sw_if_index0 = t0->sw_if_index;
727 4 : len0 = vlib_buffer_length_in_chain (vm, b0);
728 :
729 : /* Required to make the l2 tag push / pop code work on l2 subifs */
730 4 : if (PREDICT_TRUE (next0 == GENEVE_INPUT_NEXT_L2_INPUT))
731 1 : vnet_update_l2_len (b0);
732 :
733 : /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
734 4 : vnet_buffer (b0)->sw_if_index[VLIB_RX] = sw_if_index0;
735 4 : sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
736 :
737 4 : pkts_decapsulated++;
738 4 : stats_n_packets += 1;
739 4 : stats_n_bytes += len0;
740 :
741 : /* Batch stats increment on the same geneve tunnel so counter
742 : is not incremented per packet */
743 4 : if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
744 : {
745 2 : stats_n_packets -= 1;
746 2 : stats_n_bytes -= len0;
747 2 : if (stats_n_packets)
748 0 : vlib_increment_combined_counter
749 : (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
750 : thread_index, stats_sw_if_index,
751 : stats_n_packets, stats_n_bytes);
752 2 : stats_n_packets = 1;
753 2 : stats_n_bytes = len0;
754 2 : stats_sw_if_index = sw_if_index0;
755 : }
756 :
757 2 : trace00:
758 6 : b0->error = error0 ? node->errors[error0] : 0;
759 :
760 6 : if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
761 : {
762 : geneve_rx_trace_t *tr
763 6 : = vlib_add_trace (vm, node, b0, sizeof (*tr));
764 6 : tr->next_index = next0;
765 6 : tr->error = error0;
766 6 : tr->tunnel_index = tunnel_index0;
767 6 : tr->vni_rsvd = vnet_get_geneve_vni (geneve0);
768 : }
769 6 : vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
770 : to_next, n_left_to_next,
771 : bi0, next0);
772 : }
773 :
774 5 : vlib_put_next_frame (vm, node, next_index, n_left_to_next);
775 : }
776 : /* Do we still need this now that tunnel tx stats is kept? */
777 5 : vlib_node_increment_counter (vm, is_ip4 ?
778 : geneve4_input_node.
779 : index : geneve6_input_node.index,
780 : GENEVE_ERROR_DECAPSULATED, pkts_decapsulated);
781 :
782 : /* Increment any remaining batch stats */
783 5 : if (stats_n_packets)
784 : {
785 5 : vlib_increment_combined_counter
786 : (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
787 : thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
788 5 : node->runtime_data[0] = stats_sw_if_index;
789 : }
790 :
791 5 : return from_frame->n_vectors;
792 : }
793 :
794 2241 : VLIB_NODE_FN (geneve4_input_node) (vlib_main_t * vm,
795 : vlib_node_runtime_t * node,
796 : vlib_frame_t * from_frame)
797 : {
798 5 : return geneve_input (vm, node, from_frame, /* is_ip4 */ 1);
799 : }
800 :
801 2236 : VLIB_NODE_FN (geneve6_input_node) (vlib_main_t * vm,
802 : vlib_node_runtime_t * node,
803 : vlib_frame_t * from_frame)
804 : {
805 0 : return geneve_input (vm, node, from_frame, /* is_ip4 */ 0);
806 : }
807 :
808 : static char *geneve_error_strings[] = {
809 : #define geneve_error(n,s) s,
810 : #include <geneve/geneve_error.def>
811 : #undef geneve_error
812 : #undef _
813 : };
814 :
815 : /* *INDENT-OFF* */
816 136120 : VLIB_REGISTER_NODE (geneve4_input_node) = {
817 : .name = "geneve4-input",
818 : /* Takes a vector of packets. */
819 : .vector_size = sizeof (u32),
820 : .n_errors = GENEVE_N_ERROR,
821 : .error_strings = geneve_error_strings,
822 : .n_next_nodes = GENEVE_INPUT_N_NEXT,
823 : .next_nodes = {
824 : #define _(s,n) [GENEVE_INPUT_NEXT_##s] = n,
825 : foreach_geneve_input_next
826 : #undef _
827 : },
828 :
829 : //temp .format_buffer = format_geneve_header,
830 : .format_trace = format_geneve_rx_trace,
831 : // $$$$ .unformat_buffer = unformat_geneve_header,
832 : };
833 :
834 136120 : VLIB_REGISTER_NODE (geneve6_input_node) = {
835 : .name = "geneve6-input",
836 : /* Takes a vector of packets. */
837 : .vector_size = sizeof (u32),
838 : .n_errors = GENEVE_N_ERROR,
839 : .error_strings = geneve_error_strings,
840 : .n_next_nodes = GENEVE_INPUT_N_NEXT,
841 : .next_nodes = {
842 : #define _(s,n) [GENEVE_INPUT_NEXT_##s] = n,
843 : foreach_geneve_input_next
844 : #undef _
845 : },
846 : //temp .format_buffer = format_geneve_header,
847 : .format_trace = format_geneve_rx_trace,
848 : // $$$$ .unformat_buffer = unformat_geneve_header,
849 : };
850 : /* *INDENT-ON* */
851 :
852 : typedef enum
853 : {
854 : IP_GENEVE_BYPASS_NEXT_DROP,
855 : IP_GENEVE_BYPASS_NEXT_GENEVE,
856 : IP_GENEVE_BYPASS_N_NEXT,
857 : } ip_vxan_bypass_next_t;
858 :
859 : always_inline uword
860 0 : ip_geneve_bypass_inline (vlib_main_t * vm,
861 : vlib_node_runtime_t * node,
862 : vlib_frame_t * frame, u32 is_ip4)
863 : {
864 0 : geneve_main_t *vxm = &geneve_main;
865 : u32 *from, *to_next, n_left_from, n_left_to_next, next_index;
866 : vlib_node_runtime_t *error_node =
867 0 : vlib_node_get_runtime (vm, ip4_input_node.index);
868 : vtep4_key_t last_vtep4; /* last IPv4 address / fib index
869 : matching a local VTEP address */
870 : vtep6_key_t last_vtep6; /* last IPv6 address / fib index
871 : matching a local VTEP address */
872 0 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
873 :
874 0 : from = vlib_frame_vector_args (frame);
875 0 : n_left_from = frame->n_vectors;
876 0 : next_index = node->cached_next_index;
877 :
878 0 : vlib_get_buffers (vm, from, bufs, n_left_from);
879 :
880 0 : if (node->flags & VLIB_NODE_FLAG_TRACE)
881 0 : ip4_forward_next_trace (vm, node, frame, VLIB_TX);
882 :
883 0 : if (is_ip4)
884 0 : vtep4_key_init (&last_vtep4);
885 : else
886 0 : vtep6_key_init (&last_vtep6);
887 :
888 0 : while (n_left_from > 0)
889 : {
890 0 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
891 :
892 0 : while (n_left_from >= 4 && n_left_to_next >= 2)
893 : {
894 : vlib_buffer_t *b0, *b1;
895 : ip4_header_t *ip40, *ip41;
896 : ip6_header_t *ip60, *ip61;
897 : udp_header_t *udp0, *udp1;
898 : u32 bi0, ip_len0, udp_len0, flags0, next0;
899 : u32 bi1, ip_len1, udp_len1, flags1, next1;
900 : i32 len_diff0, len_diff1;
901 : u8 error0, good_udp0, proto0;
902 : u8 error1, good_udp1, proto1;
903 :
904 : /* Prefetch next iteration. */
905 : {
906 0 : vlib_prefetch_buffer_header (b[2], LOAD);
907 0 : vlib_prefetch_buffer_header (b[3], LOAD);
908 :
909 0 : CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
910 0 : CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
911 : }
912 :
913 0 : bi0 = to_next[0] = from[0];
914 0 : bi1 = to_next[1] = from[1];
915 0 : from += 2;
916 0 : n_left_from -= 2;
917 0 : to_next += 2;
918 0 : n_left_to_next -= 2;
919 :
920 0 : b0 = b[0];
921 0 : b1 = b[1];
922 0 : b += 2;
923 0 : if (is_ip4)
924 : {
925 0 : ip40 = vlib_buffer_get_current (b0);
926 0 : ip41 = vlib_buffer_get_current (b1);
927 : }
928 : else
929 : {
930 0 : ip60 = vlib_buffer_get_current (b0);
931 0 : ip61 = vlib_buffer_get_current (b1);
932 : }
933 :
934 : /* Setup packet for next IP feature */
935 0 : vnet_feature_next (&next0, b0);
936 0 : vnet_feature_next (&next1, b1);
937 :
938 0 : if (is_ip4)
939 : {
940 : /* Treat IP frag packets as "experimental" protocol for now
941 : until support of IP frag reassembly is implemented */
942 0 : proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol;
943 0 : proto1 = ip4_is_fragment (ip41) ? 0xfe : ip41->protocol;
944 : }
945 : else
946 : {
947 0 : proto0 = ip60->protocol;
948 0 : proto1 = ip61->protocol;
949 : }
950 :
951 : /* Process packet 0 */
952 0 : if (proto0 != IP_PROTOCOL_UDP)
953 0 : goto exit0; /* not UDP packet */
954 :
955 0 : if (is_ip4)
956 0 : udp0 = ip4_next_header (ip40);
957 : else
958 0 : udp0 = ip6_next_header (ip60);
959 :
960 0 : if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
961 0 : goto exit0; /* not GENEVE packet */
962 :
963 : /* Validate DIP against VTEPs */
964 0 : if (is_ip4)
965 : {
966 : #ifdef CLIB_HAVE_VEC512
967 0 : if (!vtep4_check_vector (&vxm->vtep_table, b0, ip40, &last_vtep4,
968 : &vxm->vtep4_u512))
969 : #else
970 0 : if (!vtep4_check (&vxm->vtep_table, b0, ip40, &last_vtep4))
971 : #endif
972 0 : goto exit0; /* no local VTEP for GENEVE packet */
973 : }
974 : else
975 : {
976 0 : if (!vtep6_check (&vxm->vtep_table, b0, ip60, &last_vtep6))
977 0 : goto exit0; /* no local VTEP for GENEVE packet */
978 : }
979 :
980 0 : flags0 = b0->flags;
981 0 : good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
982 :
983 : /* Don't verify UDP checksum for packets with explicit zero checksum. */
984 0 : good_udp0 |= udp0->checksum == 0;
985 :
986 : /* Verify UDP length */
987 0 : if (is_ip4)
988 0 : ip_len0 = clib_net_to_host_u16 (ip40->length);
989 : else
990 0 : ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
991 0 : udp_len0 = clib_net_to_host_u16 (udp0->length);
992 0 : len_diff0 = ip_len0 - udp_len0;
993 :
994 : /* Verify UDP checksum */
995 0 : if (PREDICT_FALSE (!good_udp0))
996 : {
997 0 : if ((flags0 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
998 : {
999 0 : if (is_ip4)
1000 0 : flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
1001 : else
1002 0 : flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
1003 0 : good_udp0 =
1004 0 : (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1005 : }
1006 : }
1007 :
1008 0 : if (is_ip4)
1009 : {
1010 0 : error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1011 0 : error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
1012 : }
1013 : else
1014 : {
1015 0 : error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1016 0 : error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
1017 : }
1018 :
1019 0 : next0 = error0 ?
1020 0 : IP_GENEVE_BYPASS_NEXT_DROP : IP_GENEVE_BYPASS_NEXT_GENEVE;
1021 0 : b0->error = error0 ? error_node->errors[error0] : 0;
1022 :
1023 : /* geneve-input node expect current at GENEVE header */
1024 0 : if (is_ip4)
1025 0 : vlib_buffer_advance (b0,
1026 : sizeof (ip4_header_t) +
1027 : sizeof (udp_header_t));
1028 : else
1029 0 : vlib_buffer_advance (b0,
1030 : sizeof (ip6_header_t) +
1031 : sizeof (udp_header_t));
1032 :
1033 0 : exit0:
1034 : /* Process packet 1 */
1035 0 : if (proto1 != IP_PROTOCOL_UDP)
1036 0 : goto exit1; /* not UDP packet */
1037 :
1038 0 : if (is_ip4)
1039 0 : udp1 = ip4_next_header (ip41);
1040 : else
1041 0 : udp1 = ip6_next_header (ip61);
1042 :
1043 0 : if (udp1->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
1044 0 : goto exit1; /* not GENEVE packet */
1045 :
1046 : /* Validate DIP against VTEPs */
1047 0 : if (is_ip4)
1048 : {
1049 : #ifdef CLIB_HAVE_VEC512
1050 0 : if (!vtep4_check_vector (&vxm->vtep_table, b1, ip41, &last_vtep4,
1051 : &vxm->vtep4_u512))
1052 : #else
1053 0 : if (!vtep4_check (&vxm->vtep_table, b1, ip41, &last_vtep4))
1054 : #endif
1055 0 : goto exit1; /* no local VTEP for GENEVE packet */
1056 : }
1057 : else
1058 : {
1059 0 : if (!vtep6_check (&vxm->vtep_table, b1, ip61, &last_vtep6))
1060 0 : goto exit1; /* no local VTEP for GENEVE packet */
1061 : }
1062 :
1063 0 : flags1 = b1->flags;
1064 0 : good_udp1 = (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1065 :
1066 : /* Don't verify UDP checksum for packets with explicit zero checksum. */
1067 0 : good_udp1 |= udp1->checksum == 0;
1068 :
1069 : /* Verify UDP length */
1070 0 : if (is_ip4)
1071 0 : ip_len1 = clib_net_to_host_u16 (ip41->length);
1072 : else
1073 0 : ip_len1 = clib_net_to_host_u16 (ip61->payload_length);
1074 0 : udp_len1 = clib_net_to_host_u16 (udp1->length);
1075 0 : len_diff1 = ip_len1 - udp_len1;
1076 :
1077 : /* Verify UDP checksum */
1078 0 : if (PREDICT_FALSE (!good_udp1))
1079 : {
1080 0 : if ((flags1 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1081 : {
1082 0 : if (is_ip4)
1083 0 : flags1 = ip4_tcp_udp_validate_checksum (vm, b1);
1084 : else
1085 0 : flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, b1);
1086 0 : good_udp1 =
1087 0 : (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1088 : }
1089 : }
1090 :
1091 0 : if (is_ip4)
1092 : {
1093 0 : error1 = good_udp1 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1094 0 : error1 = (len_diff1 >= 0) ? error1 : IP4_ERROR_UDP_LENGTH;
1095 : }
1096 : else
1097 : {
1098 0 : error1 = good_udp1 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1099 0 : error1 = (len_diff1 >= 0) ? error1 : IP6_ERROR_UDP_LENGTH;
1100 : }
1101 :
1102 0 : next1 = error1 ?
1103 0 : IP_GENEVE_BYPASS_NEXT_DROP : IP_GENEVE_BYPASS_NEXT_GENEVE;
1104 0 : b1->error = error1 ? error_node->errors[error1] : 0;
1105 :
1106 : /* geneve-input node expect current at GENEVE header */
1107 0 : if (is_ip4)
1108 0 : vlib_buffer_advance (b1,
1109 : sizeof (ip4_header_t) +
1110 : sizeof (udp_header_t));
1111 : else
1112 0 : vlib_buffer_advance (b1,
1113 : sizeof (ip6_header_t) +
1114 : sizeof (udp_header_t));
1115 :
1116 0 : exit1:
1117 0 : vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1118 : to_next, n_left_to_next,
1119 : bi0, bi1, next0, next1);
1120 : }
1121 :
1122 0 : while (n_left_from > 0 && n_left_to_next > 0)
1123 : {
1124 : vlib_buffer_t *b0;
1125 : ip4_header_t *ip40;
1126 : ip6_header_t *ip60;
1127 : udp_header_t *udp0;
1128 : u32 bi0, ip_len0, udp_len0, flags0, next0;
1129 : i32 len_diff0;
1130 : u8 error0, good_udp0, proto0;
1131 :
1132 0 : bi0 = to_next[0] = from[0];
1133 0 : from += 1;
1134 0 : n_left_from -= 1;
1135 0 : to_next += 1;
1136 0 : n_left_to_next -= 1;
1137 :
1138 0 : b0 = b[0];
1139 0 : b++;
1140 0 : if (is_ip4)
1141 0 : ip40 = vlib_buffer_get_current (b0);
1142 : else
1143 0 : ip60 = vlib_buffer_get_current (b0);
1144 :
1145 : /* Setup packet for next IP feature */
1146 0 : vnet_feature_next (&next0, b0);
1147 :
1148 0 : if (is_ip4)
1149 : /* Treat IP4 frag packets as "experimental" protocol for now
1150 : until support of IP frag reassembly is implemented */
1151 0 : proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol;
1152 : else
1153 0 : proto0 = ip60->protocol;
1154 :
1155 0 : if (proto0 != IP_PROTOCOL_UDP)
1156 0 : goto exit; /* not UDP packet */
1157 :
1158 0 : if (is_ip4)
1159 0 : udp0 = ip4_next_header (ip40);
1160 : else
1161 0 : udp0 = ip6_next_header (ip60);
1162 :
1163 0 : if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
1164 0 : goto exit; /* not GENEVE packet */
1165 :
1166 : /* Validate DIP against VTEPs */
1167 0 : if (is_ip4)
1168 : {
1169 : #ifdef CLIB_HAVE_VEC512
1170 0 : if (!vtep4_check_vector (&vxm->vtep_table, b0, ip40, &last_vtep4,
1171 : &vxm->vtep4_u512))
1172 : #else
1173 0 : if (!vtep4_check (&vxm->vtep_table, b0, ip40, &last_vtep4))
1174 : #endif
1175 0 : goto exit; /* no local VTEP for GENEVE packet */
1176 : }
1177 : else
1178 : {
1179 0 : if (!vtep6_check (&vxm->vtep_table, b0, ip60, &last_vtep6))
1180 0 : goto exit; /* no local VTEP for GENEVE packet */
1181 : }
1182 :
1183 0 : flags0 = b0->flags;
1184 0 : good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1185 :
1186 : /* Don't verify UDP checksum for packets with explicit zero checksum. */
1187 0 : good_udp0 |= udp0->checksum == 0;
1188 :
1189 : /* Verify UDP length */
1190 0 : if (is_ip4)
1191 0 : ip_len0 = clib_net_to_host_u16 (ip40->length);
1192 : else
1193 0 : ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
1194 0 : udp_len0 = clib_net_to_host_u16 (udp0->length);
1195 0 : len_diff0 = ip_len0 - udp_len0;
1196 :
1197 : /* Verify UDP checksum */
1198 0 : if (PREDICT_FALSE (!good_udp0))
1199 : {
1200 0 : if ((flags0 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1201 : {
1202 0 : if (is_ip4)
1203 0 : flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
1204 : else
1205 0 : flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
1206 0 : good_udp0 =
1207 0 : (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1208 : }
1209 : }
1210 :
1211 0 : if (is_ip4)
1212 : {
1213 0 : error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1214 0 : error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
1215 : }
1216 : else
1217 : {
1218 0 : error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1219 0 : error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
1220 : }
1221 :
1222 0 : next0 = error0 ?
1223 0 : IP_GENEVE_BYPASS_NEXT_DROP : IP_GENEVE_BYPASS_NEXT_GENEVE;
1224 0 : b0->error = error0 ? error_node->errors[error0] : 0;
1225 :
1226 : /* geneve-input node expect current at GENEVE header */
1227 0 : if (is_ip4)
1228 0 : vlib_buffer_advance (b0,
1229 : sizeof (ip4_header_t) +
1230 : sizeof (udp_header_t));
1231 : else
1232 0 : vlib_buffer_advance (b0,
1233 : sizeof (ip6_header_t) +
1234 : sizeof (udp_header_t));
1235 :
1236 0 : exit:
1237 0 : vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1238 : to_next, n_left_to_next,
1239 : bi0, next0);
1240 : }
1241 :
1242 0 : vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1243 : }
1244 :
1245 0 : return frame->n_vectors;
1246 : }
1247 :
1248 2236 : VLIB_NODE_FN (ip4_geneve_bypass_node) (vlib_main_t * vm,
1249 : vlib_node_runtime_t * node,
1250 : vlib_frame_t * frame)
1251 : {
1252 0 : return ip_geneve_bypass_inline (vm, node, frame, /* is_ip4 */ 1);
1253 : }
1254 :
1255 : /* *INDENT-OFF* */
1256 136120 : VLIB_REGISTER_NODE (ip4_geneve_bypass_node) =
1257 : {
1258 : .name = "ip4-geneve-bypass",
1259 : .vector_size = sizeof (u32),
1260 : .n_next_nodes = IP_GENEVE_BYPASS_N_NEXT,.next_nodes =
1261 : {
1262 : [IP_GENEVE_BYPASS_NEXT_DROP] = "error-drop",
1263 : [IP_GENEVE_BYPASS_NEXT_GENEVE] = "geneve4-input",
1264 : },
1265 : .format_buffer = format_ip4_header,
1266 : .format_trace = format_ip4_forward_next_trace,
1267 : };
1268 : /* *INDENT-ON* */
1269 :
1270 2236 : VLIB_NODE_FN (ip6_geneve_bypass_node) (vlib_main_t * vm,
1271 : vlib_node_runtime_t * node,
1272 : vlib_frame_t * frame)
1273 : {
1274 0 : return ip_geneve_bypass_inline (vm, node, frame, /* is_ip4 */ 0);
1275 : }
1276 :
1277 : /* *INDENT-OFF* */
1278 136120 : VLIB_REGISTER_NODE (ip6_geneve_bypass_node) =
1279 : {
1280 : .name = "ip6-geneve-bypass",
1281 : .vector_size = sizeof (u32),
1282 : .n_next_nodes = IP_GENEVE_BYPASS_N_NEXT,
1283 : .next_nodes =
1284 : {
1285 : [IP_GENEVE_BYPASS_NEXT_DROP] = "error-drop",
1286 : [IP_GENEVE_BYPASS_NEXT_GENEVE] = "geneve6-input",
1287 : },
1288 : .format_buffer = format_ip6_header,
1289 : .format_trace = format_ip6_forward_next_trace,
1290 : };
1291 : /* *INDENT-ON* */
1292 :
1293 : /*
1294 : * fd.io coding-style-patch-verification: ON
1295 : *
1296 : * Local Variables:
1297 : * eval: (c-set-style "gnu")
1298 : * End:
1299 : */
|
