Line data Source code
1 : /*
2 : * Copyright (c) 2020 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef __CNAT_NODE_H__
17 : #define __CNAT_NODE_H__
18 :
19 : #include <vlibmemory/api.h>
20 : #include <vnet/dpo/load_balance.h>
21 : #include <vnet/dpo/load_balance_map.h>
22 :
23 : #include <cnat/cnat_session.h>
24 : #include <cnat/cnat_client.h>
25 : #include <cnat/cnat_inline.h>
26 : #include <cnat/cnat_translation.h>
27 :
28 : #include <vnet/ip/ip4_inlines.h>
29 : #include <vnet/ip/ip6_inlines.h>
30 :
31 : typedef uword (*cnat_node_sub_t) (vlib_main_t * vm,
32 : vlib_node_runtime_t * node,
33 : vlib_buffer_t * b,
34 : cnat_node_ctx_t * ctx, int rv,
35 : cnat_session_t * session);
36 :
37 : typedef struct cnat_trace_element_t_
38 : {
39 : cnat_session_t session;
40 : cnat_translation_t tr;
41 : u32 sw_if_index[VLIB_N_RX_TX];
42 : u32 snat_policy_result;
43 : u8 flags;
44 : } cnat_trace_element_t;
45 :
46 : typedef enum cnat_trace_element_flag_t_
47 : {
48 : CNAT_TRACE_SESSION_FOUND = (1 << 0),
49 : CNAT_TRACE_SESSION_CREATED = (1 << 1),
50 : CNAT_TRACE_TRANSLATION_FOUND = (1 << 2),
51 : CNAT_TRACE_NO_NAT = (1 << 3),
52 : } cnat_trace_element_flag_t;
53 :
54 : static_always_inline void
55 5520 : cnat_add_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b,
56 : cnat_session_t *session, const cnat_translation_t *ct,
57 : u8 flags)
58 : {
59 : cnat_trace_element_t *t;
60 5520 : if (NULL != ct)
61 348 : flags |= CNAT_TRACE_TRANSLATION_FOUND;
62 :
63 5520 : t = vlib_add_trace (vm, node, b, sizeof (*t));
64 5520 : t->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_RX];
65 5520 : t->sw_if_index[VLIB_TX] = vnet_buffer (b)->sw_if_index[VLIB_TX];
66 :
67 5520 : if (flags & (CNAT_TRACE_SESSION_FOUND | CNAT_TRACE_SESSION_CREATED))
68 4230 : clib_memcpy (&t->session, session, sizeof (t->session));
69 5520 : if (flags & CNAT_TRACE_TRANSLATION_FOUND)
70 348 : clib_memcpy (&t->tr, ct, sizeof (cnat_translation_t));
71 5520 : t->flags = flags;
72 5520 : }
73 :
74 : static u8 *
75 5640 : format_cnat_trace (u8 *s, va_list *args)
76 : {
77 5640 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
78 5640 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
79 5640 : cnat_trace_element_t *t = va_arg (*args, cnat_trace_element_t *);
80 5640 : u32 indent = format_get_indent (s);
81 5640 : vnet_main_t *vnm = vnet_get_main ();
82 :
83 5640 : if (t->flags & CNAT_TRACE_SESSION_CREATED)
84 370 : s = format (s, "created session");
85 5270 : else if (t->flags & CNAT_TRACE_SESSION_FOUND)
86 3950 : s = format (s, "found session");
87 : else
88 1320 : s = format (s, "session not found");
89 :
90 5640 : if (t->flags & (CNAT_TRACE_NO_NAT))
91 0 : s = format (s, " [policy:skip]");
92 :
93 5640 : s = format (s, "\n%Uin:%U out:%U ", format_white_space, indent,
94 : format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_RX],
95 : format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_TX]);
96 :
97 5640 : if (t->flags & (CNAT_TRACE_SESSION_CREATED | CNAT_TRACE_SESSION_FOUND))
98 4320 : s = format (s, "\n%U%U", format_white_space, indent, format_cnat_session,
99 : &t->session, 1);
100 :
101 5640 : if (t->flags & CNAT_TRACE_TRANSLATION_FOUND)
102 350 : s = format (s, "\n%Utranslation: %U", format_white_space, indent,
103 : format_cnat_translation, &t->tr, 0);
104 :
105 5640 : return s;
106 : }
107 :
108 : static_always_inline u8
109 295 : icmp_type_is_error_message (u8 icmp_type)
110 : {
111 295 : switch (icmp_type)
112 : {
113 227 : case ICMP4_destination_unreachable:
114 : case ICMP4_time_exceeded:
115 : case ICMP4_parameter_problem:
116 : case ICMP4_source_quench:
117 : case ICMP4_redirect:
118 : case ICMP4_alternate_host_address:
119 227 : return 1;
120 : }
121 68 : return 0;
122 : }
123 :
124 : static_always_inline u8
125 68 : icmp_type_is_echo (u8 icmp_type)
126 : {
127 68 : switch (icmp_type)
128 : {
129 68 : case ICMP4_echo_request:
130 : case ICMP4_echo_reply:
131 68 : return 1;
132 : }
133 0 : return 0;
134 : }
135 :
136 : static_always_inline u8
137 204 : icmp6_type_is_echo (u8 icmp_type)
138 : {
139 204 : switch (icmp_type)
140 : {
141 204 : case ICMP6_echo_request:
142 : case ICMP6_echo_reply:
143 204 : return 1;
144 : }
145 0 : return 0;
146 : }
147 :
148 : static_always_inline u8
149 702 : icmp6_type_is_error_message (u8 icmp_type)
150 : {
151 702 : switch (icmp_type)
152 : {
153 498 : case ICMP6_destination_unreachable:
154 : case ICMP6_time_exceeded:
155 : case ICMP6_parameter_problem:
156 498 : return 1;
157 : }
158 204 : return 0;
159 : }
160 :
161 : static_always_inline u8
162 135 : cmp_ip6_address (const ip6_address_t * a1, const ip6_address_t * a2)
163 : {
164 135 : return ((a1->as_u64[0] == a2->as_u64[0])
165 135 : && (a1->as_u64[1] == a2->as_u64[1]));
166 : }
167 :
168 : /**
169 : * Inline translation functions
170 : */
171 :
172 : static_always_inline u8
173 10350 : has_ip6_address (ip6_address_t * a)
174 : {
175 10350 : return ((0 != a->as_u64[0]) || (0 != a->as_u64[1]));
176 : }
177 :
178 : static_always_inline void
179 1815 : cnat_ip4_translate_l4 (ip4_header_t * ip4, udp_header_t * udp,
180 : ip_csum_t * sum,
181 : ip4_address_t new_addr[VLIB_N_DIR],
182 : u16 new_port[VLIB_N_DIR])
183 : {
184 : u16 old_port[VLIB_N_DIR];
185 : ip4_address_t old_addr[VLIB_N_DIR];
186 :
187 : /* Fastpath no checksum */
188 1815 : if (PREDICT_TRUE (0 == *sum))
189 : {
190 0 : udp->dst_port = new_port[VLIB_TX];
191 0 : udp->src_port = new_port[VLIB_RX];
192 0 : return;
193 : }
194 :
195 1815 : old_port[VLIB_TX] = udp->dst_port;
196 1815 : old_port[VLIB_RX] = udp->src_port;
197 1815 : old_addr[VLIB_TX] = ip4->dst_address;
198 1815 : old_addr[VLIB_RX] = ip4->src_address;
199 :
200 1815 : if (new_addr[VLIB_TX].as_u32)
201 : {
202 1815 : *sum =
203 1815 : ip_csum_update (*sum, old_addr[VLIB_TX].as_u32,
204 : new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
205 : }
206 1815 : if (new_port[VLIB_TX])
207 : {
208 1815 : udp->dst_port = new_port[VLIB_TX];
209 1815 : *sum = ip_csum_update (*sum, old_port[VLIB_TX], new_port[VLIB_TX],
210 : ip4_header_t /* cheat */ ,
211 : length /* changed member */ );
212 : }
213 1815 : if (new_addr[VLIB_RX].as_u32)
214 : {
215 1815 : *sum =
216 1815 : ip_csum_update (*sum, old_addr[VLIB_RX].as_u32,
217 : new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
218 : }
219 1815 : if (new_port[VLIB_RX])
220 : {
221 1815 : udp->src_port = new_port[VLIB_RX];
222 1815 : *sum = ip_csum_update (*sum, old_port[VLIB_RX], new_port[VLIB_RX],
223 : ip4_header_t /* cheat */ ,
224 : length /* changed member */ );
225 : }
226 : }
227 :
228 : static_always_inline void
229 0 : cnat_ip4_translate_sctp (ip4_header_t *ip4, sctp_header_t *sctp,
230 : u16 new_port[VLIB_N_DIR])
231 : {
232 : /* Fastpath no checksum */
233 0 : if (PREDICT_TRUE (0 == sctp->checksum))
234 : {
235 0 : sctp->dst_port = new_port[VLIB_TX];
236 0 : sctp->src_port = new_port[VLIB_RX];
237 0 : return;
238 : }
239 :
240 0 : if (new_port[VLIB_TX])
241 0 : sctp->dst_port = new_port[VLIB_TX];
242 0 : if (new_port[VLIB_RX])
243 0 : sctp->src_port = new_port[VLIB_RX];
244 :
245 0 : sctp->checksum = 0;
246 0 : sctp->checksum = clib_host_to_little_u32 (~clib_crc32c_with_init (
247 0 : (u8 *) sctp, ntohs (ip4->length) - sizeof (ip4_header_t),
248 : ~0 /* init value */));
249 : }
250 :
251 : static_always_inline void
252 1920 : cnat_ip4_translate_l3 (ip4_header_t * ip4, ip4_address_t new_addr[VLIB_N_DIR])
253 : {
254 : ip4_address_t old_addr[VLIB_N_DIR];
255 : ip_csum_t sum;
256 :
257 1920 : old_addr[VLIB_TX] = ip4->dst_address;
258 1920 : old_addr[VLIB_RX] = ip4->src_address;
259 :
260 1920 : sum = ip4->checksum;
261 1920 : if (new_addr[VLIB_TX].as_u32)
262 : {
263 1920 : ip4->dst_address = new_addr[VLIB_TX];
264 : sum =
265 1920 : ip_csum_update (sum, old_addr[VLIB_TX].as_u32,
266 : new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
267 : }
268 1920 : if (new_addr[VLIB_RX].as_u32)
269 : {
270 1920 : ip4->src_address = new_addr[VLIB_RX];
271 : sum =
272 1920 : ip_csum_update (sum, old_addr[VLIB_RX].as_u32,
273 : new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
274 : }
275 1920 : ip4->checksum = ip_csum_fold (sum);
276 1920 : }
277 :
278 : static_always_inline void
279 2460 : cnat_tcp_update_session_lifetime (tcp_header_t * tcp, u32 index)
280 : {
281 2460 : cnat_main_t *cm = &cnat_main;
282 2460 : if (PREDICT_FALSE (tcp_fin (tcp)))
283 : {
284 0 : cnat_timestamp_set_lifetime (index, CNAT_DEFAULT_TCP_RST_TIMEOUT);
285 : }
286 :
287 2460 : if (PREDICT_FALSE (tcp_rst (tcp)))
288 : {
289 0 : cnat_timestamp_set_lifetime (index, CNAT_DEFAULT_TCP_RST_TIMEOUT);
290 : }
291 :
292 2460 : if (PREDICT_FALSE (tcp_syn (tcp) && tcp_ack (tcp)))
293 : {
294 0 : cnat_timestamp_set_lifetime (index, cm->tcp_max_age);
295 : }
296 2460 : }
297 :
298 : static_always_inline void
299 30 : cnat_translation_icmp4_echo (ip4_header_t * ip4, icmp46_header_t * icmp,
300 : ip4_address_t new_addr[VLIB_N_DIR],
301 : u16 new_port[VLIB_N_DIR])
302 : {
303 : ip_csum_t sum;
304 : u16 old_port;
305 30 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
306 :
307 30 : cnat_ip4_translate_l3 (ip4, new_addr);
308 30 : old_port = echo->identifier;
309 30 : echo->identifier = new_port[VLIB_RX];
310 :
311 30 : sum = icmp->checksum;
312 30 : sum = ip_csum_update (sum, old_port, new_port[VLIB_RX],
313 : ip4_header_t /* cheat */ ,
314 : length /* changed member */ );
315 :
316 30 : icmp->checksum = ip_csum_fold (sum);
317 30 : }
318 :
319 : static_always_inline void
320 75 : cnat_translation_icmp4_error (ip4_header_t * outer_ip4,
321 : icmp46_header_t * icmp,
322 : ip4_address_t outer_new_addr[VLIB_N_DIR],
323 : u16 outer_new_port[VLIB_N_DIR],
324 : u8 snat_outer_ip)
325 : {
326 : ip4_address_t new_addr[VLIB_N_DIR];
327 : ip4_address_t old_addr[VLIB_N_DIR];
328 : u16 new_port[VLIB_N_DIR];
329 : u16 old_port[VLIB_N_DIR];
330 : ip_csum_t sum, old_ip_sum, inner_l4_sum, inner_l4_old_sum;
331 :
332 75 : ip4_header_t *ip4 = (ip4_header_t *) (icmp + 2);
333 75 : udp_header_t *udp = (udp_header_t *) (ip4 + 1);
334 75 : tcp_header_t *tcp = (tcp_header_t *) udp;
335 :
336 : /* Swap inner ports */
337 75 : new_addr[VLIB_TX] = outer_new_addr[VLIB_RX];
338 75 : new_addr[VLIB_RX] = outer_new_addr[VLIB_TX];
339 75 : new_port[VLIB_TX] = outer_new_port[VLIB_RX];
340 75 : new_port[VLIB_RX] = outer_new_port[VLIB_TX];
341 :
342 75 : old_addr[VLIB_TX] = ip4->dst_address;
343 75 : old_addr[VLIB_RX] = ip4->src_address;
344 75 : old_port[VLIB_RX] = udp->src_port;
345 75 : old_port[VLIB_TX] = udp->dst_port;
346 :
347 75 : sum = icmp->checksum;
348 75 : old_ip_sum = ip4->checksum;
349 :
350 : /* translate outer ip. */
351 75 : if (!snat_outer_ip)
352 0 : outer_new_addr[VLIB_RX] = outer_ip4->src_address;
353 75 : cnat_ip4_translate_l3 (outer_ip4, outer_new_addr);
354 :
355 75 : if (ip4->protocol == IP_PROTOCOL_TCP)
356 : {
357 45 : inner_l4_old_sum = inner_l4_sum = tcp->checksum;
358 45 : cnat_ip4_translate_l4 (ip4, udp, &inner_l4_sum, new_addr, new_port);
359 45 : tcp->checksum = ip_csum_fold (inner_l4_sum);
360 : }
361 30 : else if (ip4->protocol == IP_PROTOCOL_UDP)
362 : {
363 30 : inner_l4_old_sum = inner_l4_sum = udp->checksum;
364 30 : cnat_ip4_translate_l4 (ip4, udp, &inner_l4_sum, new_addr, new_port);
365 30 : udp->checksum = ip_csum_fold (inner_l4_sum);
366 : }
367 : else
368 0 : return;
369 :
370 : /* UDP/TCP checksum changed */
371 75 : sum = ip_csum_update (sum, inner_l4_old_sum, inner_l4_sum,
372 : ip4_header_t, checksum);
373 :
374 : /* UDP/TCP Ports changed */
375 75 : if (old_port[VLIB_TX] && new_port[VLIB_TX])
376 75 : sum = ip_csum_update (sum, old_port[VLIB_TX], new_port[VLIB_TX],
377 : ip4_header_t /* cheat */ ,
378 : length /* changed member */ );
379 :
380 75 : if (old_port[VLIB_RX] && new_port[VLIB_RX])
381 75 : sum = ip_csum_update (sum, old_port[VLIB_RX], new_port[VLIB_RX],
382 : ip4_header_t /* cheat */ ,
383 : length /* changed member */ );
384 :
385 :
386 75 : cnat_ip4_translate_l3 (ip4, new_addr);
387 75 : ip_csum_t new_ip_sum = ip4->checksum;
388 : /* IP checksum changed */
389 75 : sum = ip_csum_update (sum, old_ip_sum, new_ip_sum, ip4_header_t, checksum);
390 :
391 : /* IP src/dst addr changed */
392 75 : if (new_addr[VLIB_TX].as_u32)
393 : sum =
394 75 : ip_csum_update (sum, old_addr[VLIB_TX].as_u32, new_addr[VLIB_TX].as_u32,
395 : ip4_header_t, dst_address);
396 :
397 75 : if (new_addr[VLIB_RX].as_u32)
398 : sum =
399 75 : ip_csum_update (sum, old_addr[VLIB_RX].as_u32, new_addr[VLIB_RX].as_u32,
400 : ip4_header_t, src_address);
401 :
402 75 : icmp->checksum = ip_csum_fold (sum);
403 : }
404 :
405 : static_always_inline void
406 1845 : cnat_translation_ip4 (const cnat_session_t * session,
407 : ip4_header_t * ip4, udp_header_t * udp)
408 : {
409 1845 : tcp_header_t *tcp = (tcp_header_t *) udp;
410 : ip4_address_t new_addr[VLIB_N_DIR];
411 : u16 new_port[VLIB_N_DIR];
412 :
413 1845 : new_addr[VLIB_TX] = session->value.cs_ip[VLIB_TX].ip4;
414 1845 : new_addr[VLIB_RX] = session->value.cs_ip[VLIB_RX].ip4;
415 1845 : new_port[VLIB_TX] = session->value.cs_port[VLIB_TX];
416 1845 : new_port[VLIB_RX] = session->value.cs_port[VLIB_RX];
417 :
418 1845 : if (ip4->protocol == IP_PROTOCOL_TCP)
419 : {
420 1125 : ip_csum_t sum = tcp->checksum;
421 1125 : cnat_ip4_translate_l4 (ip4, udp, &sum, new_addr, new_port);
422 1125 : tcp->checksum = ip_csum_fold (sum);
423 1125 : cnat_ip4_translate_l3 (ip4, new_addr);
424 1125 : cnat_tcp_update_session_lifetime (tcp, session->value.cs_ts_index);
425 : }
426 720 : else if (ip4->protocol == IP_PROTOCOL_UDP)
427 : {
428 615 : ip_csum_t sum = udp->checksum;
429 615 : cnat_ip4_translate_l4 (ip4, udp, &sum, new_addr, new_port);
430 615 : udp->checksum = ip_csum_fold (sum);
431 615 : cnat_ip4_translate_l3 (ip4, new_addr);
432 : }
433 105 : else if (ip4->protocol == IP_PROTOCOL_SCTP)
434 : {
435 0 : sctp_header_t *sctp = (sctp_header_t *) udp;
436 0 : cnat_ip4_translate_sctp (ip4, sctp, new_port);
437 0 : cnat_ip4_translate_l3 (ip4, new_addr);
438 : }
439 105 : else if (ip4->protocol == IP_PROTOCOL_ICMP)
440 : {
441 105 : icmp46_header_t *icmp = (icmp46_header_t *) udp;
442 105 : if (icmp_type_is_error_message (icmp->type))
443 : {
444 : /* SNAT only if src_addr was translated */
445 75 : u8 snat_outer_ip =
446 75 : (ip4->src_address.as_u32 ==
447 75 : session->key.cs_ip[VLIB_RX].ip4.as_u32);
448 75 : cnat_translation_icmp4_error (ip4, icmp, new_addr, new_port,
449 : snat_outer_ip);
450 : }
451 30 : else if (icmp_type_is_echo (icmp->type))
452 30 : cnat_translation_icmp4_echo (ip4, icmp, new_addr, new_port);
453 : }
454 1845 : }
455 :
456 : static_always_inline void
457 2520 : cnat_ip6_translate_l3 (ip6_header_t * ip6, ip6_address_t new_addr[VLIB_N_DIR])
458 : {
459 2520 : if (has_ip6_address (&new_addr[VLIB_TX]))
460 2520 : ip6_address_copy (&ip6->dst_address, &new_addr[VLIB_TX]);
461 2520 : if (has_ip6_address (&new_addr[VLIB_RX]))
462 2520 : ip6_address_copy (&ip6->src_address, &new_addr[VLIB_RX]);
463 2520 : }
464 :
465 : static_always_inline void
466 2295 : cnat_ip6_translate_l4 (ip6_header_t * ip6, udp_header_t * udp,
467 : ip_csum_t * sum,
468 : ip6_address_t new_addr[VLIB_N_DIR],
469 : u16 new_port[VLIB_N_DIR])
470 : {
471 : u16 old_port[VLIB_N_DIR];
472 : ip6_address_t old_addr[VLIB_N_DIR];
473 :
474 : /* Fastpath no checksum */
475 2295 : if (PREDICT_TRUE (0 == *sum))
476 : {
477 0 : udp->dst_port = new_port[VLIB_TX];
478 0 : udp->src_port = new_port[VLIB_RX];
479 0 : return;
480 : }
481 :
482 2295 : old_port[VLIB_TX] = udp->dst_port;
483 2295 : old_port[VLIB_RX] = udp->src_port;
484 2295 : ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
485 2295 : ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
486 :
487 2295 : if (has_ip6_address (&new_addr[VLIB_TX]))
488 : {
489 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_TX].as_u64[0]);
490 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_TX].as_u64[1]);
491 2295 : *sum = ip_csum_sub_even (*sum, old_addr[VLIB_TX].as_u64[0]);
492 2295 : *sum = ip_csum_sub_even (*sum, old_addr[VLIB_TX].as_u64[1]);
493 : }
494 :
495 2295 : if (new_port[VLIB_TX])
496 : {
497 2295 : udp->dst_port = new_port[VLIB_TX];
498 2295 : *sum = ip_csum_update (*sum, old_port[VLIB_TX], new_port[VLIB_TX],
499 : ip4_header_t /* cheat */ ,
500 : length /* changed member */ );
501 : }
502 2295 : if (has_ip6_address (&new_addr[VLIB_RX]))
503 : {
504 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_RX].as_u64[0]);
505 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_RX].as_u64[1]);
506 2295 : *sum = ip_csum_sub_even (*sum, old_addr[VLIB_RX].as_u64[0]);
507 2295 : *sum = ip_csum_sub_even (*sum, old_addr[VLIB_RX].as_u64[1]);
508 : }
509 :
510 2295 : if (new_port[VLIB_RX])
511 : {
512 2295 : udp->src_port = new_port[VLIB_RX];
513 2295 : *sum = ip_csum_update (*sum, old_port[VLIB_RX], new_port[VLIB_RX],
514 : ip4_header_t /* cheat */ ,
515 : length /* changed member */ );
516 : }
517 : }
518 :
519 : static_always_inline void
520 90 : cnat_translation_icmp6_echo (ip6_header_t * ip6, icmp46_header_t * icmp,
521 : ip6_address_t new_addr[VLIB_N_DIR],
522 : u16 new_port[VLIB_N_DIR])
523 : {
524 90 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
525 : ip6_address_t old_addr[VLIB_N_DIR];
526 : ip_csum_t sum;
527 : u16 old_port;
528 90 : old_port = echo->identifier;
529 90 : ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
530 90 : ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
531 :
532 90 : sum = icmp->checksum;
533 :
534 90 : cnat_ip6_translate_l3 (ip6, new_addr);
535 90 : if (has_ip6_address (&new_addr[VLIB_TX]))
536 : {
537 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[0]);
538 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[1]);
539 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[0]);
540 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[1]);
541 : }
542 :
543 90 : if (has_ip6_address (&new_addr[VLIB_RX]))
544 : {
545 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[0]);
546 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[1]);
547 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[0]);
548 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[1]);
549 : }
550 :
551 90 : echo->identifier = new_port[VLIB_RX];
552 90 : sum = ip_csum_update (sum, old_port, new_port[VLIB_RX],
553 : ip4_header_t /* cheat */ ,
554 : length /* changed member */ );
555 :
556 90 : icmp->checksum = ip_csum_fold (sum);
557 90 : }
558 :
559 : static_always_inline void
560 135 : cnat_translation_icmp6_error (ip6_header_t * outer_ip6,
561 : icmp46_header_t * icmp,
562 : ip6_address_t outer_new_addr[VLIB_N_DIR],
563 : u16 outer_new_port[VLIB_N_DIR],
564 : u8 snat_outer_ip)
565 : {
566 : ip6_address_t new_addr[VLIB_N_DIR];
567 : ip6_address_t old_addr[VLIB_N_DIR];
568 : ip6_address_t outer_old_addr[VLIB_N_DIR];
569 : u16 new_port[VLIB_N_DIR];
570 : u16 old_port[VLIB_N_DIR];
571 : ip_csum_t sum, inner_l4_sum, inner_l4_old_sum;
572 :
573 135 : if (!icmp6_type_is_error_message (icmp->type))
574 0 : return;
575 :
576 135 : ip6_header_t *ip6 = (ip6_header_t *) (icmp + 2);
577 135 : udp_header_t *udp = (udp_header_t *) (ip6 + 1);
578 135 : tcp_header_t *tcp = (tcp_header_t *) udp;
579 :
580 : /* Swap inner ports */
581 135 : ip6_address_copy (&new_addr[VLIB_RX], &outer_new_addr[VLIB_TX]);
582 135 : ip6_address_copy (&new_addr[VLIB_TX], &outer_new_addr[VLIB_RX]);
583 135 : new_port[VLIB_TX] = outer_new_port[VLIB_RX];
584 135 : new_port[VLIB_RX] = outer_new_port[VLIB_TX];
585 :
586 135 : ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
587 135 : ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
588 135 : old_port[VLIB_RX] = udp->src_port;
589 135 : old_port[VLIB_TX] = udp->dst_port;
590 :
591 135 : sum = icmp->checksum;
592 : /* Translate outer ip */
593 135 : ip6_address_copy (&outer_old_addr[VLIB_TX], &outer_ip6->dst_address);
594 135 : ip6_address_copy (&outer_old_addr[VLIB_RX], &outer_ip6->src_address);
595 135 : if (!snat_outer_ip)
596 0 : ip6_address_copy (&outer_new_addr[VLIB_RX], &outer_ip6->src_address);
597 135 : cnat_ip6_translate_l3 (outer_ip6, outer_new_addr);
598 135 : if (has_ip6_address (&outer_new_addr[VLIB_TX]))
599 : {
600 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_TX].as_u64[0]);
601 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_TX].as_u64[1]);
602 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_TX].as_u64[0]);
603 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_TX].as_u64[1]);
604 : }
605 :
606 135 : if (has_ip6_address (&outer_new_addr[VLIB_RX]))
607 : {
608 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_RX].as_u64[0]);
609 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_RX].as_u64[1]);
610 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_RX].as_u64[0]);
611 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_RX].as_u64[1]);
612 : }
613 :
614 : /* Translate inner TCP / UDP */
615 135 : if (ip6->protocol == IP_PROTOCOL_TCP)
616 : {
617 75 : inner_l4_old_sum = inner_l4_sum = tcp->checksum;
618 75 : cnat_ip6_translate_l4 (ip6, udp, &inner_l4_sum, new_addr, new_port);
619 75 : tcp->checksum = ip_csum_fold (inner_l4_sum);
620 : }
621 60 : else if (ip6->protocol == IP_PROTOCOL_UDP)
622 : {
623 60 : inner_l4_old_sum = inner_l4_sum = udp->checksum;
624 60 : cnat_ip6_translate_l4 (ip6, udp, &inner_l4_sum, new_addr, new_port);
625 60 : udp->checksum = ip_csum_fold (inner_l4_sum);
626 : }
627 : else
628 0 : return;
629 :
630 : /* UDP/TCP checksum changed */
631 135 : sum = ip_csum_update (sum, inner_l4_old_sum, inner_l4_sum,
632 : ip4_header_t /* cheat */ ,
633 : checksum);
634 :
635 : /* UDP/TCP Ports changed */
636 135 : if (old_port[VLIB_TX] && new_port[VLIB_TX])
637 135 : sum = ip_csum_update (sum, old_port[VLIB_TX], new_port[VLIB_TX],
638 : ip4_header_t /* cheat */ ,
639 : length /* changed member */ );
640 :
641 135 : if (old_port[VLIB_RX] && new_port[VLIB_RX])
642 135 : sum = ip_csum_update (sum, old_port[VLIB_RX], new_port[VLIB_RX],
643 : ip4_header_t /* cheat */ ,
644 : length /* changed member */ );
645 :
646 :
647 135 : cnat_ip6_translate_l3 (ip6, new_addr);
648 : /* IP src/dst addr changed */
649 135 : if (has_ip6_address (&new_addr[VLIB_TX]))
650 : {
651 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[0]);
652 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[1]);
653 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[0]);
654 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[1]);
655 : }
656 :
657 135 : if (has_ip6_address (&new_addr[VLIB_RX]))
658 : {
659 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[0]);
660 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[1]);
661 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[0]);
662 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[1]);
663 : }
664 :
665 135 : icmp->checksum = ip_csum_fold (sum);
666 : }
667 :
668 : static_always_inline void
669 2385 : cnat_translation_ip6 (const cnat_session_t * session,
670 : ip6_header_t * ip6, udp_header_t * udp)
671 : {
672 2385 : tcp_header_t *tcp = (tcp_header_t *) udp;
673 : ip6_address_t new_addr[VLIB_N_DIR];
674 : u16 new_port[VLIB_N_DIR];
675 :
676 2385 : ip6_address_copy (&new_addr[VLIB_TX], &session->value.cs_ip[VLIB_TX].ip6);
677 2385 : ip6_address_copy (&new_addr[VLIB_RX], &session->value.cs_ip[VLIB_RX].ip6);
678 2385 : new_port[VLIB_TX] = session->value.cs_port[VLIB_TX];
679 2385 : new_port[VLIB_RX] = session->value.cs_port[VLIB_RX];
680 :
681 2385 : if (ip6->protocol == IP_PROTOCOL_TCP)
682 : {
683 1335 : ip_csum_t sum = tcp->checksum;
684 1335 : cnat_ip6_translate_l4 (ip6, udp, &sum, new_addr, new_port);
685 1335 : tcp->checksum = ip_csum_fold (sum);
686 1335 : cnat_ip6_translate_l3 (ip6, new_addr);
687 1335 : cnat_tcp_update_session_lifetime (tcp, session->value.cs_ts_index);
688 : }
689 1050 : else if (ip6->protocol == IP_PROTOCOL_UDP)
690 : {
691 825 : ip_csum_t sum = udp->checksum;
692 825 : cnat_ip6_translate_l4 (ip6, udp, &sum, new_addr, new_port);
693 825 : udp->checksum = ip_csum_fold (sum);
694 825 : cnat_ip6_translate_l3 (ip6, new_addr);
695 : }
696 225 : else if (ip6->protocol == IP_PROTOCOL_ICMP6)
697 : {
698 225 : icmp46_header_t *icmp = (icmp46_header_t *) udp;
699 225 : if (icmp6_type_is_error_message (icmp->type))
700 : {
701 : /* SNAT only if src_addr was translated */
702 135 : u8 snat_outer_ip = cmp_ip6_address (&ip6->src_address,
703 : &session->key.
704 : cs_ip[VLIB_RX].ip6);
705 135 : cnat_translation_icmp6_error (ip6, icmp, new_addr, new_port,
706 : snat_outer_ip);
707 : }
708 90 : else if (icmp6_type_is_echo (icmp->type))
709 90 : cnat_translation_icmp6_echo (ip6, icmp, new_addr, new_port);
710 : }
711 2385 : }
712 :
713 : static_always_inline void
714 6992 : cnat_session_make_key (vlib_buffer_t *b, ip_address_family_t af,
715 : cnat_session_location_t cs_loc, cnat_bihash_kv_t *bkey)
716 : {
717 : udp_header_t *udp;
718 6992 : cnat_session_t *session = (cnat_session_t *) bkey;
719 6992 : u32 iph_offset = 0;
720 6992 : session->key.cs_af = af;
721 :
722 6992 : session->key.cs_loc = cs_loc;
723 6992 : session->key.__cs_pad = 0;
724 6992 : if (cs_loc == CNAT_LOCATION_OUTPUT)
725 : /* rewind buffer */
726 0 : iph_offset = vnet_buffer (b)->ip.save_rewrite_length;
727 :
728 6992 : if (AF_IP4 == af)
729 : {
730 : ip4_header_t *ip4;
731 3116 : ip4 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
732 :
733 3116 : if (PREDICT_FALSE (ip4->protocol == IP_PROTOCOL_ICMP))
734 : {
735 190 : icmp46_header_t *icmp = (icmp46_header_t *) (ip4 + 1);
736 190 : if (icmp_type_is_error_message (icmp->type))
737 : {
738 152 : ip4 = (ip4_header_t *) (icmp + 2); /* Use inner packet */
739 152 : udp = (udp_header_t *) (ip4 + 1);
740 : /* Swap dst & src for search as ICMP payload is reversed */
741 152 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
742 152 : &ip4->dst_address);
743 152 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
744 152 : &ip4->src_address);
745 152 : session->key.cs_proto = ip4->protocol;
746 152 : session->key.cs_port[VLIB_TX] = udp->src_port;
747 152 : session->key.cs_port[VLIB_RX] = udp->dst_port;
748 : }
749 38 : else if (icmp_type_is_echo (icmp->type))
750 : {
751 38 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
752 38 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
753 38 : &ip4->dst_address);
754 38 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
755 38 : &ip4->src_address);
756 38 : session->key.cs_proto = ip4->protocol;
757 38 : session->key.cs_port[VLIB_TX] = echo->identifier;
758 38 : session->key.cs_port[VLIB_RX] = echo->identifier;
759 : }
760 : else
761 0 : goto error;
762 : }
763 2926 : else if (ip4->protocol == IP_PROTOCOL_UDP ||
764 1900 : ip4->protocol == IP_PROTOCOL_TCP)
765 : {
766 2926 : udp = (udp_header_t *) (ip4 + 1);
767 2926 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
768 2926 : &ip4->dst_address);
769 2926 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
770 2926 : &ip4->src_address);
771 2926 : session->key.cs_proto = ip4->protocol;
772 2926 : session->key.cs_port[VLIB_RX] = udp->src_port;
773 2926 : session->key.cs_port[VLIB_TX] = udp->dst_port;
774 : }
775 0 : else if (ip4->protocol == IP_PROTOCOL_SCTP)
776 : {
777 : sctp_header_t *sctp;
778 0 : sctp = (sctp_header_t *) (ip4 + 1);
779 0 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
780 0 : &ip4->dst_address);
781 0 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
782 0 : &ip4->src_address);
783 0 : session->key.cs_proto = ip4->protocol;
784 0 : session->key.cs_port[VLIB_RX] = sctp->src_port;
785 0 : session->key.cs_port[VLIB_TX] = sctp->dst_port;
786 : }
787 : else
788 0 : goto error;
789 : }
790 : else
791 : {
792 : ip6_header_t *ip6;
793 3876 : ip6 = (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
794 3876 : if (PREDICT_FALSE (ip6->protocol == IP_PROTOCOL_ICMP6))
795 : {
796 342 : icmp46_header_t *icmp = (icmp46_header_t *) (ip6 + 1);
797 342 : if (icmp6_type_is_error_message (icmp->type))
798 : {
799 228 : ip6 = (ip6_header_t *) (icmp + 2); /* Use inner packet */
800 228 : udp = (udp_header_t *) (ip6 + 1);
801 : /* Swap dst & src for search as ICMP payload is reversed */
802 228 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
803 228 : &ip6->dst_address);
804 228 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
805 228 : &ip6->src_address);
806 228 : session->key.cs_proto = ip6->protocol;
807 228 : session->key.cs_port[VLIB_TX] = udp->src_port;
808 228 : session->key.cs_port[VLIB_RX] = udp->dst_port;
809 : }
810 114 : else if (icmp6_type_is_echo (icmp->type))
811 : {
812 114 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
813 114 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
814 114 : &ip6->dst_address);
815 114 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
816 114 : &ip6->src_address);
817 114 : session->key.cs_proto = ip6->protocol;
818 114 : session->key.cs_port[VLIB_TX] = echo->identifier;
819 114 : session->key.cs_port[VLIB_RX] = echo->identifier;
820 : }
821 : else
822 0 : goto error;
823 : }
824 3534 : else if (ip6->protocol == IP_PROTOCOL_UDP ||
825 2204 : ip6->protocol == IP_PROTOCOL_TCP)
826 : {
827 3534 : udp = (udp_header_t *) (ip6 + 1);
828 3534 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
829 3534 : &ip6->dst_address);
830 3534 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
831 3534 : &ip6->src_address);
832 3534 : session->key.cs_port[VLIB_RX] = udp->src_port;
833 3534 : session->key.cs_port[VLIB_TX] = udp->dst_port;
834 3534 : session->key.cs_proto = ip6->protocol;
835 : }
836 : else
837 0 : goto error;
838 : }
839 6992 : return;
840 :
841 0 : error:
842 : /* Ensure we dont find anything */
843 0 : session->key.cs_proto = 0;
844 0 : return;
845 : }
846 :
847 : static_always_inline cnat_ep_trk_t *
848 348 : cnat_load_balance (const cnat_translation_t *ct, ip_address_family_t af,
849 : ip4_header_t *ip4, ip6_header_t *ip6, u32 *dpoi_index)
850 : {
851 348 : cnat_main_t *cm = &cnat_main;
852 : const load_balance_t *lb0;
853 : const dpo_id_t *dpo0;
854 : u32 hash_c0, bucket0;
855 :
856 348 : lb0 = load_balance_get (ct->ct_lb.dpoi_index);
857 348 : if (PREDICT_FALSE (!lb0->lb_n_buckets))
858 0 : return (NULL);
859 :
860 : /* session table miss */
861 348 : hash_c0 = (AF_IP4 == af ? ip4_compute_flow_hash (ip4, lb0->lb_hash_config) :
862 174 : ip6_compute_flow_hash (ip6, lb0->lb_hash_config));
863 :
864 348 : if (PREDICT_FALSE (ct->lb_type == CNAT_LB_MAGLEV))
865 0 : bucket0 = ct->lb_maglev[hash_c0 % cm->maglev_len];
866 : else
867 348 : bucket0 = hash_c0 % lb0->lb_n_buckets;
868 :
869 348 : dpo0 = load_balance_get_fwd_bucket (lb0, bucket0);
870 :
871 348 : *dpoi_index = dpo0->dpoi_index;
872 :
873 348 : return &ct->ct_active_paths[bucket0];
874 : }
875 :
876 : /**
877 : * Create NAT sessions
878 : * rsession_location is the location the (return) session will be
879 : * matched at
880 : */
881 : static_always_inline void
882 368 : cnat_session_create (cnat_session_t *session, cnat_node_ctx_t *ctx,
883 : cnat_session_location_t rsession_location,
884 : u8 rsession_flags)
885 : {
886 : cnat_client_t *cc;
887 : cnat_bihash_kv_t rkey;
888 368 : cnat_session_t *rsession = (cnat_session_t *) & rkey;
889 368 : cnat_bihash_kv_t *bkey = (cnat_bihash_kv_t *) session;
890 368 : int rv, n_retries = 0;
891 : static u32 sport_seed = 0;
892 :
893 368 : session->value.cs_ts_index = cnat_timestamp_new (ctx->now);
894 :
895 : /* First create the return session */
896 368 : ip46_address_copy (&rsession->key.cs_ip[VLIB_RX],
897 368 : &session->value.cs_ip[VLIB_TX]);
898 368 : ip46_address_copy (&rsession->key.cs_ip[VLIB_TX],
899 368 : &session->value.cs_ip[VLIB_RX]);
900 368 : rsession->key.cs_proto = session->key.cs_proto;
901 368 : rsession->key.cs_loc = rsession_location;
902 368 : rsession->key.__cs_pad = 0;
903 368 : rsession->key.cs_af = ctx->af;
904 368 : rsession->key.cs_port[VLIB_RX] = session->value.cs_port[VLIB_TX];
905 368 : rsession->key.cs_port[VLIB_TX] = session->value.cs_port[VLIB_RX];
906 :
907 368 : ip46_address_copy (&rsession->value.cs_ip[VLIB_RX],
908 368 : &session->key.cs_ip[VLIB_TX]);
909 368 : ip46_address_copy (&rsession->value.cs_ip[VLIB_TX],
910 368 : &session->key.cs_ip[VLIB_RX]);
911 368 : rsession->value.cs_ts_index = session->value.cs_ts_index;
912 368 : rsession->value.cs_lbi = INDEX_INVALID;
913 368 : rsession->value.flags = rsession_flags | CNAT_SESSION_IS_RETURN;
914 368 : rsession->value.cs_port[VLIB_TX] = session->key.cs_port[VLIB_RX];
915 368 : rsession->value.cs_port[VLIB_RX] = session->key.cs_port[VLIB_TX];
916 :
917 368 : retry_add_ression:
918 368 : rv = cnat_bihash_add_del (&cnat_session_db, &rkey,
919 : 2 /* add but don't overwrite */);
920 368 : if (rv)
921 : {
922 270 : if (!(rsession_flags & CNAT_SESSION_RETRY_SNAT))
923 270 : return;
924 :
925 : /* return session add failed pick an new random src port */
926 0 : rsession->value.cs_port[VLIB_TX] = session->key.cs_port[VLIB_RX] =
927 0 : random_u32 (&sport_seed);
928 0 : if (n_retries++ < 100)
929 0 : goto retry_add_ression;
930 : else
931 : {
932 0 : clib_warning ("Could not find a free port after 100 tries");
933 : /* translate this packet, but don't create state */
934 0 : return;
935 : }
936 : }
937 :
938 98 : cnat_bihash_add_del (&cnat_session_db, bkey, 1 /* add */);
939 :
940 98 : if (!(rsession_flags & CNAT_SESSION_FLAG_NO_CLIENT))
941 : {
942 : /* is this the first time we've seen this source address */
943 196 : cc = (AF_IP4 == ctx->af ?
944 98 : cnat_client_ip4_find (&session->value.cs_ip[VLIB_RX].ip4) :
945 54 : cnat_client_ip6_find (&session->value.cs_ip[VLIB_RX].ip6));
946 :
947 98 : if (NULL == cc)
948 : {
949 : ip_address_t addr;
950 : uword *p;
951 : u32 refcnt;
952 :
953 24 : addr.version = ctx->af;
954 24 : ip46_address_copy (&addr.ip, &session->value.cs_ip[VLIB_RX]);
955 :
956 : /* Throttle */
957 24 : clib_spinlock_lock (&cnat_client_db.throttle_lock);
958 :
959 24 : p = hash_get_mem (cnat_client_db.throttle_mem, &addr);
960 24 : if (p)
961 : {
962 0 : refcnt = p[0] + 1;
963 0 : hash_set_mem (cnat_client_db.throttle_mem, &addr, refcnt);
964 : }
965 : else
966 24 : hash_set_mem_alloc (&cnat_client_db.throttle_mem, &addr, 0);
967 :
968 24 : clib_spinlock_unlock (&cnat_client_db.throttle_lock);
969 :
970 : /* fire client create to the main thread */
971 24 : if (!p)
972 24 : vl_api_rpc_call_main_thread (cnat_client_learn, (u8 *) &addr,
973 : sizeof (addr));
974 : }
975 : else
976 : {
977 : /* Refcount reverse session */
978 74 : cnat_client_cnt_session (cc);
979 : }
980 : }
981 :
982 : }
983 :
984 : always_inline uword
985 368 : cnat_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
986 : vlib_frame_t *frame, cnat_node_sub_t cnat_sub,
987 : ip_address_family_t af, cnat_session_location_t cs_loc,
988 : u8 do_trace)
989 : {
990 : u32 n_left, *from, thread_index;
991 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
992 368 : vlib_buffer_t **b = bufs;
993 : u16 nexts[VLIB_FRAME_SIZE], *next;
994 : f64 now;
995 :
996 368 : thread_index = vm->thread_index;
997 368 : from = vlib_frame_vector_args (frame);
998 368 : n_left = frame->n_vectors;
999 368 : next = nexts;
1000 368 : vlib_get_buffers (vm, from, bufs, n_left);
1001 368 : now = vlib_time_now (vm);
1002 : cnat_session_t *session[4];
1003 : cnat_bihash_kv_t bkey[4], bvalue[4];
1004 : u64 hash[4];
1005 : int rv[4];
1006 :
1007 368 : cnat_node_ctx_t ctx = { now, thread_index, af, do_trace };
1008 :
1009 368 : if (n_left >= 8)
1010 : {
1011 : /* Kickstart our state */
1012 368 : cnat_session_make_key (b[3], af, cs_loc, &bkey[3]);
1013 368 : cnat_session_make_key (b[2], af, cs_loc, &bkey[2]);
1014 368 : cnat_session_make_key (b[1], af, cs_loc, &bkey[1]);
1015 368 : cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
1016 :
1017 368 : hash[3] = cnat_bihash_hash (&bkey[3]);
1018 368 : hash[2] = cnat_bihash_hash (&bkey[2]);
1019 368 : hash[1] = cnat_bihash_hash (&bkey[1]);
1020 368 : hash[0] = cnat_bihash_hash (&bkey[0]);
1021 : }
1022 :
1023 1104 : while (n_left >= 8)
1024 : {
1025 736 : if (n_left >= 12)
1026 : {
1027 368 : vlib_prefetch_buffer_header (b[11], LOAD);
1028 368 : vlib_prefetch_buffer_header (b[10], LOAD);
1029 368 : vlib_prefetch_buffer_header (b[9], LOAD);
1030 368 : vlib_prefetch_buffer_header (b[8], LOAD);
1031 : }
1032 :
1033 736 : rv[3] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[3], &bkey[3],
1034 : &bvalue[3]);
1035 736 : session[3] = (cnat_session_t *) (rv[3] ? &bkey[3] : &bvalue[3]);
1036 736 : next[3] = cnat_sub (vm, node, b[3], &ctx, rv[3], session[3]);
1037 :
1038 736 : rv[2] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[2], &bkey[2],
1039 : &bvalue[2]);
1040 736 : session[2] = (cnat_session_t *) (rv[2] ? &bkey[2] : &bvalue[2]);
1041 736 : next[2] = cnat_sub (vm, node, b[2], &ctx, rv[2], session[2]);
1042 :
1043 736 : rv[1] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[1], &bkey[1],
1044 : &bvalue[1]);
1045 736 : session[1] = (cnat_session_t *) (rv[1] ? &bkey[1] : &bvalue[1]);
1046 736 : next[1] = cnat_sub (vm, node, b[1], &ctx, rv[1], session[1]);
1047 :
1048 736 : rv[0] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[0], &bkey[0],
1049 : &bvalue[0]);
1050 736 : session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
1051 736 : next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
1052 :
1053 736 : cnat_session_make_key (b[7], af, cs_loc, &bkey[3]);
1054 736 : cnat_session_make_key (b[6], af, cs_loc, &bkey[2]);
1055 736 : cnat_session_make_key (b[5], af, cs_loc, &bkey[1]);
1056 736 : cnat_session_make_key (b[4], af, cs_loc, &bkey[0]);
1057 :
1058 736 : hash[3] = cnat_bihash_hash (&bkey[3]);
1059 736 : hash[2] = cnat_bihash_hash (&bkey[2]);
1060 736 : hash[1] = cnat_bihash_hash (&bkey[1]);
1061 736 : hash[0] = cnat_bihash_hash (&bkey[0]);
1062 :
1063 736 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[3]);
1064 736 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[2]);
1065 736 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[1]);
1066 736 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[0]);
1067 :
1068 736 : cnat_bihash_prefetch_data (&cnat_session_db, hash[3]);
1069 736 : cnat_bihash_prefetch_data (&cnat_session_db, hash[2]);
1070 736 : cnat_bihash_prefetch_data (&cnat_session_db, hash[1]);
1071 736 : cnat_bihash_prefetch_data (&cnat_session_db, hash[0]);
1072 :
1073 736 : b += 4;
1074 736 : next += 4;
1075 736 : n_left -= 4;
1076 : }
1077 :
1078 2944 : while (n_left > 0)
1079 : {
1080 2576 : cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
1081 2576 : rv[0] = cnat_bihash_search_i2 (&cnat_session_db, &bkey[0], &bvalue[0]);
1082 :
1083 2576 : session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
1084 2576 : next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
1085 :
1086 2576 : b++;
1087 2576 : next++;
1088 2576 : n_left--;
1089 : }
1090 :
1091 368 : vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1092 :
1093 368 : return frame->n_vectors;
1094 : }
1095 :
1096 : /*
1097 : * fd.io coding-style-patch-verification: ON
1098 : *
1099 : * Local Variables:
1100 : * eval: (c-set-style "gnu")
1101 : * End:
1102 : */
1103 :
1104 : #endif
|
