Line data Source code
1 : /*
2 : * Copyright (c) 2020 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef __CNAT_NODE_H__
17 : #define __CNAT_NODE_H__
18 :
19 : #include <vlibmemory/api.h>
20 : #include <vnet/dpo/load_balance.h>
21 : #include <vnet/dpo/load_balance_map.h>
22 : #include <vnet/ip/ip_psh_cksum.h>
23 :
24 : #include <cnat/cnat_session.h>
25 : #include <cnat/cnat_client.h>
26 : #include <cnat/cnat_inline.h>
27 : #include <cnat/cnat_translation.h>
28 :
29 : #include <vnet/ip/ip4_inlines.h>
30 : #include <vnet/ip/ip6_inlines.h>
31 :
32 : typedef uword (*cnat_node_sub_t) (vlib_main_t * vm,
33 : vlib_node_runtime_t * node,
34 : vlib_buffer_t * b,
35 : cnat_node_ctx_t * ctx, int rv,
36 : cnat_session_t * session);
37 :
38 : typedef struct cnat_trace_element_t_
39 : {
40 : cnat_session_t session;
41 : cnat_translation_t tr;
42 : u32 sw_if_index[VLIB_N_RX_TX];
43 : u32 snat_policy_result;
44 : u8 flags;
45 : } cnat_trace_element_t;
46 :
47 : typedef enum cnat_trace_element_flag_t_
48 : {
49 : CNAT_TRACE_SESSION_FOUND = (1 << 0),
50 : CNAT_TRACE_SESSION_CREATED = (1 << 1),
51 : CNAT_TRACE_TRANSLATION_FOUND = (1 << 2),
52 : CNAT_TRACE_NO_NAT = (1 << 3),
53 : } cnat_trace_element_flag_t;
54 :
55 : static_always_inline void
56 5880 : cnat_add_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b,
57 : cnat_session_t *session, const cnat_translation_t *ct,
58 : u8 flags)
59 : {
60 : cnat_trace_element_t *t;
61 5880 : if (NULL != ct)
62 120 : flags |= CNAT_TRACE_TRANSLATION_FOUND;
63 :
64 5880 : t = vlib_add_trace (vm, node, b, sizeof (*t));
65 5880 : t->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_RX];
66 5880 : t->sw_if_index[VLIB_TX] = vnet_buffer (b)->sw_if_index[VLIB_TX];
67 :
68 5880 : if (flags & (CNAT_TRACE_SESSION_FOUND | CNAT_TRACE_SESSION_CREATED))
69 4590 : clib_memcpy (&t->session, session, sizeof (t->session));
70 5880 : if (flags & CNAT_TRACE_TRANSLATION_FOUND)
71 120 : clib_memcpy (&t->tr, ct, sizeof (cnat_translation_t));
72 5880 : t->flags = flags;
73 5880 : }
74 :
75 : static u8 *
76 6015 : format_cnat_trace (u8 *s, va_list *args)
77 : {
78 6015 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
79 6015 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
80 6015 : cnat_trace_element_t *t = va_arg (*args, cnat_trace_element_t *);
81 6015 : u32 indent = format_get_indent (s);
82 6015 : vnet_main_t *vnm = vnet_get_main ();
83 :
84 6015 : if (t->flags & CNAT_TRACE_SESSION_CREATED)
85 143 : s = format (s, "created session");
86 5872 : else if (t->flags & CNAT_TRACE_SESSION_FOUND)
87 4552 : s = format (s, "found session");
88 : else
89 1320 : s = format (s, "session not found");
90 :
91 6015 : if (t->flags & (CNAT_TRACE_NO_NAT))
92 0 : s = format (s, " [policy:skip]");
93 :
94 6015 : s = format (s, "\n%Uin:%U out:%U ", format_white_space, indent,
95 : format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_RX],
96 : format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_TX]);
97 :
98 6015 : if (t->flags & (CNAT_TRACE_SESSION_CREATED | CNAT_TRACE_SESSION_FOUND))
99 4695 : s = format (s, "\n%U%U", format_white_space, indent, format_cnat_session,
100 : &t->session, 1);
101 :
102 6015 : if (t->flags & CNAT_TRACE_TRANSLATION_FOUND)
103 123 : s = format (s, "\n%Utranslation: %U", format_white_space, indent,
104 : format_cnat_translation, &t->tr, 0);
105 :
106 6015 : return s;
107 : }
108 :
109 : static_always_inline u8
110 295 : icmp_type_is_error_message (u8 icmp_type)
111 : {
112 295 : switch (icmp_type)
113 : {
114 227 : case ICMP4_destination_unreachable:
115 : case ICMP4_time_exceeded:
116 : case ICMP4_parameter_problem:
117 : case ICMP4_source_quench:
118 : case ICMP4_redirect:
119 : case ICMP4_alternate_host_address:
120 227 : return 1;
121 : }
122 68 : return 0;
123 : }
124 :
125 : static_always_inline u8
126 68 : icmp_type_is_echo (u8 icmp_type)
127 : {
128 68 : switch (icmp_type)
129 : {
130 68 : case ICMP4_echo_request:
131 : case ICMP4_echo_reply:
132 68 : return 1;
133 : }
134 0 : return 0;
135 : }
136 :
137 : static_always_inline u8
138 204 : icmp6_type_is_echo (u8 icmp_type)
139 : {
140 204 : switch (icmp_type)
141 : {
142 204 : case ICMP6_echo_request:
143 : case ICMP6_echo_reply:
144 204 : return 1;
145 : }
146 0 : return 0;
147 : }
148 :
149 : static_always_inline u8
150 702 : icmp6_type_is_error_message (u8 icmp_type)
151 : {
152 702 : switch (icmp_type)
153 : {
154 498 : case ICMP6_destination_unreachable:
155 : case ICMP6_time_exceeded:
156 : case ICMP6_parameter_problem:
157 498 : return 1;
158 : }
159 204 : return 0;
160 : }
161 :
162 : static_always_inline u8
163 135 : cmp_ip6_address (const ip6_address_t * a1, const ip6_address_t * a2)
164 : {
165 135 : return ((a1->as_u64[0] == a2->as_u64[0])
166 135 : && (a1->as_u64[1] == a2->as_u64[1]));
167 : }
168 :
169 : /**
170 : * Inline translation functions
171 : */
172 :
173 : static_always_inline u16
174 0 : ip4_pseudo_header_cksum2 (ip4_header_t *ip4, ip4_address_t address[VLIB_N_DIR])
175 : {
176 0 : ip4_psh_t psh = { 0 };
177 0 : psh.src = address[VLIB_RX];
178 0 : psh.dst = address[VLIB_TX];
179 0 : psh.proto = ip4->protocol;
180 0 : psh.l4len = clib_host_to_net_u16 (clib_net_to_host_u16 (ip4->length) -
181 : sizeof (ip4_header_t));
182 0 : return ~(clib_ip_csum ((u8 *) &psh, sizeof (ip4_psh_t)));
183 : }
184 :
185 : static_always_inline void
186 2175 : cnat_ip4_translate_l4 (ip4_header_t *ip4, udp_header_t *udp, ip_csum_t *sum,
187 : ip4_address_t new_addr[VLIB_N_DIR],
188 : u16 new_port[VLIB_N_DIR], u32 oflags)
189 : {
190 : u16 old_port[VLIB_N_DIR];
191 2175 : old_port[VLIB_TX] = udp->dst_port;
192 2175 : old_port[VLIB_RX] = udp->src_port;
193 :
194 2175 : udp->dst_port = new_port[VLIB_TX];
195 2175 : udp->src_port = new_port[VLIB_RX];
196 :
197 2175 : if (oflags &
198 : (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
199 : {
200 0 : *sum = ip4_pseudo_header_cksum2 (ip4, new_addr);
201 0 : return;
202 : }
203 :
204 2175 : *sum = ip_csum_update (*sum, ip4->dst_address.as_u32,
205 : new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
206 2175 : *sum = ip_csum_update (*sum, ip4->src_address.as_u32,
207 : new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
208 :
209 2175 : *sum = ip_csum_update (*sum, old_port[VLIB_TX], new_port[VLIB_TX],
210 : udp_header_t, dst_port);
211 2175 : *sum = ip_csum_update (*sum, old_port[VLIB_RX], new_port[VLIB_RX],
212 : udp_header_t, src_port);
213 : }
214 :
215 : static_always_inline void
216 0 : cnat_ip4_translate_sctp (ip4_header_t *ip4, sctp_header_t *sctp,
217 : u16 new_port[VLIB_N_DIR])
218 : {
219 : /* Fastpath no checksum */
220 0 : if (PREDICT_TRUE (0 == sctp->checksum))
221 : {
222 0 : sctp->dst_port = new_port[VLIB_TX];
223 0 : sctp->src_port = new_port[VLIB_RX];
224 0 : return;
225 : }
226 :
227 0 : if (new_port[VLIB_TX])
228 0 : sctp->dst_port = new_port[VLIB_TX];
229 0 : if (new_port[VLIB_RX])
230 0 : sctp->src_port = new_port[VLIB_RX];
231 :
232 0 : sctp->checksum = 0;
233 0 : sctp->checksum = clib_host_to_little_u32 (~clib_crc32c_with_init (
234 0 : (u8 *) sctp, ntohs (ip4->length) - sizeof (ip4_header_t),
235 : ~0 /* init value */));
236 : }
237 :
238 : static_always_inline void
239 2280 : cnat_ip4_translate_l3 (ip4_header_t *ip4, ip4_address_t new_addr[VLIB_N_DIR],
240 : u32 oflags)
241 : {
242 : ip4_address_t old_addr[VLIB_N_DIR];
243 : ip_csum_t sum;
244 2280 : old_addr[VLIB_TX] = ip4->dst_address;
245 2280 : old_addr[VLIB_RX] = ip4->src_address;
246 :
247 2280 : ip4->dst_address = new_addr[VLIB_TX];
248 2280 : ip4->src_address = new_addr[VLIB_RX];
249 :
250 : // We always compute the IP checksum even if oflags &
251 : // VNET_BUFFER_OFFLOAD_F_IP_CKSUM is set as this is relatively inexpensive
252 : // and will allow avoiding issues in driver that do not behave properly
253 : // downstream.
254 2280 : sum = ip4->checksum;
255 2280 : sum = ip_csum_update (sum, old_addr[VLIB_TX].as_u32,
256 : new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
257 2280 : sum = ip_csum_update (sum, old_addr[VLIB_RX].as_u32,
258 : new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
259 2280 : ip4->checksum = ip_csum_fold (sum);
260 2280 : }
261 :
262 : static_always_inline void
263 2820 : cnat_tcp_update_session_lifetime (tcp_header_t * tcp, u32 index)
264 : {
265 2820 : cnat_main_t *cm = &cnat_main;
266 2820 : if (PREDICT_FALSE (tcp_fin (tcp)))
267 0 : cnat_timestamp_set_lifetime (index, CNAT_DEFAULT_TCP_RST_TIMEOUT);
268 :
269 2820 : if (PREDICT_FALSE (tcp_rst (tcp)))
270 0 : cnat_timestamp_set_lifetime (index, CNAT_DEFAULT_TCP_RST_TIMEOUT);
271 :
272 2820 : if (PREDICT_FALSE (tcp_syn (tcp) && tcp_ack (tcp)))
273 0 : cnat_timestamp_set_lifetime (index, cm->tcp_max_age);
274 2820 : }
275 :
276 : static_always_inline void
277 30 : cnat_translation_icmp4_echo (ip4_header_t *ip4, icmp46_header_t *icmp,
278 : ip4_address_t new_addr[VLIB_N_DIR],
279 : u16 new_port[VLIB_N_DIR], u32 oflags)
280 : {
281 : ip_csum_t sum;
282 : u16 old_port;
283 30 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
284 :
285 30 : cnat_ip4_translate_l3 (ip4, new_addr, oflags);
286 30 : old_port = echo->identifier;
287 30 : echo->identifier = new_port[VLIB_RX];
288 :
289 30 : sum = icmp->checksum;
290 : sum =
291 30 : ip_csum_update (sum, old_port, new_port[VLIB_RX], udp_header_t, src_port);
292 :
293 30 : icmp->checksum = ip_csum_fold (sum);
294 30 : }
295 :
296 : static_always_inline void
297 75 : cnat_translation_icmp4_error (ip4_header_t *outer_ip4, icmp46_header_t *icmp,
298 : ip4_address_t outer_new_addr[VLIB_N_DIR],
299 : u16 outer_new_port[VLIB_N_DIR], u8 snat_outer_ip,
300 : u32 oflags)
301 : {
302 : ip4_address_t new_addr[VLIB_N_DIR];
303 : ip4_address_t old_addr[VLIB_N_DIR];
304 : u16 new_port[VLIB_N_DIR];
305 : u16 old_port[VLIB_N_DIR];
306 : ip_csum_t sum, old_ip_sum, inner_l4_sum, inner_l4_old_sum;
307 :
308 75 : ip4_header_t *ip4 = (ip4_header_t *) (icmp + 2);
309 75 : udp_header_t *udp = (udp_header_t *) (ip4 + 1);
310 75 : tcp_header_t *tcp = (tcp_header_t *) udp;
311 :
312 : /* Swap inner ports */
313 75 : new_addr[VLIB_TX] = outer_new_addr[VLIB_RX];
314 75 : new_addr[VLIB_RX] = outer_new_addr[VLIB_TX];
315 75 : new_port[VLIB_TX] = outer_new_port[VLIB_RX];
316 75 : new_port[VLIB_RX] = outer_new_port[VLIB_TX];
317 :
318 75 : old_addr[VLIB_TX] = ip4->dst_address;
319 75 : old_addr[VLIB_RX] = ip4->src_address;
320 75 : old_port[VLIB_RX] = udp->src_port;
321 75 : old_port[VLIB_TX] = udp->dst_port;
322 :
323 75 : sum = icmp->checksum;
324 75 : old_ip_sum = ip4->checksum;
325 :
326 : /* translate outer ip. */
327 75 : if (!snat_outer_ip)
328 0 : outer_new_addr[VLIB_RX] = outer_ip4->src_address;
329 75 : cnat_ip4_translate_l3 (outer_ip4, outer_new_addr, oflags);
330 :
331 75 : if (ip4->protocol == IP_PROTOCOL_TCP)
332 : {
333 45 : inner_l4_old_sum = inner_l4_sum = tcp->checksum;
334 45 : cnat_ip4_translate_l4 (ip4, udp, &inner_l4_sum, new_addr, new_port,
335 : 0 /* flags */);
336 45 : tcp->checksum = ip_csum_fold (inner_l4_sum);
337 : }
338 30 : else if (ip4->protocol == IP_PROTOCOL_UDP)
339 : {
340 30 : inner_l4_old_sum = inner_l4_sum = udp->checksum;
341 30 : cnat_ip4_translate_l4 (ip4, udp, &inner_l4_sum, new_addr, new_port,
342 : 0 /* flags */);
343 30 : udp->checksum = ip_csum_fold (inner_l4_sum);
344 : }
345 : else
346 0 : return;
347 :
348 : /* UDP/TCP checksum changed */
349 75 : sum = ip_csum_update (sum, inner_l4_old_sum, inner_l4_sum,
350 : ip4_header_t, checksum);
351 :
352 : /* UDP/TCP Ports changed */
353 75 : if (old_port[VLIB_TX] && new_port[VLIB_TX])
354 75 : sum = ip_csum_update (sum, old_port[VLIB_TX], new_port[VLIB_TX],
355 : udp_header_t, dst_port);
356 :
357 75 : if (old_port[VLIB_RX] && new_port[VLIB_RX])
358 75 : sum = ip_csum_update (sum, old_port[VLIB_RX], new_port[VLIB_RX],
359 : udp_header_t, src_port);
360 :
361 75 : cnat_ip4_translate_l3 (ip4, new_addr, 0 /* oflags */);
362 75 : ip_csum_t new_ip_sum = ip4->checksum;
363 : /* IP checksum changed */
364 75 : sum = ip_csum_update (sum, old_ip_sum, new_ip_sum, ip4_header_t, checksum);
365 :
366 : /* IP src/dst addr changed */
367 75 : sum = ip_csum_update (sum, old_addr[VLIB_TX].as_u32,
368 : new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
369 :
370 75 : sum = ip_csum_update (sum, old_addr[VLIB_RX].as_u32,
371 : new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
372 :
373 75 : icmp->checksum = ip_csum_fold (sum);
374 : }
375 :
376 : static_always_inline void
377 2205 : cnat_translation_ip4 (const cnat_session_t *session, ip4_header_t *ip4,
378 : udp_header_t *udp, u32 oflags)
379 : {
380 2205 : tcp_header_t *tcp = (tcp_header_t *) udp;
381 : ip4_address_t new_addr[VLIB_N_DIR];
382 : u16 new_port[VLIB_N_DIR];
383 :
384 2205 : new_addr[VLIB_TX] = session->value.cs_ip[VLIB_TX].ip4;
385 2205 : new_addr[VLIB_RX] = session->value.cs_ip[VLIB_RX].ip4;
386 2205 : new_port[VLIB_TX] = session->value.cs_port[VLIB_TX];
387 2205 : new_port[VLIB_RX] = session->value.cs_port[VLIB_RX];
388 :
389 2205 : if (ip4->protocol == IP_PROTOCOL_TCP)
390 : {
391 1485 : ip_csum_t sum = tcp->checksum;
392 1485 : cnat_ip4_translate_l4 (ip4, udp, &sum, new_addr, new_port, oflags);
393 1485 : tcp->checksum = ip_csum_fold (sum);
394 1485 : cnat_ip4_translate_l3 (ip4, new_addr, oflags);
395 1485 : cnat_tcp_update_session_lifetime (tcp, session->value.cs_ts_index);
396 : }
397 720 : else if (ip4->protocol == IP_PROTOCOL_UDP)
398 : {
399 615 : ip_csum_t sum = udp->checksum;
400 615 : cnat_ip4_translate_l4 (ip4, udp, &sum, new_addr, new_port, oflags);
401 615 : udp->checksum = ip_csum_fold (sum);
402 615 : cnat_ip4_translate_l3 (ip4, new_addr, oflags);
403 : }
404 105 : else if (ip4->protocol == IP_PROTOCOL_SCTP)
405 : {
406 0 : sctp_header_t *sctp = (sctp_header_t *) udp;
407 0 : cnat_ip4_translate_sctp (ip4, sctp, new_port);
408 0 : cnat_ip4_translate_l3 (ip4, new_addr, oflags);
409 : }
410 105 : else if (ip4->protocol == IP_PROTOCOL_ICMP)
411 : {
412 105 : icmp46_header_t *icmp = (icmp46_header_t *) udp;
413 105 : if (icmp_type_is_error_message (icmp->type))
414 : {
415 : /* SNAT only if src_addr was translated */
416 75 : u8 snat_outer_ip =
417 75 : (ip4->src_address.as_u32 ==
418 75 : session->key.cs_ip[VLIB_RX].ip4.as_u32);
419 75 : cnat_translation_icmp4_error (ip4, icmp, new_addr, new_port,
420 : snat_outer_ip, oflags);
421 : }
422 30 : else if (icmp_type_is_echo (icmp->type))
423 30 : cnat_translation_icmp4_echo (ip4, icmp, new_addr, new_port, oflags);
424 : }
425 2205 : }
426 :
427 : static_always_inline void
428 2520 : cnat_ip6_translate_l3 (ip6_header_t * ip6, ip6_address_t new_addr[VLIB_N_DIR])
429 : {
430 2520 : ip6_address_copy (&ip6->dst_address, &new_addr[VLIB_TX]);
431 2520 : ip6_address_copy (&ip6->src_address, &new_addr[VLIB_RX]);
432 2520 : }
433 :
434 : static_always_inline u16
435 0 : ip6_pseudo_header_cksum2 (ip6_header_t *ip6, ip6_address_t address[VLIB_N_DIR])
436 : {
437 0 : ip6_psh_t psh = { 0 };
438 0 : psh.src = address[VLIB_RX];
439 0 : psh.dst = address[VLIB_TX];
440 0 : psh.l4len = ip6->payload_length;
441 0 : psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol);
442 0 : return ~(clib_ip_csum ((u8 *) &psh, sizeof (ip6_psh_t)));
443 : }
444 :
445 : static_always_inline void
446 2295 : cnat_ip6_translate_l4 (ip6_header_t *ip6, udp_header_t *udp, ip_csum_t *sum,
447 : ip6_address_t new_addr[VLIB_N_DIR],
448 : u16 new_port[VLIB_N_DIR], u32 oflags)
449 : {
450 : u16 old_port[VLIB_N_DIR];
451 2295 : old_port[VLIB_TX] = udp->dst_port;
452 2295 : old_port[VLIB_RX] = udp->src_port;
453 :
454 2295 : udp->dst_port = new_port[VLIB_TX];
455 2295 : udp->src_port = new_port[VLIB_RX];
456 :
457 2295 : if (oflags &
458 : (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
459 : {
460 0 : *sum = ip6_pseudo_header_cksum2 (ip6, new_addr);
461 0 : return;
462 : }
463 :
464 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_TX].as_u64[0]);
465 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_TX].as_u64[1]);
466 2295 : *sum = ip_csum_sub_even (*sum, ip6->dst_address.as_u64[0]);
467 2295 : *sum = ip_csum_sub_even (*sum, ip6->dst_address.as_u64[1]);
468 :
469 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_RX].as_u64[0]);
470 2295 : *sum = ip_csum_add_even (*sum, new_addr[VLIB_RX].as_u64[1]);
471 2295 : *sum = ip_csum_sub_even (*sum, ip6->src_address.as_u64[0]);
472 2295 : *sum = ip_csum_sub_even (*sum, ip6->src_address.as_u64[1]);
473 :
474 2295 : *sum = ip_csum_update (*sum, old_port[VLIB_TX], new_port[VLIB_TX],
475 : udp_header_t, dst_port);
476 :
477 2295 : *sum = ip_csum_update (*sum, old_port[VLIB_RX], new_port[VLIB_RX],
478 : udp_header_t, src_port);
479 : }
480 :
481 : static_always_inline void
482 90 : cnat_translation_icmp6_echo (ip6_header_t * ip6, icmp46_header_t * icmp,
483 : ip6_address_t new_addr[VLIB_N_DIR],
484 : u16 new_port[VLIB_N_DIR])
485 : {
486 90 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
487 : ip6_address_t old_addr[VLIB_N_DIR];
488 : ip_csum_t sum;
489 : u16 old_port;
490 90 : old_port = echo->identifier;
491 90 : ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
492 90 : ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
493 :
494 90 : sum = icmp->checksum;
495 :
496 90 : cnat_ip6_translate_l3 (ip6, new_addr);
497 :
498 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[0]);
499 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[1]);
500 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[0]);
501 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[1]);
502 :
503 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[0]);
504 90 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[1]);
505 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[0]);
506 90 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[1]);
507 :
508 90 : echo->identifier = new_port[VLIB_RX];
509 : sum =
510 90 : ip_csum_update (sum, old_port, new_port[VLIB_RX], udp_header_t, src_port);
511 :
512 90 : icmp->checksum = ip_csum_fold (sum);
513 90 : }
514 :
515 : static_always_inline void
516 135 : cnat_translation_icmp6_error (ip6_header_t * outer_ip6,
517 : icmp46_header_t * icmp,
518 : ip6_address_t outer_new_addr[VLIB_N_DIR],
519 : u16 outer_new_port[VLIB_N_DIR],
520 : u8 snat_outer_ip)
521 : {
522 : ip6_address_t new_addr[VLIB_N_DIR];
523 : ip6_address_t old_addr[VLIB_N_DIR];
524 : ip6_address_t outer_old_addr[VLIB_N_DIR];
525 : u16 new_port[VLIB_N_DIR];
526 : u16 old_port[VLIB_N_DIR];
527 : ip_csum_t sum, inner_l4_sum, inner_l4_old_sum;
528 :
529 135 : if (!icmp6_type_is_error_message (icmp->type))
530 0 : return;
531 :
532 135 : ip6_header_t *ip6 = (ip6_header_t *) (icmp + 2);
533 135 : udp_header_t *udp = (udp_header_t *) (ip6 + 1);
534 135 : tcp_header_t *tcp = (tcp_header_t *) udp;
535 :
536 : /* Swap inner ports */
537 135 : ip6_address_copy (&new_addr[VLIB_RX], &outer_new_addr[VLIB_TX]);
538 135 : ip6_address_copy (&new_addr[VLIB_TX], &outer_new_addr[VLIB_RX]);
539 135 : new_port[VLIB_TX] = outer_new_port[VLIB_RX];
540 135 : new_port[VLIB_RX] = outer_new_port[VLIB_TX];
541 :
542 135 : ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
543 135 : ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
544 135 : old_port[VLIB_RX] = udp->src_port;
545 135 : old_port[VLIB_TX] = udp->dst_port;
546 :
547 135 : sum = icmp->checksum;
548 : /* Translate outer ip */
549 135 : ip6_address_copy (&outer_old_addr[VLIB_TX], &outer_ip6->dst_address);
550 135 : ip6_address_copy (&outer_old_addr[VLIB_RX], &outer_ip6->src_address);
551 135 : if (!snat_outer_ip)
552 0 : ip6_address_copy (&outer_new_addr[VLIB_RX], &outer_ip6->src_address);
553 135 : cnat_ip6_translate_l3 (outer_ip6, outer_new_addr);
554 :
555 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_TX].as_u64[0]);
556 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_TX].as_u64[1]);
557 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_TX].as_u64[0]);
558 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_TX].as_u64[1]);
559 :
560 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_RX].as_u64[0]);
561 135 : sum = ip_csum_add_even (sum, outer_new_addr[VLIB_RX].as_u64[1]);
562 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_RX].as_u64[0]);
563 135 : sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_RX].as_u64[1]);
564 :
565 : /* Translate inner TCP / UDP */
566 135 : if (ip6->protocol == IP_PROTOCOL_TCP)
567 : {
568 75 : inner_l4_old_sum = inner_l4_sum = tcp->checksum;
569 75 : cnat_ip6_translate_l4 (ip6, udp, &inner_l4_sum, new_addr, new_port,
570 : 0 /* oflags */);
571 75 : tcp->checksum = ip_csum_fold (inner_l4_sum);
572 : }
573 60 : else if (ip6->protocol == IP_PROTOCOL_UDP)
574 : {
575 60 : inner_l4_old_sum = inner_l4_sum = udp->checksum;
576 60 : cnat_ip6_translate_l4 (ip6, udp, &inner_l4_sum, new_addr, new_port,
577 : 0 /* oflags */);
578 60 : udp->checksum = ip_csum_fold (inner_l4_sum);
579 : }
580 : else
581 0 : return;
582 :
583 : /* UDP/TCP checksum changed */
584 135 : sum = ip_csum_update (sum, inner_l4_old_sum, inner_l4_sum, ip4_header_t,
585 : checksum);
586 :
587 : /* UDP/TCP Ports changed */
588 135 : sum = ip_csum_update (sum, old_port[VLIB_TX], new_port[VLIB_TX],
589 : udp_header_t, dst_port);
590 :
591 135 : sum = ip_csum_update (sum, old_port[VLIB_RX], new_port[VLIB_RX],
592 : udp_header_t, src_port);
593 :
594 135 : cnat_ip6_translate_l3 (ip6, new_addr);
595 : /* IP src/dst addr changed */
596 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[0]);
597 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[1]);
598 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[0]);
599 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[1]);
600 :
601 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[0]);
602 135 : sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[1]);
603 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[0]);
604 135 : sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[1]);
605 :
606 135 : icmp->checksum = ip_csum_fold (sum);
607 : }
608 :
609 : static_always_inline void
610 2385 : cnat_translation_ip6 (const cnat_session_t *session, ip6_header_t *ip6,
611 : udp_header_t *udp, u32 oflags)
612 : {
613 2385 : tcp_header_t *tcp = (tcp_header_t *) udp;
614 : ip6_address_t new_addr[VLIB_N_DIR];
615 : u16 new_port[VLIB_N_DIR];
616 :
617 2385 : ip6_address_copy (&new_addr[VLIB_TX], &session->value.cs_ip[VLIB_TX].ip6);
618 2385 : ip6_address_copy (&new_addr[VLIB_RX], &session->value.cs_ip[VLIB_RX].ip6);
619 2385 : new_port[VLIB_TX] = session->value.cs_port[VLIB_TX];
620 2385 : new_port[VLIB_RX] = session->value.cs_port[VLIB_RX];
621 :
622 2385 : if (ip6->protocol == IP_PROTOCOL_TCP)
623 : {
624 1335 : ip_csum_t sum = tcp->checksum;
625 1335 : cnat_ip6_translate_l4 (ip6, udp, &sum, new_addr, new_port, oflags);
626 1335 : tcp->checksum = ip_csum_fold (sum);
627 1335 : cnat_ip6_translate_l3 (ip6, new_addr);
628 1335 : cnat_tcp_update_session_lifetime (tcp, session->value.cs_ts_index);
629 : }
630 1050 : else if (ip6->protocol == IP_PROTOCOL_UDP)
631 : {
632 825 : ip_csum_t sum = udp->checksum;
633 825 : cnat_ip6_translate_l4 (ip6, udp, &sum, new_addr, new_port, oflags);
634 825 : udp->checksum = ip_csum_fold (sum);
635 825 : cnat_ip6_translate_l3 (ip6, new_addr);
636 : }
637 225 : else if (ip6->protocol == IP_PROTOCOL_ICMP6)
638 : {
639 225 : icmp46_header_t *icmp = (icmp46_header_t *) udp;
640 225 : if (icmp6_type_is_error_message (icmp->type))
641 : {
642 : /* SNAT only if src_addr was translated */
643 135 : u8 snat_outer_ip = cmp_ip6_address (&ip6->src_address,
644 : &session->key.
645 : cs_ip[VLIB_RX].ip6);
646 135 : cnat_translation_icmp6_error (ip6, icmp, new_addr, new_port,
647 : snat_outer_ip);
648 : }
649 90 : else if (icmp6_type_is_echo (icmp->type))
650 90 : cnat_translation_icmp6_echo (ip6, icmp, new_addr, new_port);
651 : }
652 2385 : }
653 :
654 : static_always_inline void
655 7448 : cnat_session_make_key (vlib_buffer_t *b, ip_address_family_t af,
656 : cnat_session_location_t cs_loc, cnat_bihash_kv_t *bkey)
657 : {
658 : udp_header_t *udp;
659 7448 : cnat_session_t *session = (cnat_session_t *) bkey;
660 7448 : u32 iph_offset = 0;
661 7448 : session->key.cs_af = af;
662 :
663 7448 : session->key.cs_loc = cs_loc;
664 7448 : session->key.__cs_pad = 0;
665 7448 : if (cs_loc == CNAT_LOCATION_OUTPUT)
666 : /* rewind buffer */
667 0 : iph_offset = vnet_buffer (b)->ip.save_rewrite_length;
668 :
669 7448 : if (AF_IP4 == af)
670 : {
671 : ip4_header_t *ip4;
672 3572 : ip4 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
673 :
674 3572 : if (PREDICT_FALSE (ip4->protocol == IP_PROTOCOL_ICMP))
675 : {
676 190 : icmp46_header_t *icmp = (icmp46_header_t *) (ip4 + 1);
677 190 : if (icmp_type_is_error_message (icmp->type))
678 : {
679 152 : ip4 = (ip4_header_t *) (icmp + 2); /* Use inner packet */
680 152 : udp = (udp_header_t *) (ip4 + 1);
681 : /* Swap dst & src for search as ICMP payload is reversed */
682 152 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
683 152 : &ip4->dst_address);
684 152 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
685 152 : &ip4->src_address);
686 152 : session->key.cs_proto = ip4->protocol;
687 152 : session->key.cs_port[VLIB_TX] = udp->src_port;
688 152 : session->key.cs_port[VLIB_RX] = udp->dst_port;
689 : }
690 38 : else if (icmp_type_is_echo (icmp->type))
691 : {
692 38 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
693 38 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
694 38 : &ip4->dst_address);
695 38 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
696 38 : &ip4->src_address);
697 38 : session->key.cs_proto = ip4->protocol;
698 38 : session->key.cs_port[VLIB_TX] = echo->identifier;
699 38 : session->key.cs_port[VLIB_RX] = echo->identifier;
700 : }
701 : else
702 0 : goto error;
703 : }
704 3382 : else if (ip4->protocol == IP_PROTOCOL_UDP ||
705 2356 : ip4->protocol == IP_PROTOCOL_TCP)
706 : {
707 3382 : udp = (udp_header_t *) (ip4 + 1);
708 3382 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
709 3382 : &ip4->dst_address);
710 3382 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
711 3382 : &ip4->src_address);
712 3382 : session->key.cs_proto = ip4->protocol;
713 3382 : session->key.cs_port[VLIB_RX] = udp->src_port;
714 3382 : session->key.cs_port[VLIB_TX] = udp->dst_port;
715 : }
716 0 : else if (ip4->protocol == IP_PROTOCOL_SCTP)
717 : {
718 : sctp_header_t *sctp;
719 0 : sctp = (sctp_header_t *) (ip4 + 1);
720 0 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
721 0 : &ip4->dst_address);
722 0 : ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
723 0 : &ip4->src_address);
724 0 : session->key.cs_proto = ip4->protocol;
725 0 : session->key.cs_port[VLIB_RX] = sctp->src_port;
726 0 : session->key.cs_port[VLIB_TX] = sctp->dst_port;
727 : }
728 : else
729 0 : goto error;
730 : }
731 : else
732 : {
733 : ip6_header_t *ip6;
734 3876 : ip6 = (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
735 3876 : if (PREDICT_FALSE (ip6->protocol == IP_PROTOCOL_ICMP6))
736 : {
737 342 : icmp46_header_t *icmp = (icmp46_header_t *) (ip6 + 1);
738 342 : if (icmp6_type_is_error_message (icmp->type))
739 : {
740 228 : ip6 = (ip6_header_t *) (icmp + 2); /* Use inner packet */
741 228 : udp = (udp_header_t *) (ip6 + 1);
742 : /* Swap dst & src for search as ICMP payload is reversed */
743 228 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
744 228 : &ip6->dst_address);
745 228 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
746 228 : &ip6->src_address);
747 228 : session->key.cs_proto = ip6->protocol;
748 228 : session->key.cs_port[VLIB_TX] = udp->src_port;
749 228 : session->key.cs_port[VLIB_RX] = udp->dst_port;
750 : }
751 114 : else if (icmp6_type_is_echo (icmp->type))
752 : {
753 114 : cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
754 114 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
755 114 : &ip6->dst_address);
756 114 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
757 114 : &ip6->src_address);
758 114 : session->key.cs_proto = ip6->protocol;
759 114 : session->key.cs_port[VLIB_TX] = echo->identifier;
760 114 : session->key.cs_port[VLIB_RX] = echo->identifier;
761 : }
762 : else
763 0 : goto error;
764 : }
765 3534 : else if (ip6->protocol == IP_PROTOCOL_UDP ||
766 2204 : ip6->protocol == IP_PROTOCOL_TCP)
767 : {
768 3534 : udp = (udp_header_t *) (ip6 + 1);
769 3534 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
770 3534 : &ip6->dst_address);
771 3534 : ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
772 3534 : &ip6->src_address);
773 3534 : session->key.cs_port[VLIB_RX] = udp->src_port;
774 3534 : session->key.cs_port[VLIB_TX] = udp->dst_port;
775 3534 : session->key.cs_proto = ip6->protocol;
776 : }
777 : else
778 0 : goto error;
779 : }
780 7448 : return;
781 :
782 0 : error:
783 : /* Ensure we dont find anything */
784 0 : session->key.cs_proto = 0;
785 0 : return;
786 : }
787 :
788 : static_always_inline cnat_ep_trk_t *
789 120 : cnat_load_balance (const cnat_translation_t *ct, ip_address_family_t af,
790 : ip4_header_t *ip4, ip6_header_t *ip6, u32 *dpoi_index)
791 : {
792 120 : cnat_main_t *cm = &cnat_main;
793 : const load_balance_t *lb0;
794 : const dpo_id_t *dpo0;
795 : u32 hash_c0, bucket0;
796 :
797 120 : lb0 = load_balance_get (ct->ct_lb.dpoi_index);
798 120 : if (PREDICT_FALSE (!lb0->lb_n_buckets))
799 0 : return (NULL);
800 :
801 : /* session table miss */
802 120 : hash_c0 = (AF_IP4 == af ? ip4_compute_flow_hash (ip4, lb0->lb_hash_config) :
803 48 : ip6_compute_flow_hash (ip6, lb0->lb_hash_config));
804 :
805 120 : if (PREDICT_FALSE (ct->lb_type == CNAT_LB_MAGLEV))
806 0 : bucket0 = ct->lb_maglev[hash_c0 % cm->maglev_len];
807 : else
808 120 : bucket0 = hash_c0 % lb0->lb_n_buckets;
809 :
810 120 : dpo0 = load_balance_get_fwd_bucket (lb0, bucket0);
811 :
812 120 : *dpoi_index = dpo0->dpoi_index;
813 :
814 120 : return &ct->ct_active_paths[bucket0];
815 : }
816 :
817 : /**
818 : * Create NAT sessions
819 : * rsession_location is the location the (return) session will be
820 : * matched at
821 : */
822 :
823 : static_always_inline void
824 140 : cnat_session_create (cnat_session_t *session, cnat_node_ctx_t *ctx)
825 : {
826 140 : cnat_bihash_kv_t *bkey = (cnat_bihash_kv_t *) session;
827 :
828 140 : session->value.cs_ts_index = cnat_timestamp_new (ctx->now);
829 140 : cnat_bihash_add_del (&cnat_session_db, bkey, 1);
830 140 : }
831 :
832 : static_always_inline void
833 140 : cnat_rsession_create (cnat_session_t *session, cnat_node_ctx_t *ctx,
834 : cnat_session_location_t rsession_location,
835 : cnat_session_flag_t rsession_flags)
836 : {
837 : cnat_client_t *cc;
838 : cnat_bihash_kv_t rkey;
839 140 : cnat_session_t *rsession = (cnat_session_t *) & rkey;
840 140 : cnat_bihash_kv_t *bkey = (cnat_bihash_kv_t *) session;
841 140 : int rv, n_retries = 0;
842 : static u32 sport_seed = 0;
843 :
844 140 : cnat_timestamp_inc_refcnt (session->value.cs_ts_index);
845 :
846 : /* First create the return session */
847 140 : ip46_address_copy (&rsession->key.cs_ip[VLIB_RX],
848 140 : &session->value.cs_ip[VLIB_TX]);
849 140 : ip46_address_copy (&rsession->key.cs_ip[VLIB_TX],
850 140 : &session->value.cs_ip[VLIB_RX]);
851 140 : rsession->key.cs_proto = session->key.cs_proto;
852 140 : rsession->key.cs_loc = rsession_location;
853 140 : rsession->key.__cs_pad = 0;
854 140 : rsession->key.cs_af = ctx->af;
855 140 : rsession->key.cs_port[VLIB_RX] = session->value.cs_port[VLIB_TX];
856 140 : rsession->key.cs_port[VLIB_TX] = session->value.cs_port[VLIB_RX];
857 :
858 140 : ip46_address_copy (&rsession->value.cs_ip[VLIB_RX],
859 140 : &session->key.cs_ip[VLIB_TX]);
860 140 : ip46_address_copy (&rsession->value.cs_ip[VLIB_TX],
861 140 : &session->key.cs_ip[VLIB_RX]);
862 140 : rsession->value.cs_ts_index = session->value.cs_ts_index;
863 140 : rsession->value.cs_lbi = INDEX_INVALID;
864 140 : rsession->value.flags = rsession_flags | CNAT_SESSION_IS_RETURN;
865 140 : rsession->value.cs_port[VLIB_TX] = session->key.cs_port[VLIB_RX];
866 140 : rsession->value.cs_port[VLIB_RX] = session->key.cs_port[VLIB_TX];
867 :
868 140 : retry_add_ression:
869 140 : rv = cnat_bihash_add_del (&cnat_session_db, &rkey,
870 : 2 /* add but don't overwrite */);
871 140 : if (rv)
872 : {
873 18 : if (!(rsession_flags & CNAT_SESSION_RETRY_SNAT))
874 18 : return;
875 :
876 : /* return session add failed pick an new random src port */
877 0 : rsession->value.cs_port[VLIB_TX] = session->key.cs_port[VLIB_RX] =
878 0 : random_u32 (&sport_seed);
879 0 : if (n_retries++ < 100)
880 0 : goto retry_add_ression;
881 : else
882 : {
883 0 : clib_warning ("Could not find a free port after 100 tries");
884 : /* translate this packet, but don't create state */
885 0 : return;
886 : }
887 : }
888 :
889 122 : cnat_bihash_add_del (&cnat_session_db, bkey, 1 /* add */);
890 :
891 122 : if (!(rsession_flags & CNAT_SESSION_FLAG_NO_CLIENT))
892 : {
893 : /* is this the first time we've seen this source address */
894 244 : cc = (AF_IP4 == ctx->af ?
895 122 : cnat_client_ip4_find (&session->value.cs_ip[VLIB_RX].ip4) :
896 54 : cnat_client_ip6_find (&session->value.cs_ip[VLIB_RX].ip6));
897 :
898 122 : if (NULL == cc)
899 : {
900 : ip_address_t addr;
901 : uword *p;
902 : u32 refcnt;
903 :
904 27 : addr.version = ctx->af;
905 27 : ip46_address_copy (&addr.ip, &session->value.cs_ip[VLIB_RX]);
906 :
907 : /* Throttle */
908 27 : clib_spinlock_lock (&cnat_client_db.throttle_lock);
909 :
910 27 : p = hash_get_mem (cnat_client_db.throttle_mem, &addr);
911 27 : if (p)
912 : {
913 0 : refcnt = p[0] + 1;
914 0 : hash_set_mem (cnat_client_db.throttle_mem, &addr, refcnt);
915 : }
916 : else
917 27 : hash_set_mem_alloc (&cnat_client_db.throttle_mem, &addr, 0);
918 :
919 27 : clib_spinlock_unlock (&cnat_client_db.throttle_lock);
920 :
921 : /* fire client create to the main thread */
922 27 : if (!p)
923 27 : vl_api_rpc_call_main_thread (cnat_client_learn, (u8 *) &addr,
924 : sizeof (addr));
925 : }
926 : else
927 : {
928 : /* Refcount reverse session */
929 95 : cnat_client_cnt_session (cc);
930 : }
931 : }
932 :
933 : }
934 :
935 : always_inline uword
936 392 : cnat_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
937 : vlib_frame_t *frame, cnat_node_sub_t cnat_sub,
938 : ip_address_family_t af, cnat_session_location_t cs_loc,
939 : u8 do_trace)
940 : {
941 : u32 n_left, *from, thread_index;
942 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
943 392 : vlib_buffer_t **b = bufs;
944 : u16 nexts[VLIB_FRAME_SIZE], *next;
945 : f64 now;
946 :
947 392 : thread_index = vm->thread_index;
948 392 : from = vlib_frame_vector_args (frame);
949 392 : n_left = frame->n_vectors;
950 392 : next = nexts;
951 392 : vlib_get_buffers (vm, from, bufs, n_left);
952 392 : now = vlib_time_now (vm);
953 : cnat_session_t *session[4];
954 : cnat_bihash_kv_t bkey[4], bvalue[4];
955 : u64 hash[4];
956 : int rv[4];
957 :
958 392 : cnat_node_ctx_t ctx = { now, thread_index, af, do_trace };
959 :
960 392 : if (n_left >= 8)
961 : {
962 : /* Kickstart our state */
963 392 : cnat_session_make_key (b[3], af, cs_loc, &bkey[3]);
964 392 : cnat_session_make_key (b[2], af, cs_loc, &bkey[2]);
965 392 : cnat_session_make_key (b[1], af, cs_loc, &bkey[1]);
966 392 : cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
967 :
968 392 : hash[3] = cnat_bihash_hash (&bkey[3]);
969 392 : hash[2] = cnat_bihash_hash (&bkey[2]);
970 392 : hash[1] = cnat_bihash_hash (&bkey[1]);
971 392 : hash[0] = cnat_bihash_hash (&bkey[0]);
972 : }
973 :
974 1176 : while (n_left >= 8)
975 : {
976 784 : if (n_left >= 12)
977 : {
978 392 : vlib_prefetch_buffer_header (b[11], LOAD);
979 392 : vlib_prefetch_buffer_header (b[10], LOAD);
980 392 : vlib_prefetch_buffer_header (b[9], LOAD);
981 392 : vlib_prefetch_buffer_header (b[8], LOAD);
982 : }
983 :
984 784 : rv[3] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[3], &bkey[3],
985 : &bvalue[3]);
986 784 : session[3] = (cnat_session_t *) (rv[3] ? &bkey[3] : &bvalue[3]);
987 784 : next[3] = cnat_sub (vm, node, b[3], &ctx, rv[3], session[3]);
988 :
989 784 : rv[2] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[2], &bkey[2],
990 : &bvalue[2]);
991 784 : session[2] = (cnat_session_t *) (rv[2] ? &bkey[2] : &bvalue[2]);
992 784 : next[2] = cnat_sub (vm, node, b[2], &ctx, rv[2], session[2]);
993 :
994 784 : rv[1] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[1], &bkey[1],
995 : &bvalue[1]);
996 784 : session[1] = (cnat_session_t *) (rv[1] ? &bkey[1] : &bvalue[1]);
997 784 : next[1] = cnat_sub (vm, node, b[1], &ctx, rv[1], session[1]);
998 :
999 784 : rv[0] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[0], &bkey[0],
1000 : &bvalue[0]);
1001 784 : session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
1002 784 : next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
1003 :
1004 784 : cnat_session_make_key (b[7], af, cs_loc, &bkey[3]);
1005 784 : cnat_session_make_key (b[6], af, cs_loc, &bkey[2]);
1006 784 : cnat_session_make_key (b[5], af, cs_loc, &bkey[1]);
1007 784 : cnat_session_make_key (b[4], af, cs_loc, &bkey[0]);
1008 :
1009 784 : hash[3] = cnat_bihash_hash (&bkey[3]);
1010 784 : hash[2] = cnat_bihash_hash (&bkey[2]);
1011 784 : hash[1] = cnat_bihash_hash (&bkey[1]);
1012 784 : hash[0] = cnat_bihash_hash (&bkey[0]);
1013 :
1014 784 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[3]);
1015 784 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[2]);
1016 784 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[1]);
1017 784 : cnat_bihash_prefetch_bucket (&cnat_session_db, hash[0]);
1018 :
1019 784 : cnat_bihash_prefetch_data (&cnat_session_db, hash[3]);
1020 784 : cnat_bihash_prefetch_data (&cnat_session_db, hash[2]);
1021 784 : cnat_bihash_prefetch_data (&cnat_session_db, hash[1]);
1022 784 : cnat_bihash_prefetch_data (&cnat_session_db, hash[0]);
1023 :
1024 784 : b += 4;
1025 784 : next += 4;
1026 784 : n_left -= 4;
1027 : }
1028 :
1029 3136 : while (n_left > 0)
1030 : {
1031 2744 : cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
1032 2744 : rv[0] = cnat_bihash_search_i2 (&cnat_session_db, &bkey[0], &bvalue[0]);
1033 :
1034 2744 : session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
1035 2744 : next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
1036 :
1037 2744 : b++;
1038 2744 : next++;
1039 2744 : n_left--;
1040 : }
1041 :
1042 392 : vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1043 :
1044 392 : return frame->n_vectors;
1045 : }
1046 :
1047 : /*
1048 : * fd.io coding-style-patch-verification: ON
1049 : *
1050 : * Local Variables:
1051 : * eval: (c-set-style "gnu")
1052 : * End:
1053 : */
1054 :
1055 : #endif
|
