Line data Source code
1 : /*
2 : * Copyright (c) 2018 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /**
16 : * @file
17 : * @brief NAT44 worker handoff
18 : */
19 :
20 : #include <vlib/vlib.h>
21 : #include <vnet/vnet.h>
22 : #include <vnet/fib/ip4_fib.h>
23 : #include <vppinfra/error.h>
24 :
25 : #include <nat/nat44-ed/nat44_ed.h>
26 : #include <nat/nat44-ed/nat44_ed_inlines.h>
27 :
28 : typedef struct
29 : {
30 : u32 next_worker_index;
31 : u32 trace_index;
32 : u8 in2out;
33 : u8 output;
34 : } nat44_handoff_trace_t;
35 :
36 : #define foreach_nat44_handoff_error \
37 : _ (CONGESTION_DROP, "congestion drop") \
38 : _ (SAME_WORKER, "same worker") \
39 : _ (DO_HANDOFF, "do handoff")
40 :
41 : typedef enum
42 : {
43 : #define _(sym, str) NAT44_HANDOFF_ERROR_##sym,
44 : foreach_nat44_handoff_error
45 : #undef _
46 : NAT44_HANDOFF_N_ERROR,
47 : } nat44_handoff_error_t;
48 :
49 : static char *nat44_handoff_error_strings[] = {
50 : #define _(sym,string) string,
51 : foreach_nat44_handoff_error
52 : #undef _
53 : };
54 :
55 : static u8 *
56 5672 : format_nat44_handoff_trace (u8 * s, va_list * args)
57 : {
58 5672 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
59 5672 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
60 5672 : nat44_handoff_trace_t *t = va_arg (*args, nat44_handoff_trace_t *);
61 : char *tag, *output;
62 :
63 5672 : tag = t->in2out ? "IN2OUT" : "OUT2IN";
64 5672 : output = t->output ? "OUTPUT-FEATURE" : "";
65 : s =
66 5672 : format (s, "NAT44_%s_WORKER_HANDOFF %s: next-worker %d trace index %d",
67 : tag, output, t->next_worker_index, t->trace_index);
68 :
69 5672 : return s;
70 : }
71 :
72 : static inline uword
73 449 : nat44_worker_handoff_fn_inline (vlib_main_t * vm,
74 : vlib_node_runtime_t * node,
75 : vlib_frame_t * frame, u8 is_output,
76 : u8 is_in2out)
77 : {
78 449 : u32 n_enq, n_left_from, *from, do_handoff = 0, same_worker = 0;
79 :
80 449 : u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
81 449 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
82 449 : snat_main_t *sm = &snat_main;
83 :
84 449 : u32 fq_index, thread_index = vm->thread_index;
85 :
86 449 : from = vlib_frame_vector_args (frame);
87 449 : n_left_from = frame->n_vectors;
88 :
89 449 : vlib_get_buffers (vm, from, b, n_left_from);
90 :
91 449 : if (is_in2out)
92 : {
93 316 : fq_index = is_output ? sm->fq_in2out_output_index : sm->fq_in2out_index;
94 : }
95 : else
96 : {
97 133 : fq_index = sm->fq_out2in_index;
98 : }
99 :
100 8686 : while (n_left_from >= 4)
101 : {
102 : u32 arc_next0, arc_next1, arc_next2, arc_next3;
103 : u32 sw_if_index0, sw_if_index1, sw_if_index2, sw_if_index3;
104 : u32 rx_fib_index0, rx_fib_index1, rx_fib_index2, rx_fib_index3;
105 8236 : u32 iph_offset0 = 0, iph_offset1 = 0, iph_offset2 = 0, iph_offset3 = 0;
106 : ip4_header_t *ip0, *ip1, *ip2, *ip3;
107 :
108 8236 : if (PREDICT_TRUE (n_left_from >= 8))
109 : {
110 8002 : vlib_prefetch_buffer_header (b[4], LOAD);
111 8002 : vlib_prefetch_buffer_header (b[5], LOAD);
112 8002 : vlib_prefetch_buffer_header (b[6], LOAD);
113 8003 : vlib_prefetch_buffer_header (b[7], LOAD);
114 8004 : clib_prefetch_load (&b[4]->data);
115 8002 : clib_prefetch_load (&b[5]->data);
116 7999 : clib_prefetch_load (&b[6]->data);
117 7998 : clib_prefetch_load (&b[7]->data);
118 : }
119 :
120 8231 : if (is_output)
121 : {
122 9 : iph_offset0 = vnet_buffer (b[0])->ip.save_rewrite_length;
123 9 : iph_offset1 = vnet_buffer (b[1])->ip.save_rewrite_length;
124 9 : iph_offset2 = vnet_buffer (b[2])->ip.save_rewrite_length;
125 9 : iph_offset3 = vnet_buffer (b[3])->ip.save_rewrite_length;
126 : }
127 :
128 8231 : ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[0]) +
129 : iph_offset0);
130 8236 : ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[1]) +
131 : iph_offset1);
132 8238 : ip2 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[2]) +
133 : iph_offset2);
134 8237 : ip3 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[3]) +
135 : iph_offset3);
136 :
137 8237 : vnet_feature_next (&arc_next0, b[0]);
138 8237 : vnet_feature_next (&arc_next1, b[1]);
139 8228 : vnet_feature_next (&arc_next2, b[2]);
140 8228 : vnet_feature_next (&arc_next3, b[3]);
141 :
142 8226 : vnet_buffer2 (b[0])->nat.arc_next = arc_next0;
143 8226 : vnet_buffer2 (b[1])->nat.arc_next = arc_next1;
144 8226 : vnet_buffer2 (b[2])->nat.arc_next = arc_next2;
145 8226 : vnet_buffer2 (b[3])->nat.arc_next = arc_next3;
146 :
147 8226 : sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
148 8226 : sw_if_index1 = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
149 8226 : sw_if_index2 = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
150 8226 : sw_if_index3 = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
151 :
152 8226 : rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
153 8232 : rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index1);
154 8229 : rx_fib_index2 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index2);
155 8230 : rx_fib_index3 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index3);
156 :
157 8231 : if (is_in2out)
158 : {
159 7050 : ti[0] = nat44_ed_get_in2out_worker_index (b[0], ip0, rx_fib_index0,
160 : is_output);
161 7051 : ti[1] = nat44_ed_get_in2out_worker_index (b[1], ip1, rx_fib_index1,
162 : is_output);
163 7054 : ti[2] = nat44_ed_get_in2out_worker_index (b[2], ip2, rx_fib_index2,
164 : is_output);
165 7052 : ti[3] = nat44_ed_get_in2out_worker_index (b[3], ip3, rx_fib_index3,
166 : is_output);
167 : }
168 : else
169 : {
170 1181 : ti[0] = nat44_ed_get_out2in_worker_index (b[0], ip0, rx_fib_index0,
171 : is_output);
172 1169 : ti[1] = nat44_ed_get_out2in_worker_index (b[1], ip1, rx_fib_index1,
173 : is_output);
174 1170 : ti[2] = nat44_ed_get_out2in_worker_index (b[2], ip2, rx_fib_index2,
175 : is_output);
176 1184 : ti[3] = nat44_ed_get_out2in_worker_index (b[3], ip3, rx_fib_index3,
177 : is_output);
178 : }
179 :
180 8237 : if (ti[0] == thread_index)
181 624 : same_worker++;
182 : else
183 7613 : do_handoff++;
184 :
185 8237 : if (ti[1] == thread_index)
186 639 : same_worker++;
187 : else
188 7598 : do_handoff++;
189 :
190 8237 : if (ti[2] == thread_index)
191 624 : same_worker++;
192 : else
193 7613 : do_handoff++;
194 :
195 8237 : if (ti[3] == thread_index)
196 614 : same_worker++;
197 : else
198 7623 : do_handoff++;
199 :
200 8237 : b += 4;
201 8237 : ti += 4;
202 8237 : n_left_from -= 4;
203 : }
204 :
205 811 : while (n_left_from > 0)
206 : {
207 : u32 arc_next0;
208 : u32 sw_if_index0;
209 : u32 rx_fib_index0;
210 361 : u32 iph_offset0 = 0;
211 : ip4_header_t *ip0;
212 :
213 :
214 361 : if (is_output)
215 6 : iph_offset0 = vnet_buffer (b[0])->ip.save_rewrite_length;
216 :
217 361 : ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[0]) +
218 : iph_offset0);
219 :
220 361 : vnet_feature_next (&arc_next0, b[0]);
221 361 : vnet_buffer2 (b[0])->nat.arc_next = arc_next0;
222 :
223 361 : sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
224 361 : rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
225 :
226 361 : if (is_in2out)
227 : {
228 204 : ti[0] = nat44_ed_get_in2out_worker_index (b[0], ip0, rx_fib_index0,
229 : is_output);
230 : }
231 : else
232 : {
233 157 : ti[0] = nat44_ed_get_out2in_worker_index (b[0], ip0, rx_fib_index0,
234 : is_output);
235 : }
236 :
237 361 : if (ti[0] == thread_index)
238 84 : same_worker++;
239 : else
240 277 : do_handoff++;
241 :
242 361 : b += 1;
243 361 : ti += 1;
244 361 : n_left_from -= 1;
245 : }
246 :
247 450 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
248 : {
249 : u32 i;
250 409 : b = bufs;
251 409 : ti = thread_indices;
252 :
253 24136 : for (i = 0; i < frame->n_vectors; i++)
254 : {
255 23739 : if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
256 : {
257 : nat44_handoff_trace_t *t =
258 23729 : vlib_add_trace (vm, node, b[0], sizeof (*t));
259 23716 : t->next_worker_index = ti[0];
260 23716 : t->trace_index = vlib_buffer_get_trace_index (b[0]);
261 23727 : t->in2out = is_in2out;
262 23727 : t->output = is_output;
263 :
264 23727 : b += 1;
265 23727 : ti += 1;
266 : }
267 : else
268 10 : break;
269 : }
270 : }
271 :
272 448 : n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
273 448 : thread_indices, frame->n_vectors, 1);
274 :
275 449 : if (n_enq < frame->n_vectors)
276 : {
277 0 : vlib_node_increment_counter (vm, node->node_index,
278 : NAT44_HANDOFF_ERROR_CONGESTION_DROP,
279 0 : frame->n_vectors - n_enq);
280 : }
281 :
282 449 : vlib_node_increment_counter (vm, node->node_index,
283 : NAT44_HANDOFF_ERROR_SAME_WORKER, same_worker);
284 449 : vlib_node_increment_counter (vm, node->node_index,
285 : NAT44_HANDOFF_ERROR_DO_HANDOFF, do_handoff);
286 449 : return frame->n_vectors;
287 : }
288 :
289 2601 : VLIB_NODE_FN (snat_in2out_worker_handoff_node) (vlib_main_t * vm,
290 : vlib_node_runtime_t * node,
291 : vlib_frame_t * frame)
292 : {
293 301 : return nat44_worker_handoff_fn_inline (vm, node, frame, 0, 1);
294 : }
295 :
296 58202 : VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
297 : .name = "nat44-in2out-worker-handoff",
298 : .vector_size = sizeof (u32),
299 : .sibling_of = "nat-default",
300 : .format_trace = format_nat44_handoff_trace,
301 : .type = VLIB_NODE_TYPE_INTERNAL,
302 : .n_errors = ARRAY_LEN(nat44_handoff_error_strings),
303 : .error_strings = nat44_handoff_error_strings,
304 : };
305 :
306 2315 : VLIB_NODE_FN (snat_in2out_output_worker_handoff_node) (vlib_main_t * vm,
307 : vlib_node_runtime_t *
308 : node,
309 : vlib_frame_t * frame)
310 : {
311 15 : return nat44_worker_handoff_fn_inline (vm, node, frame, 1, 1);
312 : }
313 :
314 58202 : VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
315 : .name = "nat44-in2out-output-worker-handoff",
316 : .vector_size = sizeof (u32),
317 : .sibling_of = "nat-default",
318 : .format_trace = format_nat44_handoff_trace,
319 : .type = VLIB_NODE_TYPE_INTERNAL,
320 : .n_errors = ARRAY_LEN(nat44_handoff_error_strings),
321 : .error_strings = nat44_handoff_error_strings,
322 : };
323 :
324 2433 : VLIB_NODE_FN (snat_out2in_worker_handoff_node) (vlib_main_t * vm,
325 : vlib_node_runtime_t * node,
326 : vlib_frame_t * frame)
327 : {
328 133 : return nat44_worker_handoff_fn_inline (vm, node, frame, 0, 0);
329 : }
330 :
331 58202 : VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
332 : .name = "nat44-out2in-worker-handoff",
333 : .vector_size = sizeof (u32),
334 : .sibling_of = "nat-default",
335 : .format_trace = format_nat44_handoff_trace,
336 : .type = VLIB_NODE_TYPE_INTERNAL,
337 : .n_errors = ARRAY_LEN(nat44_handoff_error_strings),
338 : .error_strings = nat44_handoff_error_strings,
339 : };
340 :
341 : /*
342 : * fd.io coding-style-patch-verification: ON
343 : *
344 : * Local Variables:
345 : * eval: (c-set-style "gnu")
346 : * End:
347 : */
|
