Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * buffer_node.h: VLIB buffer handling node helper macros/inlines
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #ifndef included_vlib_buffer_node_h
41 : #define included_vlib_buffer_node_h
42 :
43 : /** \file
44 : vlib buffer/node functions
45 : */
46 :
47 : /** \brief Finish enqueueing two buffers forward in the graph.
48 : Standard dual loop boilerplate element. This is a MACRO,
49 : with MULTIPLE SIDE EFFECTS. In the ideal case,
50 : <code>next_index == next0 == next1</code>,
51 : which means that the speculative enqueue at the top of the dual loop
52 : has correctly dealt with both packets. In that case, the macro does
53 : nothing at all.
54 :
55 : @param vm vlib_main_t pointer, varies by thread
56 : @param node current node vlib_node_runtime_t pointer
57 : @param next_index speculated next index used for both packets
58 : @param to_next speculated vector pointer used for both packets
59 : @param n_left_to_next number of slots left in speculated vector
60 : @param bi0 first buffer index
61 : @param bi1 second buffer index
62 : @param next0 actual next index to be used for the first packet
63 : @param next1 actual next index to be used for the second packet
64 :
65 : @return @c next_index -- speculative next index to be used for future packets
66 : @return @c to_next -- speculative frame to be used for future packets
67 : @return @c n_left_to_next -- number of slots left in speculative frame
68 : */
69 :
70 : #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
71 : do { \
72 : ASSERT (bi0 != 0); \
73 : ASSERT (bi1 != 0); \
74 : int enqueue_code = (next0 != next_index) + 2*(next1 != next_index); \
75 : \
76 : if (PREDICT_FALSE (enqueue_code != 0)) \
77 : { \
78 : switch (enqueue_code) \
79 : { \
80 : case 1: \
81 : /* A B A */ \
82 : to_next[-2] = bi1; \
83 : to_next -= 1; \
84 : n_left_to_next += 1; \
85 : vlib_set_next_frame_buffer (vm, node, next0, bi0); \
86 : break; \
87 : \
88 : case 2: \
89 : /* A A B */ \
90 : to_next -= 1; \
91 : n_left_to_next += 1; \
92 : vlib_set_next_frame_buffer (vm, node, next1, bi1); \
93 : break; \
94 : \
95 : case 3: \
96 : /* A B B or A B C */ \
97 : to_next -= 2; \
98 : n_left_to_next += 2; \
99 : vlib_set_next_frame_buffer (vm, node, next0, bi0); \
100 : vlib_set_next_frame_buffer (vm, node, next1, bi1); \
101 : if (next0 == next1) \
102 : { \
103 : vlib_put_next_frame (vm, node, next_index, \
104 : n_left_to_next); \
105 : next_index = next1; \
106 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
107 : } \
108 : } \
109 : } \
110 : } while (0)
111 :
112 :
113 : /** \brief Finish enqueueing four buffers forward in the graph.
114 : Standard quad loop boilerplate element. This is a MACRO,
115 : with MULTIPLE SIDE EFFECTS. In the ideal case,
116 : <code>next_index == next0 == next1 == next2 == next3</code>,
117 : which means that the speculative enqueue at the top of the quad loop
118 : has correctly dealt with all four packets. In that case, the macro does
119 : nothing at all.
120 :
121 : @param vm vlib_main_t pointer, varies by thread
122 : @param node current node vlib_node_runtime_t pointer
123 : @param next_index speculated next index used for both packets
124 : @param to_next speculated vector pointer used for both packets
125 : @param n_left_to_next number of slots left in speculated vector
126 : @param bi0 first buffer index
127 : @param bi1 second buffer index
128 : @param bi2 third buffer index
129 : @param bi3 fourth buffer index
130 : @param next0 actual next index to be used for the first packet
131 : @param next1 actual next index to be used for the second packet
132 : @param next2 actual next index to be used for the third packet
133 : @param next3 actual next index to be used for the fourth packet
134 :
135 : @return @c next_index -- speculative next index to be used for future packets
136 : @return @c to_next -- speculative frame to be used for future packets
137 : @return @c n_left_to_next -- number of slots left in speculative frame
138 : */
139 :
140 : #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
141 : do { \
142 : ASSERT (bi0 != 0); \
143 : ASSERT (bi1 != 0); \
144 : ASSERT (bi2 != 0); \
145 : ASSERT (bi3 != 0); \
146 : /* After the fact: check the [speculative] enqueue to "next" */ \
147 : u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1) \
148 : | (next_index ^ next2) | (next_index ^ next3); \
149 : if (PREDICT_FALSE(fix_speculation)) \
150 : { \
151 : /* rewind... */ \
152 : to_next -= 4; \
153 : n_left_to_next += 4; \
154 : \
155 : /* If bi0 belongs to "next", send it there */ \
156 : if (next_index == next0) \
157 : { \
158 : to_next[0] = bi0; \
159 : to_next++; \
160 : n_left_to_next --; \
161 : } \
162 : else /* send it where it needs to go */ \
163 : vlib_set_next_frame_buffer (vm, node, next0, bi0); \
164 : \
165 : if (next_index == next1) \
166 : { \
167 : to_next[0] = bi1; \
168 : to_next++; \
169 : n_left_to_next --; \
170 : } \
171 : else \
172 : vlib_set_next_frame_buffer (vm, node, next1, bi1); \
173 : \
174 : if (next_index == next2) \
175 : { \
176 : to_next[0] = bi2; \
177 : to_next++; \
178 : n_left_to_next --; \
179 : } \
180 : else \
181 : vlib_set_next_frame_buffer (vm, node, next2, bi2); \
182 : \
183 : if (next_index == next3) \
184 : { \
185 : to_next[0] = bi3; \
186 : to_next++; \
187 : n_left_to_next --; \
188 : } \
189 : else \
190 : { \
191 : vlib_set_next_frame_buffer (vm, node, next3, bi3); \
192 : \
193 : /* Change speculation: last 2 packets went to the same node*/ \
194 : if (next2 == next3) \
195 : { \
196 : vlib_put_next_frame (vm, node, next_index, n_left_to_next); \
197 : next_index = next3; \
198 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
199 : } \
200 : } \
201 : } \
202 : } while(0);
203 :
204 : /** \brief Finish enqueueing one buffer forward in the graph.
205 : Standard single loop boilerplate element. This is a MACRO,
206 : with MULTIPLE SIDE EFFECTS. In the ideal case,
207 : <code>next_index == next0</code>,
208 : which means that the speculative enqueue at the top of the single loop
209 : has correctly dealt with the packet in hand. In that case, the macro does
210 : nothing at all.
211 :
212 : @param vm vlib_main_t pointer, varies by thread
213 : @param node current node vlib_node_runtime_t pointer
214 : @param next_index speculated next index used for both packets
215 : @param to_next speculated vector pointer used for both packets
216 : @param n_left_to_next number of slots left in speculated vector
217 : @param bi0 first buffer index
218 : @param next0 actual next index to be used for the first packet
219 :
220 : @return @c next_index -- speculative next index to be used for future packets
221 : @return @c to_next -- speculative frame to be used for future packets
222 : @return @c n_left_to_next -- number of slots left in speculative frame
223 : */
224 : #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
225 : do { \
226 : ASSERT (bi0 != 0); \
227 : if (PREDICT_FALSE (next0 != next_index)) \
228 : { \
229 : vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
230 : next_index = next0; \
231 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
232 : \
233 : to_next[0] = bi0; \
234 : to_next += 1; \
235 : n_left_to_next -= 1; \
236 : } \
237 : } while (0)
238 :
239 : /** \brief Finish enqueueing one buffer forward in the graph, along with its
240 : aux_data if possible. Standard single loop boilerplate element. This is a
241 : MACRO, with MULTIPLE SIDE EFFECTS. In the ideal case, <code>next_index ==
242 : next0</code>, which means that the speculative enqueue at the top of the
243 : single loop has correctly dealt with the packet in hand. In that case, the
244 : macro does nothing at all. This function MAY return to_next_aux = NULL if
245 : next_index does not support aux data
246 :
247 : @param vm vlib_main_t pointer, varies by thread
248 : @param node current node vlib_node_runtime_t pointer
249 : @param next_index speculated next index used for both packets
250 : @param to_next speculated vector pointer used for both packets
251 : @param to_next_aux speculated aux_data pointer used for both packets
252 : @param n_left_to_next number of slots left in speculated vector
253 : @param bi0 first buffer index
254 : @param aux0 first aux_data
255 : @param next0 actual next index to be used for the first packet
256 :
257 : @return @c next_index -- speculative next index to be used for future packets
258 : @return @c to_next -- speculative frame to be used for future packets
259 : @return @c n_left_to_next -- number of slots left in speculative frame
260 : */
261 : #define vlib_validate_buffer_enqueue_with_aux_x1( \
262 : vm, node, next_index, to_next, to_next_aux, n_left_to_next, bi0, aux0, \
263 : next0) \
264 : do \
265 : { \
266 : ASSERT (bi0 != 0); \
267 : if (PREDICT_FALSE (next0 != next_index)) \
268 : { \
269 : vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
270 : next_index = next0; \
271 : vlib_get_next_frame_with_aux_safe (vm, node, next_index, to_next, \
272 : to_next_aux, n_left_to_next); \
273 : \
274 : to_next[0] = bi0; \
275 : to_next += 1; \
276 : if (to_next_aux) \
277 : { \
278 : to_next_aux[0] = aux0; \
279 : to_next_aux += 1; \
280 : } \
281 : n_left_to_next -= 1; \
282 : } \
283 : } \
284 : while (0)
285 :
286 : always_inline uword
287 : generic_buffer_node_inline (vlib_main_t * vm,
288 : vlib_node_runtime_t * node,
289 : vlib_frame_t * frame,
290 : uword sizeof_trace,
291 : void *opaque1,
292 : uword opaque2,
293 : void (*two_buffers) (vlib_main_t * vm,
294 : void *opaque1,
295 : uword opaque2,
296 : vlib_buffer_t * b0,
297 : vlib_buffer_t * b1,
298 : u32 * next0, u32 * next1),
299 : void (*one_buffer) (vlib_main_t * vm,
300 : void *opaque1, uword opaque2,
301 : vlib_buffer_t * b0,
302 : u32 * next0))
303 : {
304 : u32 n_left_from, *from, *to_next;
305 : u32 next_index;
306 :
307 : from = vlib_frame_vector_args (frame);
308 : n_left_from = frame->n_vectors;
309 : next_index = node->cached_next_index;
310 :
311 : if (node->flags & VLIB_NODE_FLAG_TRACE)
312 : vlib_trace_frame_buffers_only (vm, node, from, frame->n_vectors,
313 : /* stride */ 1, sizeof_trace);
314 :
315 : while (n_left_from > 0)
316 : {
317 : u32 n_left_to_next;
318 :
319 : vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
320 :
321 : while (n_left_from >= 4 && n_left_to_next >= 2)
322 : {
323 : vlib_buffer_t *p0, *p1;
324 : u32 pi0, next0;
325 : u32 pi1, next1;
326 :
327 : /* Prefetch next iteration. */
328 : {
329 : vlib_buffer_t *p2, *p3;
330 :
331 : p2 = vlib_get_buffer (vm, from[2]);
332 : p3 = vlib_get_buffer (vm, from[3]);
333 :
334 : vlib_prefetch_buffer_header (p2, LOAD);
335 : vlib_prefetch_buffer_header (p3, LOAD);
336 :
337 : clib_prefetch_load (p2->data);
338 : clib_prefetch_load (p3->data);
339 : }
340 :
341 : pi0 = to_next[0] = from[0];
342 : pi1 = to_next[1] = from[1];
343 : from += 2;
344 : to_next += 2;
345 : n_left_from -= 2;
346 : n_left_to_next -= 2;
347 :
348 : p0 = vlib_get_buffer (vm, pi0);
349 : p1 = vlib_get_buffer (vm, pi1);
350 :
351 : two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
352 :
353 : vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
354 : to_next, n_left_to_next,
355 : pi0, pi1, next0, next1);
356 : }
357 :
358 : while (n_left_from > 0 && n_left_to_next > 0)
359 : {
360 : vlib_buffer_t *p0;
361 : u32 pi0, next0;
362 :
363 : pi0 = from[0];
364 : to_next[0] = pi0;
365 : from += 1;
366 : to_next += 1;
367 : n_left_from -= 1;
368 : n_left_to_next -= 1;
369 :
370 : p0 = vlib_get_buffer (vm, pi0);
371 :
372 : one_buffer (vm, opaque1, opaque2, p0, &next0);
373 :
374 : vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
375 : to_next, n_left_to_next,
376 : pi0, next0);
377 : }
378 :
379 : vlib_put_next_frame (vm, node, next_index, n_left_to_next);
380 : }
381 :
382 : return frame->n_vectors;
383 : }
384 :
385 : /* Minimum size for the 'buffers' and 'nexts' arrays to be used when calling
386 : * vlib_buffer_enqueue_to_next().
387 : * Because of optimizations, vlib_buffer_enqueue_to_next() will access
388 : * past 'count' elements in the 'buffers' and 'nexts' arrays, IOW it
389 : * will overflow.
390 : * Those overflow elements are ignored in the final result so they do not
391 : * need to be properly initialized, however if the array is allocated right
392 : * before the end of a page and the next page is not mapped, accessing the
393 : * overflow elements will trigger a segfault. */
394 : #define VLIB_BUFFER_ENQUEUE_MIN_SIZE(n) round_pow2 ((n), 64)
395 :
396 : static_always_inline void
397 6926651 : vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
398 : u32 * buffers, u16 * nexts, uword count)
399 : {
400 : vlib_buffer_enqueue_to_next_fn_t *fn;
401 6926651 : fn = vlib_buffer_func_main.buffer_enqueue_to_next_fn;
402 6926651 : (fn) (vm, node, buffers, nexts, count);
403 6926649 : }
404 :
405 : static_always_inline void
406 : vlib_buffer_enqueue_to_next_with_aux (vlib_main_t *vm,
407 : vlib_node_runtime_t *node, u32 *buffers,
408 : u32 *aux_data, u16 *nexts, uword count)
409 : {
410 : vlib_buffer_enqueue_to_next_with_aux_fn_t *fn;
411 : fn = vlib_buffer_func_main.buffer_enqueue_to_next_with_aux_fn;
412 : (fn) (vm, node, buffers, aux_data, nexts, count);
413 : }
414 :
415 : static_always_inline void
416 43437 : vlib_buffer_enqueue_to_next_vec (vlib_main_t *vm, vlib_node_runtime_t *node,
417 : u32 **buffers, u16 **nexts, uword count)
418 : {
419 43437 : const u32 bl = vec_len (*buffers), nl = vec_len (*nexts);
420 43437 : const u32 c = VLIB_BUFFER_ENQUEUE_MIN_SIZE (count);
421 43437 : ASSERT (bl >= count && nl >= count);
422 43437 : vec_validate (*buffers, c);
423 43437 : vec_validate (*nexts, c);
424 43437 : vlib_buffer_enqueue_to_next (vm, node, *buffers, *nexts, count);
425 43437 : vec_set_len (*buffers, bl);
426 43437 : vec_set_len (*nexts, nl);
427 43437 : }
428 :
429 : static_always_inline void
430 97 : vlib_buffer_enqueue_to_single_next (vlib_main_t * vm,
431 : vlib_node_runtime_t * node, u32 * buffers,
432 : u16 next_index, u32 count)
433 : {
434 : vlib_buffer_enqueue_to_single_next_fn_t *fn;
435 97 : fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_fn;
436 97 : (fn) (vm, node, buffers, next_index, count);
437 97 : }
438 :
439 : static_always_inline void
440 : vlib_buffer_enqueue_to_single_next_with_aux (vlib_main_t *vm,
441 : vlib_node_runtime_t *node,
442 : u32 *buffers, u32 *aux_data,
443 : u16 next_index, u32 count)
444 : {
445 : vlib_buffer_enqueue_to_single_next_with_aux_fn_t *fn;
446 : fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_with_aux_fn;
447 : (fn) (vm, node, buffers, aux_data, next_index, count);
448 : }
449 :
450 : static_always_inline u32
451 613 : vlib_buffer_enqueue_to_thread (vlib_main_t *vm, vlib_node_runtime_t *node,
452 : u32 frame_queue_index, u32 *buffer_indices,
453 : u16 *thread_indices, u32 n_packets,
454 : int drop_on_congestion)
455 : {
456 : vlib_buffer_enqueue_to_thread_fn_t *fn;
457 613 : fn = vlib_buffer_func_main.buffer_enqueue_to_thread_fn;
458 613 : return (fn) (vm, node, frame_queue_index, buffer_indices, thread_indices,
459 : n_packets, drop_on_congestion);
460 : }
461 :
462 : static_always_inline u32
463 0 : vlib_buffer_enqueue_to_thread_with_aux (vlib_main_t *vm,
464 : vlib_node_runtime_t *node,
465 : u32 frame_queue_index,
466 : u32 *buffer_indices, u32 *aux,
467 : u16 *thread_indices, u32 n_packets,
468 : int drop_on_congestion)
469 : {
470 : vlib_buffer_enqueue_to_thread_with_aux_fn_t *fn;
471 0 : fn = vlib_buffer_func_main.buffer_enqueue_to_thread_with_aux_fn;
472 0 : return (fn) (vm, node, frame_queue_index, buffer_indices, aux,
473 : thread_indices, n_packets, drop_on_congestion);
474 : }
475 :
476 : #endif /* included_vlib_buffer_node_h */
477 :
478 : /*
479 : * fd.io coding-style-patch-verification: ON
480 : *
481 : * Local Variables:
482 : * eval: (c-set-style "gnu")
483 : * End:
484 : */
|
