Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * node_funcs.h: processing nodes global functions/inlines
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : /** \file
41 : vlib node functions
42 : */
43 :
44 :
45 : #ifndef included_vlib_node_funcs_h
46 : #define included_vlib_node_funcs_h
47 :
48 : #include <vppinfra/clib.h>
49 : #include <vppinfra/fifo.h>
50 : #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
51 : #include <vppinfra/interrupt.h>
52 :
53 : #ifdef CLIB_SANITIZE_ADDR
54 : #include <sanitizer/asan_interface.h>
55 : #endif
56 :
57 : static_always_inline void
58 4571088 : vlib_process_start_switch_stack (vlib_main_t * vm, vlib_process_t * p)
59 : {
60 : #ifdef CLIB_SANITIZE_ADDR
61 : void *stack = p ? (void *) p->stack : vlib_thread_stacks[vm->thread_index];
62 : u32 stack_bytes =
63 : p ? (1ULL < p->log2_n_stack_bytes) : VLIB_THREAD_STACK_SIZE;
64 : __sanitizer_start_switch_fiber (&vm->asan_stack_save, stack, stack_bytes);
65 : #endif
66 4571088 : }
67 :
68 : static_always_inline void
69 4571089 : vlib_process_finish_switch_stack (vlib_main_t * vm)
70 : {
71 : #ifdef CLIB_SANITIZE_ADDR
72 : const void *bottom_old;
73 : size_t size_old;
74 :
75 : __sanitizer_finish_switch_fiber (&vm->asan_stack_save, &bottom_old,
76 : &size_old);
77 : #endif
78 4571089 : }
79 :
80 : /** \brief Get vlib node by index.
81 : @warning This function will ASSERT if @c i is out of range.
82 : @param vm vlib_main_t pointer, varies by thread
83 : @param i node index.
84 : @return pointer to the requested vlib_node_t.
85 : */
86 :
87 : always_inline vlib_node_t *
88 21622846935 : vlib_get_node (vlib_main_t * vm, u32 i)
89 : {
90 21622846935 : return vec_elt (vm->node_main.nodes, i);
91 : }
92 :
93 : /** \brief Get vlib node by graph arc (next) index.
94 : @param vm vlib_main_t pointer, varies by thread
95 : @param node_index index of original node
96 : @param next_index graph arc index
97 : @return pointer to the vlib_node_t at the end of the indicated arc
98 : */
99 :
100 : always_inline vlib_node_t *
101 1 : vlib_get_next_node (vlib_main_t * vm, u32 node_index, u32 next_index)
102 : {
103 1 : vlib_node_main_t *nm = &vm->node_main;
104 : vlib_node_t *n;
105 :
106 1 : n = vec_elt (nm->nodes, node_index);
107 1 : ASSERT (next_index < vec_len (n->next_nodes));
108 1 : return vlib_get_node (vm, n->next_nodes[next_index]);
109 : }
110 :
111 : /** \brief Get node runtime by node index.
112 : @param vm vlib_main_t pointer, varies by thread
113 : @param node_index index of node
114 : @return pointer to the indicated vlib_node_runtime_t
115 : */
116 :
117 : always_inline vlib_node_runtime_t *
118 490222573 : vlib_node_get_runtime (vlib_main_t * vm, u32 node_index)
119 : {
120 490222573 : vlib_node_main_t *nm = &vm->node_main;
121 490222573 : vlib_node_t *n = vec_elt (nm->nodes, node_index);
122 : vlib_process_t *p;
123 490222623 : if (n->type != VLIB_NODE_TYPE_PROCESS)
124 457505077 : return vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
125 : else
126 : {
127 32716744 : p = vec_elt (nm->processes, n->runtime_index);
128 32716744 : return &p->node_runtime;
129 : }
130 : }
131 :
132 : /** \brief Get node runtime private data by node index.
133 : @param vm vlib_main_t pointer, varies by thread
134 : @param node_index index of the node
135 : @return pointer to the indicated vlib_node_runtime_t private data
136 : */
137 :
138 : always_inline void *
139 220555 : vlib_node_get_runtime_data (vlib_main_t * vm, u32 node_index)
140 : {
141 220555 : vlib_node_runtime_t *r = vlib_node_get_runtime (vm, node_index);
142 220555 : return r->runtime_data;
143 : }
144 :
145 : /** \brief Set node runtime private data.
146 : @param vm vlib_main_t pointer, varies by thread
147 : @param node_index index of the node
148 : @param runtime_data arbitrary runtime private data
149 : @param n_runtime_data_bytes size of runtime private data
150 : */
151 :
152 : always_inline void
153 : vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index,
154 : void *runtime_data, u32 n_runtime_data_bytes)
155 : {
156 : vlib_node_t *n = vlib_get_node (vm, node_index);
157 : vlib_node_runtime_t *r = vlib_node_get_runtime (vm, node_index);
158 :
159 : n->runtime_data_bytes = n_runtime_data_bytes;
160 : vec_free (n->runtime_data);
161 : vec_add (n->runtime_data, runtime_data, n_runtime_data_bytes);
162 :
163 : ASSERT (vec_len (n->runtime_data) <= sizeof (vlib_node_runtime_t) -
164 : STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data));
165 :
166 : if (vec_len (n->runtime_data) > 0)
167 : clib_memcpy_fast (r->runtime_data, n->runtime_data,
168 : vec_len (n->runtime_data));
169 : }
170 :
171 : /** \brief Set node dispatch state.
172 : @param vm vlib_main_t pointer, varies by thread
173 : @param node_index index of the node
174 : @param new_state new state for node, see vlib_node_state_t
175 : */
176 : always_inline void
177 659952 : vlib_node_set_state (vlib_main_t * vm, u32 node_index,
178 : vlib_node_state_t new_state)
179 : {
180 659952 : vlib_node_main_t *nm = &vm->node_main;
181 : vlib_node_t *n;
182 : vlib_node_runtime_t *r;
183 :
184 659952 : n = vec_elt (nm->nodes, node_index);
185 659951 : if (n->type == VLIB_NODE_TYPE_PROCESS)
186 : {
187 740 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
188 740 : r = &p->node_runtime;
189 :
190 : /* When disabling make sure flags are cleared. */
191 740 : p->flags &= ~(VLIB_PROCESS_RESUME_PENDING
192 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
193 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT);
194 : }
195 : else
196 659211 : r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
197 :
198 659949 : ASSERT (new_state < VLIB_N_NODE_STATE);
199 :
200 659949 : if (n->type == VLIB_NODE_TYPE_INPUT)
201 : {
202 658460 : ASSERT (nm->input_node_counts_by_state[n->state] > 0);
203 658461 : nm->input_node_counts_by_state[n->state] -= 1;
204 658461 : nm->input_node_counts_by_state[new_state] += 1;
205 : }
206 :
207 659950 : if (PREDICT_FALSE (r->state == VLIB_NODE_STATE_DISABLED))
208 16743 : vlib_node_runtime_perf_counter (vm, r, 0, 0, 0,
209 : VLIB_NODE_RUNTIME_PERF_RESET);
210 :
211 659950 : n->state = new_state;
212 659950 : r->state = new_state;
213 659950 : }
214 :
215 : /** \brief Get node dispatch state.
216 : @param vm vlib_main_t pointer, varies by thread
217 : @param node_index index of the node
218 : @return state for node, see vlib_node_state_t
219 : */
220 : always_inline vlib_node_state_t
221 19586 : vlib_node_get_state (vlib_main_t * vm, u32 node_index)
222 : {
223 19586 : vlib_node_main_t *nm = &vm->node_main;
224 : vlib_node_t *n;
225 19586 : n = vec_elt (nm->nodes, node_index);
226 19586 : return n->state;
227 : }
228 :
229 : always_inline void
230 1825 : vlib_node_set_flag (vlib_main_t *vm, u32 node_index, u16 flag, u8 enable)
231 : {
232 : vlib_node_runtime_t *r;
233 : vlib_node_t *n;
234 :
235 1825 : n = vlib_get_node (vm, node_index);
236 1825 : r = vlib_node_get_runtime (vm, node_index);
237 :
238 1825 : if (enable)
239 : {
240 0 : n->flags |= flag;
241 0 : r->flags |= flag;
242 : }
243 : else
244 : {
245 1825 : n->flags &= ~flag;
246 1825 : r->flags &= ~flag;
247 : }
248 1825 : }
249 :
250 : always_inline void
251 548472 : vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index)
252 : {
253 548472 : vlib_node_main_t *nm = &vm->node_main;
254 548472 : vlib_node_t *n = vec_elt (nm->nodes, node_index);
255 548472 : void *interrupts = 0;
256 :
257 548472 : if (n->type == VLIB_NODE_TYPE_INPUT)
258 548472 : interrupts = nm->input_node_interrupts;
259 0 : else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
260 0 : interrupts = nm->pre_input_node_interrupts;
261 : else
262 : {
263 0 : ASSERT (0);
264 0 : return;
265 : }
266 :
267 548472 : if (vm != vlib_get_main ())
268 450 : clib_interrupt_set_atomic (interrupts, n->runtime_index);
269 : else
270 548022 : clib_interrupt_set (interrupts, n->runtime_index);
271 :
272 548472 : __atomic_store_n (&nm->pending_interrupts, 1, __ATOMIC_RELEASE);
273 : }
274 :
275 : always_inline vlib_process_t *
276 250144 : vlib_get_process_from_node (vlib_main_t * vm, vlib_node_t * node)
277 : {
278 250144 : vlib_node_main_t *nm = &vm->node_main;
279 250144 : ASSERT (node->type == VLIB_NODE_TYPE_PROCESS);
280 250144 : return vec_elt (nm->processes, node->runtime_index);
281 : }
282 :
283 : always_inline vlib_frame_t *
284 53073445 : vlib_get_frame (vlib_main_t * vm, vlib_frame_t * f)
285 : {
286 53073445 : ASSERT (f != NULL);
287 53073445 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
288 53073445 : return f;
289 : }
290 :
291 : always_inline void
292 1265845 : vlib_frame_no_append (vlib_frame_t * f)
293 : {
294 1265845 : f->frame_flags |= VLIB_FRAME_NO_APPEND;
295 1265845 : }
296 :
297 : /** \brief Get pointer to frame vector data.
298 : @param f vlib_frame_t pointer
299 : @return pointer to first vector element in frame
300 : */
301 : always_inline void *
302 55014725 : vlib_frame_vector_args (vlib_frame_t * f)
303 : {
304 55014725 : ASSERT (f->vector_offset);
305 55014725 : return (void *) f + f->vector_offset;
306 : }
307 :
308 : /** \brief Get pointer to frame vector aux data.
309 : @param f vlib_frame_t pointer
310 : @return pointer to first vector aux data element in frame
311 : */
312 : always_inline void *
313 0 : vlib_frame_aux_args (vlib_frame_t *f)
314 : {
315 0 : ASSERT (f->aux_offset);
316 0 : return (void *) f + f->aux_offset;
317 : }
318 :
319 : /** \brief Get pointer to frame scalar data.
320 :
321 : @param f vlib_frame_t pointer
322 :
323 : @return arbitrary node scalar data
324 :
325 : @sa vlib_frame_vector_args
326 : */
327 : always_inline void *
328 7700394 : vlib_frame_scalar_args (vlib_frame_t * f)
329 : {
330 7700394 : ASSERT (f->scalar_offset);
331 7700394 : return (void *) f + f->scalar_offset;
332 : }
333 :
334 : always_inline vlib_next_frame_t *
335 61137356 : vlib_node_runtime_get_next_frame (vlib_main_t * vm,
336 : vlib_node_runtime_t * n, u32 next_index)
337 : {
338 61137356 : vlib_node_main_t *nm = &vm->node_main;
339 : vlib_next_frame_t *nf;
340 :
341 61137356 : ASSERT (next_index < n->n_next_nodes);
342 61137358 : nf = vec_elt_at_index (nm->next_frames, n->next_frame_index + next_index);
343 :
344 : if (CLIB_DEBUG > 0)
345 : {
346 : vlib_node_t *node, *next;
347 61137336 : node = vec_elt (nm->nodes, n->node_index);
348 61137315 : next = vec_elt (nm->nodes, node->next_nodes[next_index]);
349 61137238 : ASSERT (nf->node_runtime_index == next->runtime_index);
350 : }
351 :
352 61137364 : return nf;
353 : }
354 :
355 : /** \brief Get pointer to frame by (@c node_index, @c next_index).
356 :
357 : @warning This is not a function that you should call directly.
358 : See @ref vlib_get_next_frame instead.
359 :
360 : @param vm vlib_main_t pointer, varies by thread
361 : @param node_index index of the node
362 : @param next_index graph arc index
363 :
364 : @return pointer to the requested vlib_next_frame_t
365 :
366 : @sa vlib_get_next_frame
367 : */
368 :
369 : always_inline vlib_next_frame_t *
370 254914 : vlib_node_get_next_frame (vlib_main_t * vm, u32 node_index, u32 next_index)
371 : {
372 254914 : vlib_node_main_t *nm = &vm->node_main;
373 : vlib_node_t *n;
374 : vlib_node_runtime_t *r;
375 :
376 254914 : n = vec_elt (nm->nodes, node_index);
377 254914 : r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
378 254914 : return vlib_node_runtime_get_next_frame (vm, r, next_index);
379 : }
380 :
381 : vlib_frame_t *vlib_get_next_frame_internal (vlib_main_t * vm,
382 : vlib_node_runtime_t * node,
383 : u32 next_index,
384 : u32 alloc_new_frame);
385 :
386 : #define vlib_get_next_frame_macro(vm, node, next_index, vectors, \
387 : n_vectors_left, alloc_new_frame) \
388 : do \
389 : { \
390 : vlib_frame_t *_f = vlib_get_next_frame_internal ( \
391 : (vm), (node), (next_index), (alloc_new_frame)); \
392 : u32 _n = _f->n_vectors; \
393 : (vectors) = vlib_frame_vector_args (_f) + _n * sizeof ((vectors)[0]); \
394 : (n_vectors_left) = VLIB_FRAME_SIZE - _n; \
395 : } \
396 : while (0)
397 :
398 : #define vlib_get_next_frame_macro_with_aux(vm, node, next_index, vectors, \
399 : n_vectors_left, alloc_new_frame, \
400 : aux_data, maybe_no_aux) \
401 : do \
402 : { \
403 : vlib_frame_t *_f = vlib_get_next_frame_internal ( \
404 : (vm), (node), (next_index), (alloc_new_frame)); \
405 : u32 _n = _f->n_vectors; \
406 : (vectors) = vlib_frame_vector_args (_f) + _n * sizeof ((vectors)[0]); \
407 : if ((maybe_no_aux) && (_f)->aux_offset == 0) \
408 : (aux_data) = NULL; \
409 : else \
410 : (aux_data) = vlib_frame_aux_args (_f) + _n * sizeof ((aux_data)[0]); \
411 : (n_vectors_left) = VLIB_FRAME_SIZE - _n; \
412 : } \
413 : while (0)
414 :
415 : /** \brief Get pointer to next frame vector data by
416 : (@c vlib_node_runtime_t, @c next_index).
417 : Standard single/dual loop boilerplate element.
418 : @attention This is a MACRO, with SIDE EFFECTS.
419 :
420 : @param vm vlib_main_t pointer, varies by thread
421 : @param node current node vlib_node_runtime_t pointer
422 : @param next_index requested graph arc index
423 :
424 : @return @c vectors -- pointer to next available vector slot
425 : @return @c n_vectors_left -- number of vector slots available
426 : */
427 : #define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left) \
428 : vlib_get_next_frame_macro (vm, node, next_index, vectors, n_vectors_left, \
429 : /* alloc new frame */ 0)
430 :
431 : #define vlib_get_new_next_frame(vm, node, next_index, vectors, \
432 : n_vectors_left) \
433 : vlib_get_next_frame_macro (vm, node, next_index, vectors, n_vectors_left, \
434 : /* alloc new frame */ 1)
435 :
436 : /** \brief Get pointer to next frame vector data and next frame aux data by
437 : (@c vlib_node_runtime_t, @c next_index).
438 : Standard single/dual loop boilerplate element.
439 : @attention This is a MACRO, with SIDE EFFECTS.
440 : @attention This MACRO is unsafe in case the next node does not support
441 : aux_data
442 :
443 : @param vm vlib_main_t pointer, varies by thread
444 : @param node current node vlib_node_runtime_t pointer
445 : @param next_index requested graph arc index
446 :
447 : @return @c vectors -- pointer to next available vector slot
448 : @return @c aux_data -- pointer to next available aux data slot
449 : @return @c n_vectors_left -- number of vector slots available
450 : */
451 : #define vlib_get_next_frame_with_aux(vm, node, next_index, vectors, aux_data, \
452 : n_vectors_left) \
453 : vlib_get_next_frame_macro_with_aux ( \
454 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 0, \
455 : aux_data, /* maybe_no_aux */ 0)
456 :
457 : #define vlib_get_new_next_frame_with_aux(vm, node, next_index, vectors, \
458 : aux_data, n_vectors_left) \
459 : vlib_get_next_frame_macro_with_aux ( \
460 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 1, \
461 : aux_data, /* maybe_no_aux */ 0)
462 :
463 : /** \brief Get pointer to next frame vector data and next frame aux data by
464 : (@c vlib_node_runtime_t, @c next_index).
465 : Standard single/dual loop boilerplate element.
466 : @attention This is a MACRO, with SIDE EFFECTS.
467 : @attention This MACRO is safe in case the next node does not support aux_data.
468 : In that case aux_data is set to NULL.
469 :
470 : @param vm vlib_main_t pointer, varies by thread
471 : @param node current node vlib_node_runtime_t pointer
472 : @param next_index requested graph arc index
473 :
474 : @return @c vectors -- pointer to next available vector slot
475 : @return @c aux_data -- pointer to next available aux data slot
476 : @return @c n_vectors_left -- number of vector slots available
477 : */
478 : #define vlib_get_next_frame_with_aux_safe(vm, node, next_index, vectors, \
479 : aux_data, n_vectors_left) \
480 : vlib_get_next_frame_macro_with_aux ( \
481 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 0, \
482 : aux_data, /* maybe_no_aux */ 1)
483 :
484 : #define vlib_get_new_next_frame_with_aux_safe(vm, node, next_index, vectors, \
485 : aux_data, n_vectors_left) \
486 : vlib_get_next_frame_macro_with_aux ( \
487 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 1, \
488 : aux_data, /* maybe_no_aux */ 1)
489 :
490 : /** \brief Release pointer to next frame vector data.
491 : Standard single/dual loop boilerplate element.
492 : @param vm vlib_main_t pointer, varies by thread
493 : @param r current node vlib_node_runtime_t pointer
494 : @param next_index graph arc index
495 : @param n_packets_left number of slots still available in vector
496 : */
497 : void
498 : vlib_put_next_frame (vlib_main_t * vm,
499 : vlib_node_runtime_t * r,
500 : u32 next_index, u32 n_packets_left);
501 :
502 : /* Combination get plus put. Returns vector argument just added. */
503 : #define vlib_set_next_frame(vm,node,next_index,v) \
504 : ({ \
505 : uword _n_left; \
506 : vlib_get_next_frame ((vm), (node), (next_index), (v), _n_left); \
507 : ASSERT (_n_left > 0); \
508 : vlib_put_next_frame ((vm), (node), (next_index), _n_left - 1); \
509 : (v); \
510 : })
511 :
512 : #define vlib_set_next_frame_with_aux_safe(vm, node, next_index, v, aux) \
513 : ({ \
514 : uword _n_left; \
515 : vlib_get_next_frame_with_aux_safe ((vm), (node), (next_index), (v), \
516 : (aux), _n_left); \
517 : ASSERT (_n_left > 0); \
518 : vlib_put_next_frame ((vm), (node), (next_index), _n_left - 1); \
519 : (v); \
520 : })
521 :
522 : always_inline void
523 947822 : vlib_set_next_frame_buffer (vlib_main_t * vm,
524 : vlib_node_runtime_t * node,
525 : u32 next_index, u32 buffer_index)
526 : {
527 : u32 *p;
528 947822 : p = vlib_set_next_frame (vm, node, next_index, p);
529 947822 : p[0] = buffer_index;
530 947822 : }
531 :
532 : always_inline void
533 : vlib_set_next_frame_buffer_with_aux_safe (vlib_main_t *vm,
534 : vlib_node_runtime_t *node,
535 : u32 next_index, u32 buffer_index,
536 : u32 aux)
537 : {
538 : u32 *p;
539 : u32 *a;
540 : p = vlib_set_next_frame_with_aux_safe (vm, node, next_index, p, a);
541 : p[0] = buffer_index;
542 : if (a)
543 : a[0] = aux;
544 : }
545 :
546 : vlib_frame_t *vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index);
547 : void vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index,
548 : vlib_frame_t * f);
549 :
550 : always_inline uword
551 1063576 : vlib_in_process_context (vlib_main_t * vm)
552 : {
553 1063576 : return vm->node_main.current_process_index != ~0;
554 : }
555 :
556 : always_inline vlib_process_t *
557 1063576 : vlib_get_current_process (vlib_main_t * vm)
558 : {
559 1063576 : vlib_node_main_t *nm = &vm->node_main;
560 1063576 : if (vlib_in_process_context (vm))
561 1063576 : return vec_elt (nm->processes, nm->current_process_index);
562 0 : return 0;
563 : }
564 :
565 : always_inline uword
566 6 : vlib_current_process (vlib_main_t * vm)
567 : {
568 6 : return vlib_get_current_process (vm)->node_runtime.node_index;
569 : }
570 :
571 : always_inline u32
572 0 : vlib_get_current_process_node_index (vlib_main_t * vm)
573 : {
574 0 : vlib_process_t *process = vlib_get_current_process (vm);
575 0 : return process->node_runtime.node_index;
576 : }
577 :
578 : /** Returns TRUE if a process suspend time is less than 10us
579 : @param dt - remaining poll time in seconds
580 : @returns 1 if dt < 10e-6, 0 otherwise
581 : */
582 : always_inline uword
583 2254270 : vlib_process_suspend_time_is_zero (f64 dt)
584 : {
585 2254270 : return dt < 10e-6;
586 : }
587 :
588 : /** Suspend a vlib cooperative multi-tasking thread for a period of time
589 : @param vm - vlib_main_t *
590 : @param dt - suspend interval in seconds
591 : @returns VLIB_PROCESS_RESUME_LONGJMP_RESUME, routinely ignored
592 : */
593 :
594 : always_inline uword
595 1446959 : vlib_process_suspend (vlib_main_t * vm, f64 dt)
596 : {
597 : uword r;
598 1446959 : vlib_node_main_t *nm = &vm->node_main;
599 1446959 : vlib_process_t *p = vec_elt (nm->processes, nm->current_process_index);
600 :
601 1446959 : if (vlib_process_suspend_time_is_zero (dt))
602 0 : return VLIB_PROCESS_RESUME_LONGJMP_RESUME;
603 :
604 1446959 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK;
605 1446959 : r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
606 2893215 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
607 : {
608 : /* expiration time in 10us ticks */
609 1446959 : p->resume_clock_interval = dt * 1e5;
610 1446959 : vlib_process_start_switch_stack (vm, 0);
611 1446959 : clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
612 : }
613 : else
614 1446257 : vlib_process_finish_switch_stack (vm);
615 :
616 1446257 : return r;
617 : }
618 :
619 : always_inline void
620 0 : vlib_process_free_event_type (vlib_process_t * p, uword t,
621 : uword is_one_time_event)
622 : {
623 0 : ASSERT (!pool_is_free_index (p->event_type_pool, t));
624 0 : pool_put_index (p->event_type_pool, t);
625 0 : if (is_one_time_event)
626 0 : p->one_time_event_type_bitmap =
627 0 : clib_bitmap_andnoti (p->one_time_event_type_bitmap, t);
628 0 : }
629 :
630 : always_inline void
631 486154 : vlib_process_maybe_free_event_type (vlib_process_t * p, uword t)
632 : {
633 486154 : ASSERT (!pool_is_free_index (p->event_type_pool, t));
634 486154 : if (clib_bitmap_get (p->one_time_event_type_bitmap, t))
635 0 : vlib_process_free_event_type (p, t, /* is_one_time_event */ 1);
636 486154 : }
637 :
638 : always_inline void *
639 7393 : vlib_process_get_event_data (vlib_main_t * vm,
640 : uword * return_event_type_opaque)
641 : {
642 7393 : vlib_node_main_t *nm = &vm->node_main;
643 : vlib_process_t *p;
644 : vlib_process_event_type_t *et;
645 : uword t;
646 : void *event_data_vector;
647 :
648 7393 : p = vec_elt (nm->processes, nm->current_process_index);
649 :
650 : /* Find first type with events ready.
651 : Return invalid type when there's nothing there. */
652 7393 : t = clib_bitmap_first_set (p->non_empty_event_type_bitmap);
653 7393 : if (t == ~0)
654 7327 : return 0;
655 :
656 66 : p->non_empty_event_type_bitmap =
657 66 : clib_bitmap_andnoti (p->non_empty_event_type_bitmap, t);
658 :
659 66 : ASSERT (_vec_len (p->pending_event_data_by_type_index[t]) > 0);
660 66 : event_data_vector = p->pending_event_data_by_type_index[t];
661 66 : p->pending_event_data_by_type_index[t] = 0;
662 :
663 66 : et = pool_elt_at_index (p->event_type_pool, t);
664 :
665 : /* Return user's opaque value and possibly index. */
666 66 : *return_event_type_opaque = et->opaque;
667 :
668 66 : vlib_process_maybe_free_event_type (p, t);
669 :
670 66 : return event_data_vector;
671 : }
672 :
673 : /* Return event data vector for later reuse. We reuse event data to avoid
674 : repeatedly allocating event vectors in cases where we care about speed. */
675 : always_inline void
676 28 : vlib_process_put_event_data (vlib_main_t * vm, void *event_data)
677 : {
678 28 : vlib_node_main_t *nm = &vm->node_main;
679 28 : vec_add1 (nm->recycled_event_data_vectors, event_data);
680 28 : }
681 :
682 : /** Return the first event type which has occurred and a vector of per-event
683 : data of that type, or a timeout indication
684 :
685 : @param vm - vlib_main_t pointer
686 : @param data_vector - pointer to a (uword *) vector to receive event data
687 : @returns either an event type and a vector of per-event instance data,
688 : or ~0 to indicate a timeout.
689 : */
690 :
691 : always_inline uword
692 847010 : vlib_process_get_events (vlib_main_t * vm, uword ** data_vector)
693 : {
694 847010 : vlib_node_main_t *nm = &vm->node_main;
695 : vlib_process_t *p;
696 : vlib_process_event_type_t *et;
697 : uword r, t, l;
698 :
699 847010 : p = vec_elt (nm->processes, nm->current_process_index);
700 :
701 : /* Find first type with events ready.
702 : Return invalid type when there's nothing there. */
703 847010 : t = clib_bitmap_first_set (p->non_empty_event_type_bitmap);
704 847010 : if (t == ~0)
705 360922 : return t;
706 :
707 486088 : p->non_empty_event_type_bitmap =
708 486088 : clib_bitmap_andnoti (p->non_empty_event_type_bitmap, t);
709 :
710 486088 : l = _vec_len (p->pending_event_data_by_type_index[t]);
711 486088 : if (data_vector)
712 486064 : vec_add (*data_vector, p->pending_event_data_by_type_index[t], l);
713 486088 : vec_set_len (p->pending_event_data_by_type_index[t], 0);
714 :
715 486088 : et = pool_elt_at_index (p->event_type_pool, t);
716 :
717 : /* Return user's opaque value. */
718 486088 : r = et->opaque;
719 :
720 486088 : vlib_process_maybe_free_event_type (p, t);
721 :
722 486088 : return r;
723 : }
724 :
725 : always_inline uword
726 : vlib_process_get_events_helper (vlib_process_t * p, uword t,
727 : uword ** data_vector)
728 : {
729 : uword l;
730 :
731 : p->non_empty_event_type_bitmap =
732 : clib_bitmap_andnoti (p->non_empty_event_type_bitmap, t);
733 :
734 : l = _vec_len (p->pending_event_data_by_type_index[t]);
735 : if (data_vector)
736 : vec_add (*data_vector, p->pending_event_data_by_type_index[t], l);
737 : vec_set_len (p->pending_event_data_by_type_index[t], 0);
738 :
739 : vlib_process_maybe_free_event_type (p, t);
740 :
741 : return l;
742 : }
743 :
744 : /* As above but query as specified type of event. Returns number of
745 : events found. */
746 : always_inline uword
747 : vlib_process_get_events_with_type (vlib_main_t * vm, uword ** data_vector,
748 : uword with_type_opaque)
749 : {
750 : vlib_node_main_t *nm = &vm->node_main;
751 : vlib_process_t *p;
752 : uword t, *h;
753 :
754 : p = vec_elt (nm->processes, nm->current_process_index);
755 : h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
756 : if (!h)
757 : /* This can happen when an event has not yet been
758 : signaled with given opaque type. */
759 : return 0;
760 :
761 : t = h[0];
762 : if (!clib_bitmap_get (p->non_empty_event_type_bitmap, t))
763 : return 0;
764 :
765 : return vlib_process_get_events_helper (p, t, data_vector);
766 : }
767 :
768 : always_inline uword *
769 34253 : vlib_process_wait_for_event (vlib_main_t * vm)
770 : {
771 34253 : vlib_node_main_t *nm = &vm->node_main;
772 : vlib_process_t *p;
773 : uword r;
774 :
775 34253 : p = vec_elt (nm->processes, nm->current_process_index);
776 34253 : if (clib_bitmap_is_zero (p->non_empty_event_type_bitmap))
777 : {
778 33867 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
779 : r =
780 33867 : clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
781 56279 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
782 : {
783 33867 : vlib_process_start_switch_stack (vm, 0);
784 33867 : clib_longjmp (&p->return_longjmp,
785 : VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
786 : }
787 : else
788 22412 : vlib_process_finish_switch_stack (vm);
789 : }
790 :
791 22798 : return p->non_empty_event_type_bitmap;
792 : }
793 :
794 : always_inline uword
795 : vlib_process_wait_for_one_time_event (vlib_main_t * vm,
796 : uword ** data_vector,
797 : uword with_type_index)
798 : {
799 : vlib_node_main_t *nm = &vm->node_main;
800 : vlib_process_t *p;
801 : uword r;
802 :
803 : p = vec_elt (nm->processes, nm->current_process_index);
804 : ASSERT (!pool_is_free_index (p->event_type_pool, with_type_index));
805 : while (!clib_bitmap_get (p->non_empty_event_type_bitmap, with_type_index))
806 : {
807 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
808 : r =
809 : clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
810 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
811 : {
812 : vlib_process_start_switch_stack (vm, 0);
813 : clib_longjmp (&p->return_longjmp,
814 : VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
815 : }
816 : else
817 : vlib_process_finish_switch_stack (vm);
818 : }
819 :
820 : return vlib_process_get_events_helper (p, with_type_index, data_vector);
821 : }
822 :
823 : always_inline uword
824 : vlib_process_wait_for_event_with_type (vlib_main_t * vm,
825 : uword ** data_vector,
826 : uword with_type_opaque)
827 : {
828 : vlib_node_main_t *nm = &vm->node_main;
829 : vlib_process_t *p;
830 : uword r, *h;
831 :
832 : p = vec_elt (nm->processes, nm->current_process_index);
833 : h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
834 : while (!h || !clib_bitmap_get (p->non_empty_event_type_bitmap, h[0]))
835 : {
836 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
837 : r =
838 : clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
839 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
840 : {
841 : vlib_process_start_switch_stack (vm, 0);
842 : clib_longjmp (&p->return_longjmp,
843 : VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
844 : }
845 : else
846 : vlib_process_finish_switch_stack (vm);
847 :
848 : /* See if unknown event type has been signaled now. */
849 : if (!h)
850 : h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
851 : }
852 :
853 : return vlib_process_get_events_helper (p, h[0], data_vector);
854 : }
855 :
856 : /** Suspend a cooperative multi-tasking thread
857 : Waits for an event, or for the indicated number of seconds to elapse
858 : @param vm - vlib_main_t pointer
859 : @param dt - timeout, in seconds.
860 : @returns the remaining time interval
861 : */
862 :
863 : always_inline f64
864 807272 : vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt)
865 : {
866 807272 : vlib_node_main_t *nm = &vm->node_main;
867 : vlib_process_t *p;
868 : f64 wakeup_time;
869 : uword r;
870 :
871 807272 : p = vec_elt (nm->processes, nm->current_process_index);
872 :
873 807272 : if (vlib_process_suspend_time_is_zero (dt)
874 804510 : || !clib_bitmap_is_zero (p->non_empty_event_type_bitmap))
875 3305 : return dt;
876 :
877 803967 : wakeup_time = vlib_time_now (vm) + dt;
878 :
879 : /* Suspend waiting for both clock and event to occur. */
880 803967 : p->flags |= (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
881 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK);
882 :
883 803967 : r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
884 1594582 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
885 : {
886 803967 : p->resume_clock_interval = dt * 1e5;
887 803967 : vlib_process_start_switch_stack (vm, 0);
888 803967 : clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
889 : }
890 : else
891 790615 : vlib_process_finish_switch_stack (vm);
892 :
893 : /* Return amount of time still left to sleep.
894 : If <= 0 then we've been waken up by the clock (and not an event). */
895 790615 : return wakeup_time - vlib_time_now (vm);
896 : }
897 :
898 : always_inline vlib_process_event_type_t *
899 4021 : vlib_process_new_event_type (vlib_process_t * p, uword with_type_opaque)
900 : {
901 : vlib_process_event_type_t *et;
902 4021 : pool_get (p->event_type_pool, et);
903 4021 : et->opaque = with_type_opaque;
904 4021 : return et;
905 : }
906 :
907 : always_inline uword
908 : vlib_process_create_one_time_event (vlib_main_t * vm, uword node_index,
909 : uword with_type_opaque)
910 : {
911 : vlib_node_main_t *nm = &vm->node_main;
912 : vlib_node_t *n = vlib_get_node (vm, node_index);
913 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
914 : vlib_process_event_type_t *et;
915 : uword t;
916 :
917 : et = vlib_process_new_event_type (p, with_type_opaque);
918 : t = et - p->event_type_pool;
919 : p->one_time_event_type_bitmap =
920 : clib_bitmap_ori (p->one_time_event_type_bitmap, t);
921 : return t;
922 : }
923 :
924 : always_inline void
925 : vlib_process_delete_one_time_event (vlib_main_t * vm, uword node_index,
926 : uword t)
927 : {
928 : vlib_node_main_t *nm = &vm->node_main;
929 : vlib_node_t *n = vlib_get_node (vm, node_index);
930 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
931 :
932 : ASSERT (clib_bitmap_get (p->one_time_event_type_bitmap, t));
933 : vlib_process_free_event_type (p, t, /* is_one_time_event */ 1);
934 : }
935 :
936 : always_inline void *
937 500034 : vlib_process_signal_event_helper (vlib_node_main_t * nm,
938 : vlib_node_t * n,
939 : vlib_process_t * p,
940 : uword t,
941 : uword n_data_elts, uword n_data_elt_bytes)
942 : {
943 : uword p_flags, add_to_pending, delete_from_wheel;
944 : u8 *data_to_be_written_by_caller;
945 500034 : vec_attr_t va = { .elt_sz = n_data_elt_bytes };
946 :
947 500034 : ASSERT (n->type == VLIB_NODE_TYPE_PROCESS);
948 :
949 500034 : ASSERT (!pool_is_free_index (p->event_type_pool, t));
950 :
951 500034 : vec_validate (p->pending_event_data_by_type_index, t);
952 :
953 : /* Resize data vector and return caller's data to be written. */
954 : {
955 500034 : u8 *data_vec = p->pending_event_data_by_type_index[t];
956 : uword l;
957 :
958 500034 : if (!data_vec && vec_len (nm->recycled_event_data_vectors))
959 : {
960 25 : data_vec = vec_pop (nm->recycled_event_data_vectors);
961 25 : vec_reset_length (data_vec);
962 : }
963 :
964 500034 : l = vec_len (data_vec);
965 :
966 500034 : data_vec = _vec_realloc_internal (data_vec, l + n_data_elts, &va);
967 :
968 500034 : p->pending_event_data_by_type_index[t] = data_vec;
969 500034 : data_to_be_written_by_caller = data_vec + l * n_data_elt_bytes;
970 : }
971 :
972 500034 : p->non_empty_event_type_bitmap =
973 500034 : clib_bitmap_ori (p->non_empty_event_type_bitmap, t);
974 :
975 500034 : p_flags = p->flags;
976 :
977 : /* Event was already signalled? */
978 500034 : add_to_pending = (p_flags & VLIB_PROCESS_RESUME_PENDING) == 0;
979 :
980 : /* Process will resume when suspend time elapses? */
981 500034 : delete_from_wheel = 0;
982 500034 : if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
983 : {
984 : /* Waiting for both event and clock? */
985 456552 : if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)
986 : {
987 454999 : if (!TW (tw_timer_handle_is_free)
988 454999 : ((TWT (tw_timer_wheel) *) nm->timing_wheel,
989 : p->stop_timer_handle))
990 442547 : delete_from_wheel = 1;
991 : else
992 : /* timer just popped so process should already be on the list */
993 12452 : add_to_pending = 0;
994 : }
995 : else
996 : /* Waiting only for clock. Event will be queue and may be
997 : handled when timer expires. */
998 1553 : add_to_pending = 0;
999 : }
1000 :
1001 : /* Never add current process to pending vector since current process is
1002 : already running. */
1003 500034 : add_to_pending &= nm->current_process_index != n->runtime_index;
1004 :
1005 500034 : if (add_to_pending)
1006 : {
1007 464975 : u32 x = vlib_timing_wheel_data_set_suspended_process (n->runtime_index);
1008 464975 : p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING;
1009 464975 : vec_add1 (nm->data_from_advancing_timing_wheel, x);
1010 464975 : if (delete_from_wheel)
1011 : {
1012 442547 : TW (tw_timer_stop)
1013 442547 : ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
1014 442547 : p->stop_timer_handle = ~0;
1015 : }
1016 : }
1017 :
1018 500034 : return data_to_be_written_by_caller;
1019 : }
1020 :
1021 : always_inline void *
1022 500034 : vlib_process_signal_event_data (vlib_main_t * vm,
1023 : uword node_index,
1024 : uword type_opaque,
1025 : uword n_data_elts, uword n_data_elt_bytes)
1026 : {
1027 500034 : vlib_node_main_t *nm = &vm->node_main;
1028 500034 : vlib_node_t *n = vlib_get_node (vm, node_index);
1029 500034 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
1030 : uword *h, t;
1031 :
1032 : /* Must be in main thread */
1033 500034 : ASSERT (vlib_get_thread_index () == 0);
1034 :
1035 500034 : h = hash_get (p->event_type_index_by_type_opaque, type_opaque);
1036 500034 : if (!h)
1037 : {
1038 : vlib_process_event_type_t *et =
1039 4021 : vlib_process_new_event_type (p, type_opaque);
1040 4021 : t = et - p->event_type_pool;
1041 4021 : hash_set (p->event_type_index_by_type_opaque, type_opaque, t);
1042 : }
1043 : else
1044 496013 : t = h[0];
1045 :
1046 500034 : return vlib_process_signal_event_helper (nm, n, p, t, n_data_elts,
1047 : n_data_elt_bytes);
1048 : }
1049 :
1050 : always_inline void *
1051 : vlib_process_signal_event_at_time (vlib_main_t * vm,
1052 : f64 dt,
1053 : uword node_index,
1054 : uword type_opaque,
1055 : uword n_data_elts, uword n_data_elt_bytes)
1056 : {
1057 : vlib_node_main_t *nm = &vm->node_main;
1058 : vlib_node_t *n = vlib_get_node (vm, node_index);
1059 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
1060 : uword *h, t;
1061 :
1062 : h = hash_get (p->event_type_index_by_type_opaque, type_opaque);
1063 : if (!h)
1064 : {
1065 : vlib_process_event_type_t *et =
1066 : vlib_process_new_event_type (p, type_opaque);
1067 : t = et - p->event_type_pool;
1068 : hash_set (p->event_type_index_by_type_opaque, type_opaque, t);
1069 : }
1070 : else
1071 : t = h[0];
1072 :
1073 : if (vlib_process_suspend_time_is_zero (dt))
1074 : return vlib_process_signal_event_helper (nm, n, p, t, n_data_elts,
1075 : n_data_elt_bytes);
1076 : else
1077 : {
1078 : vlib_signal_timed_event_data_t *te;
1079 :
1080 : pool_get_aligned (nm->signal_timed_event_data_pool, te, sizeof (te[0]));
1081 :
1082 : te->n_data_elts = n_data_elts;
1083 : te->n_data_elt_bytes = n_data_elt_bytes;
1084 : te->n_data_bytes = n_data_elts * n_data_elt_bytes;
1085 :
1086 : /* Assert that structure fields are big enough. */
1087 : ASSERT (te->n_data_elts == n_data_elts);
1088 : ASSERT (te->n_data_elt_bytes == n_data_elt_bytes);
1089 : ASSERT (te->n_data_bytes == n_data_elts * n_data_elt_bytes);
1090 :
1091 : te->process_node_index = n->runtime_index;
1092 : te->event_type_index = t;
1093 :
1094 : p->stop_timer_handle =
1095 : TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1096 : vlib_timing_wheel_data_set_timed_event
1097 : (te - nm->signal_timed_event_data_pool),
1098 : 0 /* timer_id */ ,
1099 : (vlib_time_now (vm) + dt) * 1e5);
1100 :
1101 : /* Inline data big enough to hold event? */
1102 : if (te->n_data_bytes < sizeof (te->inline_event_data))
1103 : return te->inline_event_data;
1104 : else
1105 : {
1106 : te->event_data_as_vector = 0;
1107 : vec_resize (te->event_data_as_vector, te->n_data_bytes);
1108 : return te->event_data_as_vector;
1109 : }
1110 : }
1111 : }
1112 :
1113 : always_inline void *
1114 : vlib_process_signal_one_time_event_data (vlib_main_t * vm,
1115 : uword node_index,
1116 : uword type_index,
1117 : uword n_data_elts,
1118 : uword n_data_elt_bytes)
1119 : {
1120 : vlib_node_main_t *nm = &vm->node_main;
1121 : vlib_node_t *n = vlib_get_node (vm, node_index);
1122 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
1123 : return vlib_process_signal_event_helper (nm, n, p, type_index, n_data_elts,
1124 : n_data_elt_bytes);
1125 : }
1126 :
1127 : always_inline void
1128 499960 : vlib_process_signal_event (vlib_main_t * vm,
1129 : uword node_index, uword type_opaque, uword data)
1130 : {
1131 499960 : uword *d = vlib_process_signal_event_data (vm, node_index, type_opaque,
1132 : 1 /* elts */ , sizeof (uword));
1133 499960 : d[0] = data;
1134 499960 : }
1135 :
1136 : always_inline void
1137 0 : vlib_process_signal_event_pointer (vlib_main_t * vm,
1138 : uword node_index,
1139 : uword type_opaque, void *data)
1140 : {
1141 0 : void **d = vlib_process_signal_event_data (vm, node_index, type_opaque,
1142 : 1 /* elts */ , sizeof (data));
1143 0 : d[0] = data;
1144 0 : }
1145 :
1146 : /**
1147 : * Signal event to process from any thread.
1148 : *
1149 : * When in doubt, use this.
1150 : */
1151 : always_inline void
1152 169 : vlib_process_signal_event_mt (vlib_main_t * vm,
1153 : uword node_index, uword type_opaque, uword data)
1154 : {
1155 169 : if (vlib_get_thread_index () != 0)
1156 : {
1157 0 : vlib_process_signal_event_mt_args_t args = {
1158 : .node_index = node_index,
1159 : .type_opaque = type_opaque,
1160 : .data = data,
1161 : };
1162 0 : vlib_rpc_call_main_thread (vlib_process_signal_event_mt_helper,
1163 : (u8 *) & args, sizeof (args));
1164 : }
1165 : else
1166 169 : vlib_process_signal_event (vm, node_index, type_opaque, data);
1167 169 : }
1168 :
1169 : always_inline void
1170 : vlib_process_signal_one_time_event (vlib_main_t * vm,
1171 : uword node_index,
1172 : uword type_index, uword data)
1173 : {
1174 : uword *d =
1175 : vlib_process_signal_one_time_event_data (vm, node_index, type_index,
1176 : 1 /* elts */ , sizeof (uword));
1177 : d[0] = data;
1178 : }
1179 :
1180 : always_inline void
1181 : vlib_signal_one_time_waiting_process (vlib_main_t * vm,
1182 : vlib_one_time_waiting_process_t * p)
1183 : {
1184 : vlib_process_signal_one_time_event (vm, p->node_index, p->one_time_event,
1185 : /* data */ ~0);
1186 : clib_memset (p, ~0, sizeof (p[0]));
1187 : }
1188 :
1189 : always_inline void
1190 : vlib_signal_one_time_waiting_process_vector (vlib_main_t * vm,
1191 : vlib_one_time_waiting_process_t
1192 : ** wps)
1193 : {
1194 : vlib_one_time_waiting_process_t *wp;
1195 : vec_foreach (wp, *wps) vlib_signal_one_time_waiting_process (vm, wp);
1196 : vec_free (*wps);
1197 : }
1198 :
1199 : always_inline void
1200 : vlib_current_process_wait_for_one_time_event (vlib_main_t * vm,
1201 : vlib_one_time_waiting_process_t
1202 : * p)
1203 : {
1204 : p->node_index = vlib_current_process (vm);
1205 : p->one_time_event = vlib_process_create_one_time_event (vm, p->node_index, /* type opaque */
1206 : ~0);
1207 : vlib_process_wait_for_one_time_event (vm,
1208 : /* don't care about data */ 0,
1209 : p->one_time_event);
1210 : }
1211 :
1212 : always_inline void
1213 : vlib_current_process_wait_for_one_time_event_vector (vlib_main_t * vm,
1214 : vlib_one_time_waiting_process_t
1215 : ** wps)
1216 : {
1217 : vlib_one_time_waiting_process_t *wp;
1218 : vec_add2 (*wps, wp, 1);
1219 : vlib_current_process_wait_for_one_time_event (vm, wp);
1220 : }
1221 :
1222 : always_inline u32
1223 797305000 : vlib_node_runtime_update_main_loop_vector_stats (vlib_main_t * vm,
1224 : vlib_node_runtime_t * node,
1225 : uword n_vectors)
1226 : {
1227 : u32 i, d, vi0, vi1;
1228 : u32 i0, i1;
1229 :
1230 797305000 : ASSERT (is_pow2 (ARRAY_LEN (node->main_loop_vector_stats)));
1231 797309000 : i = ((vm->main_loop_count >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE)
1232 : & (ARRAY_LEN (node->main_loop_vector_stats) - 1));
1233 797309000 : i0 = i ^ 0;
1234 797309000 : i1 = i ^ 1;
1235 797309000 : d = ((vm->main_loop_count >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE)
1236 : -
1237 797309000 : (node->main_loop_count_last_dispatch >>
1238 : VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE));
1239 797309000 : vi0 = node->main_loop_vector_stats[i0];
1240 797309000 : vi1 = node->main_loop_vector_stats[i1];
1241 797309000 : vi0 = d == 0 ? vi0 : 0;
1242 797309000 : vi1 = d <= 1 ? vi1 : 0;
1243 797309000 : vi0 += n_vectors;
1244 797309000 : node->main_loop_vector_stats[i0] = vi0;
1245 797309000 : node->main_loop_vector_stats[i1] = vi1;
1246 797309000 : node->main_loop_count_last_dispatch = vm->main_loop_count;
1247 : /* Return previous counter. */
1248 797309000 : return node->main_loop_vector_stats[i1];
1249 : }
1250 :
1251 : always_inline f64
1252 : vlib_node_vectors_per_main_loop_as_float (vlib_main_t * vm, u32 node_index)
1253 : {
1254 : vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index);
1255 : u32 v;
1256 :
1257 : v = vlib_node_runtime_update_main_loop_vector_stats (vm, rt, /* n_vectors */
1258 : 0);
1259 : return (f64) v / (1 << VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE);
1260 : }
1261 :
1262 : always_inline u32
1263 : vlib_node_vectors_per_main_loop_as_integer (vlib_main_t * vm, u32 node_index)
1264 : {
1265 : vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index);
1266 : u32 v;
1267 :
1268 : v = vlib_node_runtime_update_main_loop_vector_stats (vm, rt, /* n_vectors */
1269 : 0);
1270 : return v >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE;
1271 : }
1272 :
1273 : void vlib_frame_free (vlib_main_t *vm, vlib_frame_t *f);
1274 :
1275 : /* Return the edge index if present, ~0 otherwise */
1276 : uword vlib_node_get_next (vlib_main_t * vm, uword node, uword next_node);
1277 :
1278 : /* Add next node to given node in given slot. */
1279 : uword
1280 : vlib_node_add_next_with_slot (vlib_main_t * vm,
1281 : uword node, uword next_node, uword slot);
1282 :
1283 : /* As above but adds to end of node's next vector. */
1284 : always_inline uword
1285 435267 : vlib_node_add_next (vlib_main_t * vm, uword node, uword next_node)
1286 : {
1287 435267 : return vlib_node_add_next_with_slot (vm, node, next_node, ~0);
1288 : }
1289 :
1290 : /* Add next node to given node in given slot. */
1291 : uword
1292 : vlib_node_add_named_next_with_slot (vlib_main_t * vm,
1293 : uword node, char *next_name, uword slot);
1294 :
1295 : /* As above but adds to end of node's next vector. */
1296 : always_inline uword
1297 412850 : vlib_node_add_named_next (vlib_main_t * vm, uword node, char *name)
1298 : {
1299 412850 : return vlib_node_add_named_next_with_slot (vm, node, name, ~0);
1300 : }
1301 :
1302 : /**
1303 : * Get list of nodes
1304 : */
1305 : void
1306 : vlib_node_get_nodes (vlib_main_t * vm, u32 max_threads, int include_stats,
1307 : int barrier_sync, vlib_node_t **** node_dupsp,
1308 : vlib_main_t *** stat_vmsp);
1309 :
1310 : /* Query node given name. */
1311 : vlib_node_t *vlib_get_node_by_name (vlib_main_t * vm, u8 * name);
1312 :
1313 : /* Rename a node. */
1314 : void vlib_node_rename (vlib_main_t * vm, u32 node_index, char *fmt, ...);
1315 :
1316 : /* Register new packet processing node. Nodes can be registered
1317 : dynamically via this call or statically via the VLIB_REGISTER_NODE
1318 : macro. */
1319 : u32 vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r,
1320 : char *fmt, ...);
1321 :
1322 : /* Register all node function variants */
1323 : void vlib_register_all_node_march_variants (vlib_main_t *vm);
1324 :
1325 : /* Register all static nodes registered via VLIB_REGISTER_NODE. */
1326 : void vlib_register_all_static_nodes (vlib_main_t * vm);
1327 :
1328 : /* Start a process. */
1329 : void vlib_start_process (vlib_main_t * vm, uword process_index);
1330 :
1331 : /* Sync up runtime and main node stats. */
1332 : void vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n);
1333 : void vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r,
1334 : uword n_calls, uword n_vectors,
1335 : uword n_clocks);
1336 : void vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r,
1337 : uword n_calls, uword n_vectors,
1338 : uword n_clocks);
1339 :
1340 : /* Node graph initialization function. */
1341 : clib_error_t *vlib_node_main_init (vlib_main_t * vm);
1342 :
1343 : format_function_t format_vlib_node_graph;
1344 : format_function_t format_vlib_node_name;
1345 : format_function_t format_vlib_next_node_name;
1346 : format_function_t format_vlib_node_and_next;
1347 : format_function_t format_vlib_cpu_time;
1348 : format_function_t format_vlib_time;
1349 : /* Parse node name -> node index. */
1350 : unformat_function_t unformat_vlib_node;
1351 :
1352 : always_inline void
1353 94626956 : vlib_node_increment_counter (vlib_main_t * vm, u32 node_index,
1354 : u32 counter_index, u64 increment)
1355 : {
1356 94626956 : vlib_node_t *n = vlib_get_node (vm, node_index);
1357 94612654 : vlib_error_main_t *em = &vm->error_main;
1358 94612654 : u32 node_counter_base_index = n->error_heap_index;
1359 94612654 : em->counters[node_counter_base_index + counter_index] += increment;
1360 94612654 : }
1361 :
1362 : /** @brief Create a vlib process
1363 : * @param vm &vlib_global_main
1364 : * @param f the process node function
1365 : * @param log2_n_stack_bytes size of the process stack, defaults to 16K
1366 : * @return newly-create node index
1367 : * @warning call only on the main thread. Barrier sync required
1368 : */
1369 : u32 vlib_process_create (vlib_main_t * vm, char *name,
1370 : vlib_node_function_t * f, u32 log2_n_stack_bytes);
1371 :
1372 : always_inline int
1373 2 : vlib_node_set_dispatch_wrapper (vlib_main_t *vm, vlib_node_function_t *fn)
1374 : {
1375 2 : if (fn && vm->dispatch_wrapper_fn)
1376 0 : return 1;
1377 2 : vm->dispatch_wrapper_fn = fn;
1378 2 : return 0;
1379 : }
1380 :
1381 : int vlib_node_set_march_variant (vlib_main_t *vm, u32 node_index,
1382 : clib_march_variant_type_t march_variant);
1383 :
1384 : vlib_node_function_t *
1385 : vlib_node_get_preferred_node_fn_variant (vlib_main_t *vm,
1386 : vlib_node_fn_registration_t *regs);
1387 :
1388 : /*
1389 : * vlib_frame_bitmap functions
1390 : */
1391 :
1392 : #define VLIB_FRAME_BITMAP_N_UWORDS \
1393 : (((VLIB_FRAME_SIZE + uword_bits - 1) & ~(uword_bits - 1)) / uword_bits)
1394 :
1395 : typedef uword vlib_frame_bitmap_t[VLIB_FRAME_BITMAP_N_UWORDS];
1396 :
1397 : static_always_inline void
1398 : vlib_frame_bitmap_init (uword *bmp, u32 n_first_bits_set)
1399 : {
1400 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1401 : while (n_first_bits_set >= (sizeof (uword) * 8) && n_left)
1402 : {
1403 : bmp++[0] = ~0;
1404 : n_first_bits_set -= sizeof (uword) * 8;
1405 : n_left--;
1406 : }
1407 :
1408 : if (n_first_bits_set && n_left)
1409 : {
1410 : bmp++[0] = pow2_mask (n_first_bits_set);
1411 : n_left--;
1412 : }
1413 :
1414 : while (n_left--)
1415 : bmp++[0] = 0;
1416 : }
1417 :
1418 : static_always_inline void
1419 : vlib_frame_bitmap_set_bit_at_index (uword *bmp, uword bit_index)
1420 : {
1421 : uword_bitmap_set_bits_at_index (bmp, bit_index, 1);
1422 : }
1423 :
1424 : static_always_inline void
1425 : _vlib_frame_bitmap_clear_bit_at_index (uword *bmp, uword bit_index)
1426 : {
1427 : uword_bitmap_clear_bits_at_index (bmp, bit_index, 1);
1428 : }
1429 :
1430 : static_always_inline void
1431 : vlib_frame_bitmap_set_bits_at_index (uword *bmp, uword bit_index, uword n_bits)
1432 : {
1433 : uword_bitmap_set_bits_at_index (bmp, bit_index, n_bits);
1434 : }
1435 :
1436 : static_always_inline void
1437 : vlib_frame_bitmap_clear_bits_at_index (uword *bmp, uword bit_index,
1438 : uword n_bits)
1439 : {
1440 : uword_bitmap_clear_bits_at_index (bmp, bit_index, n_bits);
1441 : }
1442 :
1443 : static_always_inline void
1444 : vlib_frame_bitmap_clear (uword *bmp)
1445 : {
1446 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1447 : while (n_left--)
1448 : bmp++[0] = 0;
1449 : }
1450 :
1451 : static_always_inline void
1452 : vlib_frame_bitmap_xor (uword *bmp, uword *bmp2)
1453 : {
1454 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1455 : while (n_left--)
1456 : bmp++[0] ^= bmp2++[0];
1457 : }
1458 :
1459 : static_always_inline void
1460 8363700 : vlib_frame_bitmap_or (uword *bmp, uword *bmp2)
1461 : {
1462 8363700 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1463 41818500 : while (n_left--)
1464 33454800 : bmp++[0] |= bmp2++[0];
1465 8363700 : }
1466 :
1467 : static_always_inline void
1468 : vlib_frame_bitmap_and (uword *bmp, uword *bmp2)
1469 : {
1470 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1471 : while (n_left--)
1472 : bmp++[0] &= bmp2++[0];
1473 : }
1474 :
1475 : static_always_inline uword
1476 : vlib_frame_bitmap_count_set_bits (uword *bmp)
1477 : {
1478 : return uword_bitmap_count_set_bits (bmp, VLIB_FRAME_BITMAP_N_UWORDS);
1479 : }
1480 :
1481 : static_always_inline uword
1482 : vlib_frame_bitmap_is_bit_set (uword *bmp, uword bit_index)
1483 : {
1484 : return uword_bitmap_is_bit_set (bmp, bit_index);
1485 : }
1486 :
1487 : static_always_inline uword
1488 : vlib_frame_bitmap_find_first_set (uword *bmp)
1489 : {
1490 : uword rv = uword_bitmap_find_first_set (bmp);
1491 : ASSERT (rv < VLIB_FRAME_BITMAP_N_UWORDS * uword_bits);
1492 : return rv;
1493 : }
1494 :
1495 : #define foreach_vlib_frame_bitmap_set_bit_index(i, v) \
1496 : for (uword _off = 0; _off < ARRAY_LEN (v); _off++) \
1497 : for (uword _tmp = \
1498 : (v[_off]) + 0 * (uword) (i = _off * uword_bits + \
1499 : get_lowest_set_bit_index (v[_off])); \
1500 : _tmp; i = _off * uword_bits + get_lowest_set_bit_index ( \
1501 : _tmp = clear_lowest_set_bit (_tmp)))
1502 :
1503 : #endif /* included_vlib_node_funcs_h */
1504 :
1505 : /*
1506 : * fd.io coding-style-patch-verification: ON
1507 : *
1508 : * Local Variables:
1509 : * eval: (c-set-style "gnu")
1510 : * End:
1511 : */
|
