Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * node_funcs.h: processing nodes global functions/inlines
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : /** \file
41 : vlib node functions
42 : */
43 :
44 :
45 : #ifndef included_vlib_node_funcs_h
46 : #define included_vlib_node_funcs_h
47 :
48 : #include <vppinfra/clib.h>
49 : #include <vppinfra/fifo.h>
50 : #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
51 : #include <vppinfra/interrupt.h>
52 :
53 : #ifdef CLIB_SANITIZE_ADDR
54 : #include <sanitizer/asan_interface.h>
55 : #endif
56 :
57 : static_always_inline void
58 3383578 : vlib_process_start_switch_stack (vlib_main_t * vm, vlib_process_t * p)
59 : {
60 : #ifdef CLIB_SANITIZE_ADDR
61 : void *stack = p ? (void *) p->stack : vlib_thread_stacks[vm->thread_index];
62 : u32 stack_bytes =
63 : p ? (1ULL < p->log2_n_stack_bytes) : VLIB_THREAD_STACK_SIZE;
64 : __sanitizer_start_switch_fiber (&vm->asan_stack_save, stack, stack_bytes);
65 : #endif
66 3383578 : }
67 :
68 : static_always_inline void
69 3383577 : vlib_process_finish_switch_stack (vlib_main_t * vm)
70 : {
71 : #ifdef CLIB_SANITIZE_ADDR
72 : const void *bottom_old;
73 : size_t size_old;
74 :
75 : __sanitizer_finish_switch_fiber (&vm->asan_stack_save, &bottom_old,
76 : &size_old);
77 : #endif
78 3383577 : }
79 :
80 : /** \brief Get vlib node by index.
81 : @warning This function will ASSERT if @c i is out of range.
82 : @param vm vlib_main_t pointer, varies by thread
83 : @param i node index.
84 : @return pointer to the requested vlib_node_t.
85 : */
86 :
87 : always_inline vlib_node_t *
88 16509314967 : vlib_get_node (vlib_main_t * vm, u32 i)
89 : {
90 16509314967 : return vec_elt (vm->node_main.nodes, i);
91 : }
92 :
93 : /** \brief Get vlib node by graph arc (next) index.
94 : @param vm vlib_main_t pointer, varies by thread
95 : @param node_index index of original node
96 : @param next_index graph arc index
97 : @return pointer to the vlib_node_t at the end of the indicated arc
98 : */
99 :
100 : always_inline vlib_node_t *
101 0 : vlib_get_next_node (vlib_main_t * vm, u32 node_index, u32 next_index)
102 : {
103 0 : vlib_node_main_t *nm = &vm->node_main;
104 : vlib_node_t *n;
105 :
106 0 : n = vec_elt (nm->nodes, node_index);
107 0 : ASSERT (next_index < vec_len (n->next_nodes));
108 0 : return vlib_get_node (vm, n->next_nodes[next_index]);
109 : }
110 :
111 : /** \brief Get node runtime by node index.
112 : @param vm vlib_main_t pointer, varies by thread
113 : @param node_index index of node
114 : @return pointer to the indicated vlib_node_runtime_t
115 : */
116 :
117 : always_inline vlib_node_runtime_t *
118 458659438 : vlib_node_get_runtime (vlib_main_t * vm, u32 node_index)
119 : {
120 458659438 : vlib_node_main_t *nm = &vm->node_main;
121 458659438 : vlib_node_t *n = vec_elt (nm->nodes, node_index);
122 : vlib_process_t *p;
123 458661647 : if (n->type != VLIB_NODE_TYPE_PROCESS)
124 428556857 : return vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
125 : else
126 : {
127 30104486 : p = vec_elt (nm->processes, n->runtime_index);
128 30104486 : return &p->node_runtime;
129 : }
130 : }
131 :
132 : /** \brief Get node runtime private data by node index.
133 : @param vm vlib_main_t pointer, varies by thread
134 : @param node_index index of the node
135 : @return pointer to the indicated vlib_node_runtime_t private data
136 : */
137 :
138 : always_inline void *
139 221620 : vlib_node_get_runtime_data (vlib_main_t * vm, u32 node_index)
140 : {
141 221620 : vlib_node_runtime_t *r = vlib_node_get_runtime (vm, node_index);
142 221623 : return r->runtime_data;
143 : }
144 :
145 : /** \brief Set node runtime private data.
146 : @param vm vlib_main_t pointer, varies by thread
147 : @param node_index index of the node
148 : @param runtime_data arbitrary runtime private data
149 : @param n_runtime_data_bytes size of runtime private data
150 : */
151 :
152 : always_inline void
153 : vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index,
154 : void *runtime_data, u32 n_runtime_data_bytes)
155 : {
156 : vlib_node_t *n = vlib_get_node (vm, node_index);
157 : vlib_node_runtime_t *r = vlib_node_get_runtime (vm, node_index);
158 :
159 : n->runtime_data_bytes = n_runtime_data_bytes;
160 : vec_free (n->runtime_data);
161 : vec_add (n->runtime_data, runtime_data, n_runtime_data_bytes);
162 :
163 : ASSERT (vec_len (n->runtime_data) <= sizeof (vlib_node_runtime_t) -
164 : STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data));
165 :
166 : if (vec_len (n->runtime_data) > 0)
167 : clib_memcpy_fast (r->runtime_data, n->runtime_data,
168 : vec_len (n->runtime_data));
169 : }
170 :
171 : /** \brief Set node dispatch state.
172 : @param vm vlib_main_t pointer, varies by thread
173 : @param node_index index of the node
174 : @param new_state new state for node, see vlib_node_state_t
175 : */
176 : always_inline void
177 694204 : vlib_node_set_state (vlib_main_t * vm, u32 node_index,
178 : vlib_node_state_t new_state)
179 : {
180 694204 : vlib_node_main_t *nm = &vm->node_main;
181 : vlib_node_t *n;
182 : vlib_node_runtime_t *r;
183 :
184 694204 : n = vec_elt (nm->nodes, node_index);
185 694204 : if (n->type == VLIB_NODE_TYPE_PROCESS)
186 : {
187 721 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
188 721 : r = &p->node_runtime;
189 :
190 : /* When disabling make sure flags are cleared. */
191 721 : p->flags &= ~(VLIB_PROCESS_RESUME_PENDING
192 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
193 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT);
194 : }
195 : else
196 693483 : r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
197 :
198 694206 : ASSERT (new_state < VLIB_N_NODE_STATE);
199 :
200 694206 : if (n->type == VLIB_NODE_TYPE_INPUT)
201 : {
202 692758 : ASSERT (nm->input_node_counts_by_state[n->state] > 0);
203 692757 : nm->input_node_counts_by_state[n->state] -= 1;
204 692757 : nm->input_node_counts_by_state[new_state] += 1;
205 : }
206 :
207 694205 : if (PREDICT_FALSE (r->state == VLIB_NODE_STATE_DISABLED))
208 15967 : vlib_node_runtime_perf_counter (vm, r, 0, 0, 0,
209 : VLIB_NODE_RUNTIME_PERF_RESET);
210 :
211 694205 : n->state = new_state;
212 694205 : r->state = new_state;
213 694205 : }
214 :
215 : /** \brief Get node dispatch state.
216 : @param vm vlib_main_t pointer, varies by thread
217 : @param node_index index of the node
218 : @return state for node, see vlib_node_state_t
219 : */
220 : always_inline vlib_node_state_t
221 19114 : vlib_node_get_state (vlib_main_t * vm, u32 node_index)
222 : {
223 19114 : vlib_node_main_t *nm = &vm->node_main;
224 : vlib_node_t *n;
225 19114 : n = vec_elt (nm->nodes, node_index);
226 19114 : return n->state;
227 : }
228 :
229 : always_inline void
230 1652 : vlib_node_set_flag (vlib_main_t *vm, u32 node_index, u16 flag, u8 enable)
231 : {
232 : vlib_node_runtime_t *r;
233 : vlib_node_t *n;
234 :
235 1652 : n = vlib_get_node (vm, node_index);
236 1652 : r = vlib_node_get_runtime (vm, node_index);
237 :
238 1652 : if (enable)
239 : {
240 0 : n->flags |= flag;
241 0 : r->flags |= flag;
242 : }
243 : else
244 : {
245 1652 : n->flags &= ~flag;
246 1652 : r->flags &= ~flag;
247 : }
248 1652 : }
249 :
250 : always_inline void
251 458072 : vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index)
252 : {
253 458072 : vlib_node_main_t *nm = &vm->node_main;
254 458072 : vlib_node_t *n = vec_elt (nm->nodes, node_index);
255 :
256 458072 : ASSERT (n->type == VLIB_NODE_TYPE_INPUT);
257 :
258 458072 : if (vm != vlib_get_main ())
259 459 : clib_interrupt_set_atomic (nm->interrupts, n->runtime_index);
260 : else
261 457613 : clib_interrupt_set (nm->interrupts, n->runtime_index);
262 :
263 458072 : __atomic_store_n (nm->pending_interrupts, 1, __ATOMIC_RELEASE);
264 458072 : }
265 :
266 : always_inline vlib_process_t *
267 243841 : vlib_get_process_from_node (vlib_main_t * vm, vlib_node_t * node)
268 : {
269 243841 : vlib_node_main_t *nm = &vm->node_main;
270 243841 : ASSERT (node->type == VLIB_NODE_TYPE_PROCESS);
271 243841 : return vec_elt (nm->processes, node->runtime_index);
272 : }
273 :
274 : always_inline vlib_frame_t *
275 45698510 : vlib_get_frame (vlib_main_t * vm, vlib_frame_t * f)
276 : {
277 45698510 : ASSERT (f != NULL);
278 45698510 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
279 45698510 : return f;
280 : }
281 :
282 : always_inline void
283 974110 : vlib_frame_no_append (vlib_frame_t * f)
284 : {
285 974110 : f->frame_flags |= VLIB_FRAME_NO_APPEND;
286 974110 : }
287 :
288 : /** \brief Get pointer to frame vector data.
289 : @param f vlib_frame_t pointer
290 : @return pointer to first vector element in frame
291 : */
292 : always_inline void *
293 47603781 : vlib_frame_vector_args (vlib_frame_t * f)
294 : {
295 47603781 : ASSERT (f->vector_offset);
296 47603781 : return (void *) f + f->vector_offset;
297 : }
298 :
299 : /** \brief Get pointer to frame vector aux data.
300 : @param f vlib_frame_t pointer
301 : @return pointer to first vector aux data element in frame
302 : */
303 : always_inline void *
304 0 : vlib_frame_aux_args (vlib_frame_t *f)
305 : {
306 0 : ASSERT (f->aux_offset);
307 0 : return (void *) f + f->aux_offset;
308 : }
309 :
310 : /** \brief Get pointer to frame scalar data.
311 :
312 : @param f vlib_frame_t pointer
313 :
314 : @return arbitrary node scalar data
315 :
316 : @sa vlib_frame_vector_args
317 : */
318 : always_inline void *
319 6574308 : vlib_frame_scalar_args (vlib_frame_t * f)
320 : {
321 6574308 : ASSERT (f->scalar_offset);
322 6574308 : return (void *) f + f->scalar_offset;
323 : }
324 :
325 : always_inline vlib_next_frame_t *
326 52944050 : vlib_node_runtime_get_next_frame (vlib_main_t * vm,
327 : vlib_node_runtime_t * n, u32 next_index)
328 : {
329 52944050 : vlib_node_main_t *nm = &vm->node_main;
330 : vlib_next_frame_t *nf;
331 :
332 52944050 : ASSERT (next_index < n->n_next_nodes);
333 52944051 : nf = vec_elt_at_index (nm->next_frames, n->next_frame_index + next_index);
334 :
335 : if (CLIB_DEBUG > 0)
336 : {
337 : vlib_node_t *node, *next;
338 52943988 : node = vec_elt (nm->nodes, n->node_index);
339 52944026 : next = vec_elt (nm->nodes, node->next_nodes[next_index]);
340 52944020 : ASSERT (nf->node_runtime_index == next->runtime_index);
341 : }
342 :
343 52944031 : return nf;
344 : }
345 :
346 : /** \brief Get pointer to frame by (@c node_index, @c next_index).
347 :
348 : @warning This is not a function that you should call directly.
349 : See @ref vlib_get_next_frame instead.
350 :
351 : @param vm vlib_main_t pointer, varies by thread
352 : @param node_index index of the node
353 : @param next_index graph arc index
354 :
355 : @return pointer to the requested vlib_next_frame_t
356 :
357 : @sa vlib_get_next_frame
358 : */
359 :
360 : always_inline vlib_next_frame_t *
361 262160 : vlib_node_get_next_frame (vlib_main_t * vm, u32 node_index, u32 next_index)
362 : {
363 262160 : vlib_node_main_t *nm = &vm->node_main;
364 : vlib_node_t *n;
365 : vlib_node_runtime_t *r;
366 :
367 262160 : n = vec_elt (nm->nodes, node_index);
368 262160 : r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
369 262160 : return vlib_node_runtime_get_next_frame (vm, r, next_index);
370 : }
371 :
372 : vlib_frame_t *vlib_get_next_frame_internal (vlib_main_t * vm,
373 : vlib_node_runtime_t * node,
374 : u32 next_index,
375 : u32 alloc_new_frame);
376 :
377 : #define vlib_get_next_frame_macro(vm, node, next_index, vectors, \
378 : n_vectors_left, alloc_new_frame) \
379 : do \
380 : { \
381 : vlib_frame_t *_f = vlib_get_next_frame_internal ( \
382 : (vm), (node), (next_index), (alloc_new_frame)); \
383 : u32 _n = _f->n_vectors; \
384 : (vectors) = vlib_frame_vector_args (_f) + _n * sizeof ((vectors)[0]); \
385 : (n_vectors_left) = VLIB_FRAME_SIZE - _n; \
386 : } \
387 : while (0)
388 :
389 : #define vlib_get_next_frame_macro_with_aux(vm, node, next_index, vectors, \
390 : n_vectors_left, alloc_new_frame, \
391 : aux_data, maybe_no_aux) \
392 : do \
393 : { \
394 : vlib_frame_t *_f = vlib_get_next_frame_internal ( \
395 : (vm), (node), (next_index), (alloc_new_frame)); \
396 : u32 _n = _f->n_vectors; \
397 : (vectors) = vlib_frame_vector_args (_f) + _n * sizeof ((vectors)[0]); \
398 : if ((maybe_no_aux) && (_f)->aux_offset == 0) \
399 : (aux_data) = NULL; \
400 : else \
401 : (aux_data) = vlib_frame_aux_args (_f) + _n * sizeof ((aux_data)[0]); \
402 : (n_vectors_left) = VLIB_FRAME_SIZE - _n; \
403 : } \
404 : while (0)
405 :
406 : /** \brief Get pointer to next frame vector data by
407 : (@c vlib_node_runtime_t, @c next_index).
408 : Standard single/dual loop boilerplate element.
409 : @attention This is a MACRO, with SIDE EFFECTS.
410 :
411 : @param vm vlib_main_t pointer, varies by thread
412 : @param node current node vlib_node_runtime_t pointer
413 : @param next_index requested graph arc index
414 :
415 : @return @c vectors -- pointer to next available vector slot
416 : @return @c n_vectors_left -- number of vector slots available
417 : */
418 : #define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left) \
419 : vlib_get_next_frame_macro (vm, node, next_index, vectors, n_vectors_left, \
420 : /* alloc new frame */ 0)
421 :
422 : #define vlib_get_new_next_frame(vm, node, next_index, vectors, \
423 : n_vectors_left) \
424 : vlib_get_next_frame_macro (vm, node, next_index, vectors, n_vectors_left, \
425 : /* alloc new frame */ 1)
426 :
427 : /** \brief Get pointer to next frame vector data and next frame aux data by
428 : (@c vlib_node_runtime_t, @c next_index).
429 : Standard single/dual loop boilerplate element.
430 : @attention This is a MACRO, with SIDE EFFECTS.
431 : @attention This MACRO is unsafe in case the next node does not support
432 : aux_data
433 :
434 : @param vm vlib_main_t pointer, varies by thread
435 : @param node current node vlib_node_runtime_t pointer
436 : @param next_index requested graph arc index
437 :
438 : @return @c vectors -- pointer to next available vector slot
439 : @return @c aux_data -- pointer to next available aux data slot
440 : @return @c n_vectors_left -- number of vector slots available
441 : */
442 : #define vlib_get_next_frame_with_aux(vm, node, next_index, vectors, aux_data, \
443 : n_vectors_left) \
444 : vlib_get_next_frame_macro_with_aux ( \
445 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 0, \
446 : aux_data, /* maybe_no_aux */ 0)
447 :
448 : #define vlib_get_new_next_frame_with_aux(vm, node, next_index, vectors, \
449 : aux_data, n_vectors_left) \
450 : vlib_get_next_frame_macro_with_aux ( \
451 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 1, \
452 : aux_data, /* maybe_no_aux */ 0)
453 :
454 : /** \brief Get pointer to next frame vector data and next frame aux data by
455 : (@c vlib_node_runtime_t, @c next_index).
456 : Standard single/dual loop boilerplate element.
457 : @attention This is a MACRO, with SIDE EFFECTS.
458 : @attention This MACRO is safe in case the next node does not support aux_data.
459 : In that case aux_data is set to NULL.
460 :
461 : @param vm vlib_main_t pointer, varies by thread
462 : @param node current node vlib_node_runtime_t pointer
463 : @param next_index requested graph arc index
464 :
465 : @return @c vectors -- pointer to next available vector slot
466 : @return @c aux_data -- pointer to next available aux data slot
467 : @return @c n_vectors_left -- number of vector slots available
468 : */
469 : #define vlib_get_next_frame_with_aux_safe(vm, node, next_index, vectors, \
470 : aux_data, n_vectors_left) \
471 : vlib_get_next_frame_macro_with_aux ( \
472 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 0, \
473 : aux_data, /* maybe_no_aux */ 1)
474 :
475 : #define vlib_get_new_next_frame_with_aux_safe(vm, node, next_index, vectors, \
476 : aux_data, n_vectors_left) \
477 : vlib_get_next_frame_macro_with_aux ( \
478 : vm, node, next_index, vectors, n_vectors_left, /* alloc new frame */ 1, \
479 : aux_data, /* maybe_no_aux */ 1)
480 :
481 : /** \brief Release pointer to next frame vector data.
482 : Standard single/dual loop boilerplate element.
483 : @param vm vlib_main_t pointer, varies by thread
484 : @param r current node vlib_node_runtime_t pointer
485 : @param next_index graph arc index
486 : @param n_packets_left number of slots still available in vector
487 : */
488 : void
489 : vlib_put_next_frame (vlib_main_t * vm,
490 : vlib_node_runtime_t * r,
491 : u32 next_index, u32 n_packets_left);
492 :
493 : /* Combination get plus put. Returns vector argument just added. */
494 : #define vlib_set_next_frame(vm,node,next_index,v) \
495 : ({ \
496 : uword _n_left; \
497 : vlib_get_next_frame ((vm), (node), (next_index), (v), _n_left); \
498 : ASSERT (_n_left > 0); \
499 : vlib_put_next_frame ((vm), (node), (next_index), _n_left - 1); \
500 : (v); \
501 : })
502 :
503 : #define vlib_set_next_frame_with_aux_safe(vm, node, next_index, v, aux) \
504 : ({ \
505 : uword _n_left; \
506 : vlib_get_next_frame_with_aux_safe ((vm), (node), (next_index), (v), \
507 : (aux), _n_left); \
508 : ASSERT (_n_left > 0); \
509 : vlib_put_next_frame ((vm), (node), (next_index), _n_left - 1); \
510 : (v); \
511 : })
512 :
513 : always_inline void
514 514330 : vlib_set_next_frame_buffer (vlib_main_t * vm,
515 : vlib_node_runtime_t * node,
516 : u32 next_index, u32 buffer_index)
517 : {
518 : u32 *p;
519 514330 : p = vlib_set_next_frame (vm, node, next_index, p);
520 514330 : p[0] = buffer_index;
521 514330 : }
522 :
523 : always_inline void
524 : vlib_set_next_frame_buffer_with_aux_safe (vlib_main_t *vm,
525 : vlib_node_runtime_t *node,
526 : u32 next_index, u32 buffer_index,
527 : u32 aux)
528 : {
529 : u32 *p;
530 : u32 *a;
531 : p = vlib_set_next_frame_with_aux_safe (vm, node, next_index, p, a);
532 : p[0] = buffer_index;
533 : if (a)
534 : a[0] = aux;
535 : }
536 :
537 : vlib_frame_t *vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index);
538 : void vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index,
539 : vlib_frame_t * f);
540 :
541 : always_inline uword
542 1023995 : vlib_in_process_context (vlib_main_t * vm)
543 : {
544 1023995 : return vm->node_main.current_process_index != ~0;
545 : }
546 :
547 : always_inline vlib_process_t *
548 1023995 : vlib_get_current_process (vlib_main_t * vm)
549 : {
550 1023995 : vlib_node_main_t *nm = &vm->node_main;
551 1023995 : if (vlib_in_process_context (vm))
552 1023995 : return vec_elt (nm->processes, nm->current_process_index);
553 0 : return 0;
554 : }
555 :
556 : always_inline uword
557 5 : vlib_current_process (vlib_main_t * vm)
558 : {
559 5 : return vlib_get_current_process (vm)->node_runtime.node_index;
560 : }
561 :
562 : always_inline u32
563 0 : vlib_get_current_process_node_index (vlib_main_t * vm)
564 : {
565 0 : vlib_process_t *process = vlib_get_current_process (vm);
566 0 : return process->node_runtime.node_index;
567 : }
568 :
569 : /** Returns TRUE if a process suspend time is less than 10us
570 : @param dt - remaining poll time in seconds
571 : @returns 1 if dt < 10e-6, 0 otherwise
572 : */
573 : always_inline uword
574 1662173 : vlib_process_suspend_time_is_zero (f64 dt)
575 : {
576 1662173 : return dt < 10e-6;
577 : }
578 :
579 : /** Suspend a vlib cooperative multi-tasking thread for a period of time
580 : @param vm - vlib_main_t *
581 : @param dt - suspend interval in seconds
582 : @returns VLIB_PROCESS_RESUME_LONGJMP_RESUME, routinely ignored
583 : */
584 :
585 : always_inline uword
586 887245 : vlib_process_suspend (vlib_main_t * vm, f64 dt)
587 : {
588 : uword r;
589 887245 : vlib_node_main_t *nm = &vm->node_main;
590 887245 : vlib_process_t *p = vec_elt (nm->processes, nm->current_process_index);
591 :
592 887245 : if (vlib_process_suspend_time_is_zero (dt))
593 0 : return VLIB_PROCESS_RESUME_LONGJMP_RESUME;
594 :
595 887245 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK;
596 887245 : r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
597 1773808 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
598 : {
599 : /* expiration time in 10us ticks */
600 887245 : p->resume_clock_interval = dt * 1e5;
601 887245 : vlib_process_start_switch_stack (vm, 0);
602 887245 : clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
603 : }
604 : else
605 886563 : vlib_process_finish_switch_stack (vm);
606 :
607 886563 : return r;
608 : }
609 :
610 : always_inline void
611 0 : vlib_process_free_event_type (vlib_process_t * p, uword t,
612 : uword is_one_time_event)
613 : {
614 0 : ASSERT (!pool_is_free_index (p->event_type_pool, t));
615 0 : pool_put_index (p->event_type_pool, t);
616 0 : if (is_one_time_event)
617 0 : p->one_time_event_type_bitmap =
618 0 : clib_bitmap_andnoti (p->one_time_event_type_bitmap, t);
619 0 : }
620 :
621 : always_inline void
622 469230 : vlib_process_maybe_free_event_type (vlib_process_t * p, uword t)
623 : {
624 469230 : ASSERT (!pool_is_free_index (p->event_type_pool, t));
625 469230 : if (clib_bitmap_get (p->one_time_event_type_bitmap, t))
626 0 : vlib_process_free_event_type (p, t, /* is_one_time_event */ 1);
627 469230 : }
628 :
629 : always_inline void *
630 6988 : vlib_process_get_event_data (vlib_main_t * vm,
631 : uword * return_event_type_opaque)
632 : {
633 6988 : vlib_node_main_t *nm = &vm->node_main;
634 : vlib_process_t *p;
635 : vlib_process_event_type_t *et;
636 : uword t;
637 : void *event_data_vector;
638 :
639 6988 : p = vec_elt (nm->processes, nm->current_process_index);
640 :
641 : /* Find first type with events ready.
642 : Return invalid type when there's nothing there. */
643 6988 : t = clib_bitmap_first_set (p->non_empty_event_type_bitmap);
644 6988 : if (t == ~0)
645 6922 : return 0;
646 :
647 66 : p->non_empty_event_type_bitmap =
648 66 : clib_bitmap_andnoti (p->non_empty_event_type_bitmap, t);
649 :
650 66 : ASSERT (_vec_len (p->pending_event_data_by_type_index[t]) > 0);
651 66 : event_data_vector = p->pending_event_data_by_type_index[t];
652 66 : p->pending_event_data_by_type_index[t] = 0;
653 :
654 66 : et = pool_elt_at_index (p->event_type_pool, t);
655 :
656 : /* Return user's opaque value and possibly index. */
657 66 : *return_event_type_opaque = et->opaque;
658 :
659 66 : vlib_process_maybe_free_event_type (p, t);
660 :
661 66 : return event_data_vector;
662 : }
663 :
664 : /* Return event data vector for later reuse. We reuse event data to avoid
665 : repeatedly allocating event vectors in cases where we care about speed. */
666 : always_inline void
667 28 : vlib_process_put_event_data (vlib_main_t * vm, void *event_data)
668 : {
669 28 : vlib_node_main_t *nm = &vm->node_main;
670 28 : vec_add1 (nm->recycled_event_data_vectors, event_data);
671 28 : }
672 :
673 : /** Return the first event type which has occurred and a vector of per-event
674 : data of that type, or a timeout indication
675 :
676 : @param vm - vlib_main_t pointer
677 : @param data_vector - pointer to a (uword *) vector to receive event data
678 : @returns either an event type and a vector of per-event instance data,
679 : or ~0 to indicate a timeout.
680 : */
681 :
682 : always_inline uword
683 814861 : vlib_process_get_events (vlib_main_t * vm, uword ** data_vector)
684 : {
685 814861 : vlib_node_main_t *nm = &vm->node_main;
686 : vlib_process_t *p;
687 : vlib_process_event_type_t *et;
688 : uword r, t, l;
689 :
690 814861 : p = vec_elt (nm->processes, nm->current_process_index);
691 :
692 : /* Find first type with events ready.
693 : Return invalid type when there's nothing there. */
694 814861 : t = clib_bitmap_first_set (p->non_empty_event_type_bitmap);
695 814861 : if (t == ~0)
696 345697 : return t;
697 :
698 469164 : p->non_empty_event_type_bitmap =
699 469164 : clib_bitmap_andnoti (p->non_empty_event_type_bitmap, t);
700 :
701 469164 : l = _vec_len (p->pending_event_data_by_type_index[t]);
702 469164 : if (data_vector)
703 469140 : vec_add (*data_vector, p->pending_event_data_by_type_index[t], l);
704 469164 : vec_set_len (p->pending_event_data_by_type_index[t], 0);
705 :
706 469164 : et = pool_elt_at_index (p->event_type_pool, t);
707 :
708 : /* Return user's opaque value. */
709 469164 : r = et->opaque;
710 :
711 469164 : vlib_process_maybe_free_event_type (p, t);
712 :
713 469164 : return r;
714 : }
715 :
716 : always_inline uword
717 : vlib_process_get_events_helper (vlib_process_t * p, uword t,
718 : uword ** data_vector)
719 : {
720 : uword l;
721 :
722 : p->non_empty_event_type_bitmap =
723 : clib_bitmap_andnoti (p->non_empty_event_type_bitmap, t);
724 :
725 : l = _vec_len (p->pending_event_data_by_type_index[t]);
726 : if (data_vector)
727 : vec_add (*data_vector, p->pending_event_data_by_type_index[t], l);
728 : vec_set_len (p->pending_event_data_by_type_index[t], 0);
729 :
730 : vlib_process_maybe_free_event_type (p, t);
731 :
732 : return l;
733 : }
734 :
735 : /* As above but query as specified type of event. Returns number of
736 : events found. */
737 : always_inline uword
738 : vlib_process_get_events_with_type (vlib_main_t * vm, uword ** data_vector,
739 : uword with_type_opaque)
740 : {
741 : vlib_node_main_t *nm = &vm->node_main;
742 : vlib_process_t *p;
743 : uword t, *h;
744 :
745 : p = vec_elt (nm->processes, nm->current_process_index);
746 : h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
747 : if (!h)
748 : /* This can happen when an event has not yet been
749 : signaled with given opaque type. */
750 : return 0;
751 :
752 : t = h[0];
753 : if (!clib_bitmap_get (p->non_empty_event_type_bitmap, t))
754 : return 0;
755 :
756 : return vlib_process_get_events_helper (p, t, data_vector);
757 : }
758 :
759 : always_inline uword *
760 33504 : vlib_process_wait_for_event (vlib_main_t * vm)
761 : {
762 33504 : vlib_node_main_t *nm = &vm->node_main;
763 : vlib_process_t *p;
764 : uword r;
765 :
766 33504 : p = vec_elt (nm->processes, nm->current_process_index);
767 33504 : if (clib_bitmap_is_zero (p->non_empty_event_type_bitmap))
768 : {
769 33128 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
770 : r =
771 33128 : clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
772 55121 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
773 : {
774 33128 : vlib_process_start_switch_stack (vm, 0);
775 33128 : clib_longjmp (&p->return_longjmp,
776 : VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
777 : }
778 : else
779 21993 : vlib_process_finish_switch_stack (vm);
780 : }
781 :
782 22369 : return p->non_empty_event_type_bitmap;
783 : }
784 :
785 : always_inline uword
786 : vlib_process_wait_for_one_time_event (vlib_main_t * vm,
787 : uword ** data_vector,
788 : uword with_type_index)
789 : {
790 : vlib_node_main_t *nm = &vm->node_main;
791 : vlib_process_t *p;
792 : uword r;
793 :
794 : p = vec_elt (nm->processes, nm->current_process_index);
795 : ASSERT (!pool_is_free_index (p->event_type_pool, with_type_index));
796 : while (!clib_bitmap_get (p->non_empty_event_type_bitmap, with_type_index))
797 : {
798 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
799 : r =
800 : clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
801 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
802 : {
803 : vlib_process_start_switch_stack (vm, 0);
804 : clib_longjmp (&p->return_longjmp,
805 : VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
806 : }
807 : else
808 : vlib_process_finish_switch_stack (vm);
809 : }
810 :
811 : return vlib_process_get_events_helper (p, with_type_index, data_vector);
812 : }
813 :
814 : always_inline uword
815 : vlib_process_wait_for_event_with_type (vlib_main_t * vm,
816 : uword ** data_vector,
817 : uword with_type_opaque)
818 : {
819 : vlib_node_main_t *nm = &vm->node_main;
820 : vlib_process_t *p;
821 : uword r, *h;
822 :
823 : p = vec_elt (nm->processes, nm->current_process_index);
824 : h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
825 : while (!h || !clib_bitmap_get (p->non_empty_event_type_bitmap, h[0]))
826 : {
827 : p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
828 : r =
829 : clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
830 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
831 : {
832 : vlib_process_start_switch_stack (vm, 0);
833 : clib_longjmp (&p->return_longjmp,
834 : VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
835 : }
836 : else
837 : vlib_process_finish_switch_stack (vm);
838 :
839 : /* See if unknown event type has been signaled now. */
840 : if (!h)
841 : h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
842 : }
843 :
844 : return vlib_process_get_events_helper (p, h[0], data_vector);
845 : }
846 :
847 : /** Suspend a cooperative multi-tasking thread
848 : Waits for an event, or for the indicated number of seconds to elapse
849 : @param vm - vlib_main_t pointer
850 : @param dt - timeout, in seconds.
851 : @returns the remaining time interval
852 : */
853 :
854 : always_inline f64
855 774894 : vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt)
856 : {
857 774894 : vlib_node_main_t *nm = &vm->node_main;
858 : vlib_process_t *p;
859 : f64 wakeup_time;
860 : uword r;
861 :
862 774894 : p = vec_elt (nm->processes, nm->current_process_index);
863 :
864 774894 : if (vlib_process_suspend_time_is_zero (dt)
865 771248 : || !clib_bitmap_is_zero (p->non_empty_event_type_bitmap))
866 4208 : return dt;
867 :
868 770686 : wakeup_time = vlib_time_now (vm) + dt;
869 :
870 : /* Suspend waiting for both clock and event to occur. */
871 770686 : p->flags |= (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
872 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK);
873 :
874 770686 : r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
875 1528948 : if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
876 : {
877 770686 : p->resume_clock_interval = dt * 1e5;
878 770686 : vlib_process_start_switch_stack (vm, 0);
879 770686 : clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
880 : }
881 : else
882 758262 : vlib_process_finish_switch_stack (vm);
883 :
884 : /* Return amount of time still left to sleep.
885 : If <= 0 then we've been waken up by the clock (and not an event). */
886 758262 : return wakeup_time - vlib_time_now (vm);
887 : }
888 :
889 : always_inline vlib_process_event_type_t *
890 3911 : vlib_process_new_event_type (vlib_process_t * p, uword with_type_opaque)
891 : {
892 : vlib_process_event_type_t *et;
893 3911 : pool_get (p->event_type_pool, et);
894 3911 : et->opaque = with_type_opaque;
895 3911 : return et;
896 : }
897 :
898 : always_inline uword
899 : vlib_process_create_one_time_event (vlib_main_t * vm, uword node_index,
900 : uword with_type_opaque)
901 : {
902 : vlib_node_main_t *nm = &vm->node_main;
903 : vlib_node_t *n = vlib_get_node (vm, node_index);
904 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
905 : vlib_process_event_type_t *et;
906 : uword t;
907 :
908 : et = vlib_process_new_event_type (p, with_type_opaque);
909 : t = et - p->event_type_pool;
910 : p->one_time_event_type_bitmap =
911 : clib_bitmap_ori (p->one_time_event_type_bitmap, t);
912 : return t;
913 : }
914 :
915 : always_inline void
916 : vlib_process_delete_one_time_event (vlib_main_t * vm, uword node_index,
917 : uword t)
918 : {
919 : vlib_node_main_t *nm = &vm->node_main;
920 : vlib_node_t *n = vlib_get_node (vm, node_index);
921 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
922 :
923 : ASSERT (clib_bitmap_get (p->one_time_event_type_bitmap, t));
924 : vlib_process_free_event_type (p, t, /* is_one_time_event */ 1);
925 : }
926 :
927 : always_inline void *
928 482691 : vlib_process_signal_event_helper (vlib_node_main_t * nm,
929 : vlib_node_t * n,
930 : vlib_process_t * p,
931 : uword t,
932 : uword n_data_elts, uword n_data_elt_bytes)
933 : {
934 : uword p_flags, add_to_pending, delete_from_wheel;
935 : u8 *data_to_be_written_by_caller;
936 482691 : vec_attr_t va = { .elt_sz = n_data_elt_bytes };
937 :
938 482691 : ASSERT (n->type == VLIB_NODE_TYPE_PROCESS);
939 :
940 482691 : ASSERT (!pool_is_free_index (p->event_type_pool, t));
941 :
942 482691 : vec_validate (p->pending_event_data_by_type_index, t);
943 :
944 : /* Resize data vector and return caller's data to be written. */
945 : {
946 482691 : u8 *data_vec = p->pending_event_data_by_type_index[t];
947 : uword l;
948 :
949 482691 : if (!data_vec && vec_len (nm->recycled_event_data_vectors))
950 : {
951 25 : data_vec = vec_pop (nm->recycled_event_data_vectors);
952 25 : vec_reset_length (data_vec);
953 : }
954 :
955 482691 : l = vec_len (data_vec);
956 :
957 482691 : data_vec = _vec_realloc_internal (data_vec, l + n_data_elts, &va);
958 :
959 482691 : p->pending_event_data_by_type_index[t] = data_vec;
960 482691 : data_to_be_written_by_caller = data_vec + l * n_data_elt_bytes;
961 : }
962 :
963 482691 : p->non_empty_event_type_bitmap =
964 482691 : clib_bitmap_ori (p->non_empty_event_type_bitmap, t);
965 :
966 482691 : p_flags = p->flags;
967 :
968 : /* Event was already signalled? */
969 482691 : add_to_pending = (p_flags & VLIB_PROCESS_RESUME_PENDING) == 0;
970 :
971 : /* Process will resume when suspend time elapses? */
972 482691 : delete_from_wheel = 0;
973 482691 : if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
974 : {
975 : /* Waiting for both event and clock? */
976 439936 : if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)
977 : {
978 438424 : if (!TW (tw_timer_handle_is_free)
979 438424 : ((TWT (tw_timer_wheel) *) nm->timing_wheel,
980 : p->stop_timer_handle))
981 426349 : delete_from_wheel = 1;
982 : else
983 : /* timer just popped so process should already be on the list */
984 12075 : add_to_pending = 0;
985 : }
986 : else
987 : /* Waiting only for clock. Event will be queue and may be
988 : handled when timer expires. */
989 1512 : add_to_pending = 0;
990 : }
991 :
992 : /* Never add current process to pending vector since current process is
993 : already running. */
994 482691 : add_to_pending &= nm->current_process_index != n->runtime_index;
995 :
996 482691 : if (add_to_pending)
997 : {
998 448357 : u32 x = vlib_timing_wheel_data_set_suspended_process (n->runtime_index);
999 448357 : p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING;
1000 448357 : vec_add1 (nm->data_from_advancing_timing_wheel, x);
1001 448357 : if (delete_from_wheel)
1002 : {
1003 426349 : TW (tw_timer_stop)
1004 426349 : ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
1005 426349 : p->stop_timer_handle = ~0;
1006 : }
1007 : }
1008 :
1009 482691 : return data_to_be_written_by_caller;
1010 : }
1011 :
1012 : always_inline void *
1013 482691 : vlib_process_signal_event_data (vlib_main_t * vm,
1014 : uword node_index,
1015 : uword type_opaque,
1016 : uword n_data_elts, uword n_data_elt_bytes)
1017 : {
1018 482691 : vlib_node_main_t *nm = &vm->node_main;
1019 482691 : vlib_node_t *n = vlib_get_node (vm, node_index);
1020 482691 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
1021 : uword *h, t;
1022 :
1023 : /* Must be in main thread */
1024 482691 : ASSERT (vlib_get_thread_index () == 0);
1025 :
1026 482691 : h = hash_get (p->event_type_index_by_type_opaque, type_opaque);
1027 482691 : if (!h)
1028 : {
1029 : vlib_process_event_type_t *et =
1030 3911 : vlib_process_new_event_type (p, type_opaque);
1031 3911 : t = et - p->event_type_pool;
1032 3911 : hash_set (p->event_type_index_by_type_opaque, type_opaque, t);
1033 : }
1034 : else
1035 478780 : t = h[0];
1036 :
1037 482691 : return vlib_process_signal_event_helper (nm, n, p, t, n_data_elts,
1038 : n_data_elt_bytes);
1039 : }
1040 :
1041 : always_inline void *
1042 : vlib_process_signal_event_at_time (vlib_main_t * vm,
1043 : f64 dt,
1044 : uword node_index,
1045 : uword type_opaque,
1046 : uword n_data_elts, uword n_data_elt_bytes)
1047 : {
1048 : vlib_node_main_t *nm = &vm->node_main;
1049 : vlib_node_t *n = vlib_get_node (vm, node_index);
1050 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
1051 : uword *h, t;
1052 :
1053 : h = hash_get (p->event_type_index_by_type_opaque, type_opaque);
1054 : if (!h)
1055 : {
1056 : vlib_process_event_type_t *et =
1057 : vlib_process_new_event_type (p, type_opaque);
1058 : t = et - p->event_type_pool;
1059 : hash_set (p->event_type_index_by_type_opaque, type_opaque, t);
1060 : }
1061 : else
1062 : t = h[0];
1063 :
1064 : if (vlib_process_suspend_time_is_zero (dt))
1065 : return vlib_process_signal_event_helper (nm, n, p, t, n_data_elts,
1066 : n_data_elt_bytes);
1067 : else
1068 : {
1069 : vlib_signal_timed_event_data_t *te;
1070 :
1071 : pool_get_aligned (nm->signal_timed_event_data_pool, te, sizeof (te[0]));
1072 :
1073 : te->n_data_elts = n_data_elts;
1074 : te->n_data_elt_bytes = n_data_elt_bytes;
1075 : te->n_data_bytes = n_data_elts * n_data_elt_bytes;
1076 :
1077 : /* Assert that structure fields are big enough. */
1078 : ASSERT (te->n_data_elts == n_data_elts);
1079 : ASSERT (te->n_data_elt_bytes == n_data_elt_bytes);
1080 : ASSERT (te->n_data_bytes == n_data_elts * n_data_elt_bytes);
1081 :
1082 : te->process_node_index = n->runtime_index;
1083 : te->event_type_index = t;
1084 :
1085 : p->stop_timer_handle =
1086 : TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1087 : vlib_timing_wheel_data_set_timed_event
1088 : (te - nm->signal_timed_event_data_pool),
1089 : 0 /* timer_id */ ,
1090 : (vlib_time_now (vm) + dt) * 1e5);
1091 :
1092 : /* Inline data big enough to hold event? */
1093 : if (te->n_data_bytes < sizeof (te->inline_event_data))
1094 : return te->inline_event_data;
1095 : else
1096 : {
1097 : te->event_data_as_vector = 0;
1098 : vec_resize (te->event_data_as_vector, te->n_data_bytes);
1099 : return te->event_data_as_vector;
1100 : }
1101 : }
1102 : }
1103 :
1104 : always_inline void *
1105 : vlib_process_signal_one_time_event_data (vlib_main_t * vm,
1106 : uword node_index,
1107 : uword type_index,
1108 : uword n_data_elts,
1109 : uword n_data_elt_bytes)
1110 : {
1111 : vlib_node_main_t *nm = &vm->node_main;
1112 : vlib_node_t *n = vlib_get_node (vm, node_index);
1113 : vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
1114 : return vlib_process_signal_event_helper (nm, n, p, type_index, n_data_elts,
1115 : n_data_elt_bytes);
1116 : }
1117 :
1118 : always_inline void
1119 482617 : vlib_process_signal_event (vlib_main_t * vm,
1120 : uword node_index, uword type_opaque, uword data)
1121 : {
1122 482617 : uword *d = vlib_process_signal_event_data (vm, node_index, type_opaque,
1123 : 1 /* elts */ , sizeof (uword));
1124 482617 : d[0] = data;
1125 482617 : }
1126 :
1127 : always_inline void
1128 0 : vlib_process_signal_event_pointer (vlib_main_t * vm,
1129 : uword node_index,
1130 : uword type_opaque, void *data)
1131 : {
1132 0 : void **d = vlib_process_signal_event_data (vm, node_index, type_opaque,
1133 : 1 /* elts */ , sizeof (data));
1134 0 : d[0] = data;
1135 0 : }
1136 :
1137 : /**
1138 : * Signal event to process from any thread.
1139 : *
1140 : * When in doubt, use this.
1141 : */
1142 : always_inline void
1143 204 : vlib_process_signal_event_mt (vlib_main_t * vm,
1144 : uword node_index, uword type_opaque, uword data)
1145 : {
1146 204 : if (vlib_get_thread_index () != 0)
1147 : {
1148 0 : vlib_process_signal_event_mt_args_t args = {
1149 : .node_index = node_index,
1150 : .type_opaque = type_opaque,
1151 : .data = data,
1152 : };
1153 0 : vlib_rpc_call_main_thread (vlib_process_signal_event_mt_helper,
1154 : (u8 *) & args, sizeof (args));
1155 : }
1156 : else
1157 204 : vlib_process_signal_event (vm, node_index, type_opaque, data);
1158 204 : }
1159 :
1160 : always_inline void
1161 : vlib_process_signal_one_time_event (vlib_main_t * vm,
1162 : uword node_index,
1163 : uword type_index, uword data)
1164 : {
1165 : uword *d =
1166 : vlib_process_signal_one_time_event_data (vm, node_index, type_index,
1167 : 1 /* elts */ , sizeof (uword));
1168 : d[0] = data;
1169 : }
1170 :
1171 : always_inline void
1172 : vlib_signal_one_time_waiting_process (vlib_main_t * vm,
1173 : vlib_one_time_waiting_process_t * p)
1174 : {
1175 : vlib_process_signal_one_time_event (vm, p->node_index, p->one_time_event,
1176 : /* data */ ~0);
1177 : clib_memset (p, ~0, sizeof (p[0]));
1178 : }
1179 :
1180 : always_inline void
1181 : vlib_signal_one_time_waiting_process_vector (vlib_main_t * vm,
1182 : vlib_one_time_waiting_process_t
1183 : ** wps)
1184 : {
1185 : vlib_one_time_waiting_process_t *wp;
1186 : vec_foreach (wp, *wps) vlib_signal_one_time_waiting_process (vm, wp);
1187 : vec_free (*wps);
1188 : }
1189 :
1190 : always_inline void
1191 : vlib_current_process_wait_for_one_time_event (vlib_main_t * vm,
1192 : vlib_one_time_waiting_process_t
1193 : * p)
1194 : {
1195 : p->node_index = vlib_current_process (vm);
1196 : p->one_time_event = vlib_process_create_one_time_event (vm, p->node_index, /* type opaque */
1197 : ~0);
1198 : vlib_process_wait_for_one_time_event (vm,
1199 : /* don't care about data */ 0,
1200 : p->one_time_event);
1201 : }
1202 :
1203 : always_inline void
1204 : vlib_current_process_wait_for_one_time_event_vector (vlib_main_t * vm,
1205 : vlib_one_time_waiting_process_t
1206 : ** wps)
1207 : {
1208 : vlib_one_time_waiting_process_t *wp;
1209 : vec_add2 (*wps, wp, 1);
1210 : vlib_current_process_wait_for_one_time_event (vm, wp);
1211 : }
1212 :
1213 : always_inline u32
1214 668760000 : vlib_node_runtime_update_main_loop_vector_stats (vlib_main_t * vm,
1215 : vlib_node_runtime_t * node,
1216 : uword n_vectors)
1217 : {
1218 : u32 i, d, vi0, vi1;
1219 : u32 i0, i1;
1220 :
1221 668760000 : ASSERT (is_pow2 (ARRAY_LEN (node->main_loop_vector_stats)));
1222 668781000 : i = ((vm->main_loop_count >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE)
1223 : & (ARRAY_LEN (node->main_loop_vector_stats) - 1));
1224 668781000 : i0 = i ^ 0;
1225 668781000 : i1 = i ^ 1;
1226 668781000 : d = ((vm->main_loop_count >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE)
1227 : -
1228 668781000 : (node->main_loop_count_last_dispatch >>
1229 : VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE));
1230 668781000 : vi0 = node->main_loop_vector_stats[i0];
1231 668781000 : vi1 = node->main_loop_vector_stats[i1];
1232 668781000 : vi0 = d == 0 ? vi0 : 0;
1233 668781000 : vi1 = d <= 1 ? vi1 : 0;
1234 668781000 : vi0 += n_vectors;
1235 668781000 : node->main_loop_vector_stats[i0] = vi0;
1236 668781000 : node->main_loop_vector_stats[i1] = vi1;
1237 668781000 : node->main_loop_count_last_dispatch = vm->main_loop_count;
1238 : /* Return previous counter. */
1239 668781000 : return node->main_loop_vector_stats[i1];
1240 : }
1241 :
1242 : always_inline f64
1243 : vlib_node_vectors_per_main_loop_as_float (vlib_main_t * vm, u32 node_index)
1244 : {
1245 : vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index);
1246 : u32 v;
1247 :
1248 : v = vlib_node_runtime_update_main_loop_vector_stats (vm, rt, /* n_vectors */
1249 : 0);
1250 : return (f64) v / (1 << VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE);
1251 : }
1252 :
1253 : always_inline u32
1254 : vlib_node_vectors_per_main_loop_as_integer (vlib_main_t * vm, u32 node_index)
1255 : {
1256 : vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index);
1257 : u32 v;
1258 :
1259 : v = vlib_node_runtime_update_main_loop_vector_stats (vm, rt, /* n_vectors */
1260 : 0);
1261 : return v >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE;
1262 : }
1263 :
1264 : void vlib_frame_free (vlib_main_t *vm, vlib_frame_t *f);
1265 :
1266 : /* Return the edge index if present, ~0 otherwise */
1267 : uword vlib_node_get_next (vlib_main_t * vm, uword node, uword next_node);
1268 :
1269 : /* Add next node to given node in given slot. */
1270 : uword
1271 : vlib_node_add_next_with_slot (vlib_main_t * vm,
1272 : uword node, uword next_node, uword slot);
1273 :
1274 : /* As above but adds to end of node's next vector. */
1275 : always_inline uword
1276 413019 : vlib_node_add_next (vlib_main_t * vm, uword node, uword next_node)
1277 : {
1278 413019 : return vlib_node_add_next_with_slot (vm, node, next_node, ~0);
1279 : }
1280 :
1281 : /* Add next node to given node in given slot. */
1282 : uword
1283 : vlib_node_add_named_next_with_slot (vlib_main_t * vm,
1284 : uword node, char *next_name, uword slot);
1285 :
1286 : /* As above but adds to end of node's next vector. */
1287 : always_inline uword
1288 401362 : vlib_node_add_named_next (vlib_main_t * vm, uword node, char *name)
1289 : {
1290 401362 : return vlib_node_add_named_next_with_slot (vm, node, name, ~0);
1291 : }
1292 :
1293 : /**
1294 : * Get list of nodes
1295 : */
1296 : void
1297 : vlib_node_get_nodes (vlib_main_t * vm, u32 max_threads, int include_stats,
1298 : int barrier_sync, vlib_node_t **** node_dupsp,
1299 : vlib_main_t *** stat_vmsp);
1300 :
1301 : /* Query node given name. */
1302 : vlib_node_t *vlib_get_node_by_name (vlib_main_t * vm, u8 * name);
1303 :
1304 : /* Rename a node. */
1305 : void vlib_node_rename (vlib_main_t * vm, u32 node_index, char *fmt, ...);
1306 :
1307 : /* Register new packet processing node. Nodes can be registered
1308 : dynamically via this call or statically via the VLIB_REGISTER_NODE
1309 : macro. */
1310 : u32 vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r,
1311 : char *fmt, ...);
1312 :
1313 : /* Register all node function variants */
1314 : void vlib_register_all_node_march_variants (vlib_main_t *vm);
1315 :
1316 : /* Register all static nodes registered via VLIB_REGISTER_NODE. */
1317 : void vlib_register_all_static_nodes (vlib_main_t * vm);
1318 :
1319 : /* Start a process. */
1320 : void vlib_start_process (vlib_main_t * vm, uword process_index);
1321 :
1322 : /* Sync up runtime and main node stats. */
1323 : void vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n);
1324 : void vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r,
1325 : uword n_calls, uword n_vectors,
1326 : uword n_clocks);
1327 : void vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r,
1328 : uword n_calls, uword n_vectors,
1329 : uword n_clocks);
1330 :
1331 : /* Node graph initialization function. */
1332 : clib_error_t *vlib_node_main_init (vlib_main_t * vm);
1333 :
1334 : format_function_t format_vlib_node_graph;
1335 : format_function_t format_vlib_node_name;
1336 : format_function_t format_vlib_next_node_name;
1337 : format_function_t format_vlib_node_and_next;
1338 : format_function_t format_vlib_cpu_time;
1339 : format_function_t format_vlib_time;
1340 : /* Parse node name -> node index. */
1341 : unformat_function_t unformat_vlib_node;
1342 :
1343 : always_inline void
1344 94277939 : vlib_node_increment_counter (vlib_main_t * vm, u32 node_index,
1345 : u32 counter_index, u64 increment)
1346 : {
1347 94277939 : vlib_node_t *n = vlib_get_node (vm, node_index);
1348 94265636 : vlib_error_main_t *em = &vm->error_main;
1349 94265636 : u32 node_counter_base_index = n->error_heap_index;
1350 94265636 : em->counters[node_counter_base_index + counter_index] += increment;
1351 94265636 : }
1352 :
1353 : /** @brief Create a vlib process
1354 : * @param vm &vlib_global_main
1355 : * @param f the process node function
1356 : * @param log2_n_stack_bytes size of the process stack, defaults to 16K
1357 : * @return newly-create node index
1358 : * @warning call only on the main thread. Barrier sync required
1359 : */
1360 : u32 vlib_process_create (vlib_main_t * vm, char *name,
1361 : vlib_node_function_t * f, u32 log2_n_stack_bytes);
1362 :
1363 : always_inline int
1364 2 : vlib_node_set_dispatch_wrapper (vlib_main_t *vm, vlib_node_function_t *fn)
1365 : {
1366 2 : if (fn && vm->dispatch_wrapper_fn)
1367 0 : return 1;
1368 2 : vm->dispatch_wrapper_fn = fn;
1369 2 : return 0;
1370 : }
1371 :
1372 : int vlib_node_set_march_variant (vlib_main_t *vm, u32 node_index,
1373 : clib_march_variant_type_t march_variant);
1374 :
1375 : vlib_node_function_t *
1376 : vlib_node_get_preferred_node_fn_variant (vlib_main_t *vm,
1377 : vlib_node_fn_registration_t *regs);
1378 :
1379 : /*
1380 : * vlib_frame_bitmap functions
1381 : */
1382 :
1383 : #define VLIB_FRAME_BITMAP_N_UWORDS \
1384 : (((VLIB_FRAME_SIZE + uword_bits - 1) & ~(uword_bits - 1)) / uword_bits)
1385 :
1386 : typedef uword vlib_frame_bitmap_t[VLIB_FRAME_BITMAP_N_UWORDS];
1387 :
1388 : static_always_inline void
1389 : vlib_frame_bitmap_init (uword *bmp, u32 n_first_bits_set)
1390 : {
1391 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1392 : while (n_first_bits_set >= (sizeof (uword) * 8) && n_left)
1393 : {
1394 : bmp++[0] = ~0;
1395 : n_first_bits_set -= sizeof (uword) * 8;
1396 : n_left--;
1397 : }
1398 :
1399 : if (n_first_bits_set && n_left)
1400 : {
1401 : bmp++[0] = pow2_mask (n_first_bits_set);
1402 : n_left--;
1403 : }
1404 :
1405 : while (n_left--)
1406 : bmp++[0] = 0;
1407 : }
1408 :
1409 : static_always_inline void
1410 : vlib_frame_bitmap_set_bit_at_index (uword *bmp, uword bit_index)
1411 : {
1412 : uword_bitmap_set_bits_at_index (bmp, bit_index, 1);
1413 : }
1414 :
1415 : static_always_inline void
1416 : _vlib_frame_bitmap_clear_bit_at_index (uword *bmp, uword bit_index)
1417 : {
1418 : uword_bitmap_clear_bits_at_index (bmp, bit_index, 1);
1419 : }
1420 :
1421 : static_always_inline void
1422 : vlib_frame_bitmap_set_bits_at_index (uword *bmp, uword bit_index, uword n_bits)
1423 : {
1424 : uword_bitmap_set_bits_at_index (bmp, bit_index, n_bits);
1425 : }
1426 :
1427 : static_always_inline void
1428 : vlib_frame_bitmap_clear_bits_at_index (uword *bmp, uword bit_index,
1429 : uword n_bits)
1430 : {
1431 : uword_bitmap_clear_bits_at_index (bmp, bit_index, n_bits);
1432 : }
1433 :
1434 : static_always_inline void
1435 : vlib_frame_bitmap_clear (uword *bmp)
1436 : {
1437 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1438 : while (n_left--)
1439 : bmp++[0] = 0;
1440 : }
1441 :
1442 : static_always_inline void
1443 : vlib_frame_bitmap_xor (uword *bmp, uword *bmp2)
1444 : {
1445 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1446 : while (n_left--)
1447 : bmp++[0] ^= bmp2++[0];
1448 : }
1449 :
1450 : static_always_inline void
1451 7345020 : vlib_frame_bitmap_or (uword *bmp, uword *bmp2)
1452 : {
1453 7345020 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1454 36725100 : while (n_left--)
1455 29380100 : bmp++[0] |= bmp2++[0];
1456 7345020 : }
1457 :
1458 : static_always_inline void
1459 : vlib_frame_bitmap_and (uword *bmp, uword *bmp2)
1460 : {
1461 : u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
1462 : while (n_left--)
1463 : bmp++[0] &= bmp2++[0];
1464 : }
1465 :
1466 : static_always_inline uword
1467 : vlib_frame_bitmap_count_set_bits (uword *bmp)
1468 : {
1469 : return uword_bitmap_count_set_bits (bmp, VLIB_FRAME_BITMAP_N_UWORDS);
1470 : }
1471 :
1472 : static_always_inline uword
1473 : vlib_frame_bitmap_is_bit_set (uword *bmp, uword bit_index)
1474 : {
1475 : return uword_bitmap_is_bit_set (bmp, bit_index);
1476 : }
1477 :
1478 : static_always_inline uword
1479 : vlib_frame_bitmap_find_first_set (uword *bmp)
1480 : {
1481 : uword rv = uword_bitmap_find_first_set (bmp);
1482 : ASSERT (rv < VLIB_FRAME_BITMAP_N_UWORDS * uword_bits);
1483 : return rv;
1484 : }
1485 :
1486 : #define foreach_vlib_frame_bitmap_set_bit_index(i, v) \
1487 : for (uword _off = 0; _off < ARRAY_LEN (v); _off++) \
1488 : for (uword _tmp = \
1489 : (v[_off]) + 0 * (uword) (i = _off * uword_bits + \
1490 : get_lowest_set_bit_index (v[_off])); \
1491 : _tmp; i = _off * uword_bits + get_lowest_set_bit_index ( \
1492 : _tmp = clear_lowest_set_bit (_tmp)))
1493 :
1494 : #endif /* included_vlib_node_funcs_h */
1495 :
1496 : /*
1497 : * fd.io coding-style-patch-verification: ON
1498 : *
1499 : * Local Variables:
1500 : * eval: (c-set-style "gnu")
1501 : * End:
1502 : */
|
