Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * main.h: VLIB main data structure
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #ifndef included_vlib_main_h
41 : #define included_vlib_main_h
42 :
43 : #include <vppinfra/clib.h>
44 : #include <vppinfra/callback_data.h>
45 : #include <vppinfra/elog.h>
46 : #include <vppinfra/format.h>
47 : #include <vppinfra/longjmp.h>
48 : #include <vppinfra/pool.h>
49 : #include <vppinfra/random_buffer.h>
50 : #include <vppinfra/time.h>
51 :
52 : #include <pthread.h>
53 :
54 :
55 : /* By default turn off node/error event logging.
56 : Override with -DVLIB_ELOG_MAIN_LOOP */
57 : #ifndef VLIB_ELOG_MAIN_LOOP
58 : #define VLIB_ELOG_MAIN_LOOP 0
59 : #endif
60 :
61 : typedef struct
62 : {
63 : u8 trace_filter_enable;
64 : u32 classify_table_index;
65 : } vlib_trace_filter_t;
66 :
67 : typedef enum
68 : {
69 : VLIB_NODE_RUNTIME_PERF_BEFORE,
70 : VLIB_NODE_RUNTIME_PERF_AFTER,
71 : VLIB_NODE_RUNTIME_PERF_RESET,
72 : } vlib_node_runtime_perf_call_type_t;
73 :
74 : typedef struct
75 : {
76 : struct vlib_main_t *vm;
77 : vlib_node_runtime_t *node;
78 : vlib_frame_t *frame;
79 : uword packets;
80 : u64 cpu_time_now;
81 : vlib_node_runtime_perf_call_type_t call_type;
82 : } vlib_node_runtime_perf_callback_args_t;
83 :
84 : struct vlib_node_runtime_perf_callback_data_t;
85 :
86 : typedef void (*vlib_node_runtime_perf_callback_fp_t)
87 : (struct vlib_node_runtime_perf_callback_data_t * data,
88 : vlib_node_runtime_perf_callback_args_t * args);
89 :
90 : typedef struct vlib_node_runtime_perf_callback_data_t
91 : {
92 : vlib_node_runtime_perf_callback_fp_t fp;
93 : union
94 : {
95 : void *v;
96 : u64 u;
97 : } u[3];
98 : } vlib_node_runtime_perf_callback_data_t;
99 :
100 : clib_callback_data_typedef (vlib_node_runtime_perf_callback_set_t,
101 : vlib_node_runtime_perf_callback_data_t);
102 :
103 : typedef struct vlib_main_t
104 : {
105 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
106 : /* Instruction level timing state. */
107 : clib_time_t clib_time;
108 : /* Offset from main thread time */
109 : f64 time_offset;
110 : f64 time_last_barrier_release;
111 :
112 : /* Time stamp of last node dispatch. */
113 : u64 cpu_time_last_node_dispatch;
114 :
115 : /* Time stamp when main loop was entered (time 0). */
116 : u64 cpu_time_main_loop_start;
117 :
118 : /* Incremented once for each main loop. */
119 : volatile u32 main_loop_count;
120 :
121 : /* Count of vectors processed this main loop. */
122 : u32 main_loop_vectors_processed;
123 : u32 main_loop_nodes_processed;
124 :
125 : /* Internal node vectors, calls */
126 : u64 internal_node_vectors;
127 : u64 internal_node_calls;
128 : u64 internal_node_vectors_last_clear;
129 : u64 internal_node_calls_last_clear;
130 :
131 : /* Instantaneous vector rate */
132 : u32 internal_node_last_vectors_per_main_loop;
133 :
134 : /* Main loop hw / sw performance counters */
135 : vlib_node_runtime_perf_callback_set_t vlib_node_runtime_perf_callbacks;
136 :
137 : /* dispatch wrapper function */
138 : vlib_node_function_t *dispatch_wrapper_fn;
139 :
140 : /* Every so often we switch to the next counter. */
141 : #define VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE 7
142 :
143 : /* Jump target to exit main loop with given code. */
144 : u32 main_loop_exit_set;
145 : /* Set e.g. in the SIGTERM signal handler, checked in a safe place... */
146 : volatile u32 main_loop_exit_now;
147 : /* Exit status that will be returned by the process upon exit. */
148 : volatile int main_loop_exit_status;
149 : clib_longjmp_t main_loop_exit;
150 : #define VLIB_MAIN_LOOP_EXIT_NONE 0
151 : #define VLIB_MAIN_LOOP_EXIT_PANIC 1
152 : /* Exit via CLI. */
153 : #define VLIB_MAIN_LOOP_EXIT_CLI 2
154 :
155 : /* Error marker to use when exiting main loop. */
156 : clib_error_t *main_loop_error;
157 :
158 : /* buffer main structure. */
159 : vlib_buffer_main_t *buffer_main;
160 :
161 : /* physical memory main structure. */
162 : vlib_physmem_main_t physmem_main;
163 :
164 : /* Node graph main structure. */
165 : vlib_node_main_t node_main;
166 :
167 : /* Packet trace buffer. */
168 : vlib_trace_main_t trace_main;
169 :
170 : /* Error handling. */
171 : vlib_error_main_t error_main;
172 :
173 : /* Punt packets to underlying operating system for when fast switching
174 : code does not know what to do. */
175 : void (*os_punt_frame) (struct vlib_main_t * vm,
176 : struct vlib_node_runtime_t * node,
177 : vlib_frame_t * frame);
178 :
179 : /* Stream index to use for distribution when MC is enabled. */
180 : u32 mc_stream_index;
181 :
182 : /* Hash table to record which init functions have been called. */
183 : uword *worker_init_functions_called;
184 :
185 : vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join;
186 :
187 : /* Event logger trace flags */
188 : int elog_trace_api_messages;
189 : int elog_trace_cli_commands;
190 : int elog_trace_graph_dispatch;
191 : int elog_trace_graph_circuit;
192 : u32 elog_trace_graph_circuit_node_index;
193 :
194 : /* Node call and return event types. */
195 : elog_event_type_t *node_call_elog_event_types;
196 : elog_event_type_t *node_return_elog_event_types;
197 :
198 : elog_event_type_t *error_elog_event_types;
199 :
200 : /* Seed for random number generator. */
201 : uword random_seed;
202 :
203 : /* Buffer of random data for various uses. */
204 : clib_random_buffer_t random_buffer;
205 :
206 : /* thread, cpu and numa_node indices */
207 : u32 thread_index;
208 : u32 cpu_id;
209 : u32 numa_node;
210 :
211 : /* control-plane API queue signal pending, length indication */
212 : volatile u32 queue_signal_pending;
213 : volatile u32 api_queue_nonempty;
214 : void (*queue_signal_callback) (struct vlib_main_t *);
215 :
216 : /* Top of (worker) dispatch loop callback */
217 : void (**volatile worker_thread_main_loop_callbacks)
218 : (struct vlib_main_t *, u64 t);
219 : void (**volatile worker_thread_main_loop_callback_tmp)
220 : (struct vlib_main_t *, u64 t);
221 : clib_spinlock_t worker_thread_main_loop_callback_lock;
222 :
223 : /* debugging */
224 : volatile int parked_at_barrier;
225 :
226 : /* Dispatch loop time accounting */
227 : u64 loops_this_reporting_interval;
228 : f64 loop_interval_end;
229 : f64 loop_interval_start;
230 : f64 loops_per_second;
231 : f64 seconds_per_loop;
232 : f64 damping_constant;
233 :
234 : /*
235 : * Barrier epoch - Set to current time, each time barrier_sync or
236 : * barrier_release is called with zero recursion.
237 : */
238 : f64 barrier_epoch;
239 :
240 : /* Earliest barrier can be closed again */
241 : f64 barrier_no_close_before;
242 :
243 : /* Barrier counter callback */
244 : void (**volatile barrier_perf_callbacks)
245 : (struct vlib_main_t *, u64 t, int leave);
246 : void (**volatile barrier_perf_callbacks_tmp)
247 : (struct vlib_main_t *, u64 t, int leave);
248 :
249 : /* Need to check the frame queues */
250 : volatile uword check_frame_queues;
251 :
252 : /* RPC requests, main thread only */
253 : uword *pending_rpc_requests;
254 : uword *processing_rpc_requests;
255 : clib_spinlock_t pending_rpc_lock;
256 :
257 : /* buffer fault injector */
258 : u32 buffer_alloc_success_seed;
259 : f64 buffer_alloc_success_rate;
260 :
261 : #ifdef CLIB_SANITIZE_ADDR
262 : /* address sanitizer stack save */
263 : void *asan_stack_save;
264 : #endif
265 : } vlib_main_t;
266 :
267 : typedef struct vlib_global_main_t
268 : {
269 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
270 :
271 : /* Per-thread Mains */
272 : vlib_main_t **vlib_mains;
273 :
274 : /* Name for e.g. syslog. */
275 : char *name;
276 :
277 : /* full path to main executable */
278 : char *exec_path;
279 :
280 : /* command line arguments */
281 : u8 **argv;
282 :
283 : /* post-mortem callbacks */
284 : void (**post_mortem_callbacks) (void);
285 :
286 : /*
287 : * Need to call vlib_worker_thread_node_runtime_update before
288 : * releasing worker thread barrier.
289 : */
290 : int need_vlib_worker_thread_node_runtime_update;
291 :
292 : /* Command line interface. */
293 : vlib_cli_main_t cli_main;
294 :
295 : /* Node registrations added by constructors */
296 : vlib_node_registration_t *node_registrations;
297 :
298 : /* Event logger. */
299 : elog_main_t elog_main;
300 : u32 configured_elog_ring_size;
301 :
302 : /* Packet trace capture filter */
303 : vlib_trace_filter_t trace_filter;
304 :
305 : /* List of init functions to call, setup by constructors */
306 : _vlib_init_function_list_elt_t *init_function_registrations;
307 : _vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
308 : _vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
309 : _vlib_init_function_list_elt_t *worker_init_function_registrations;
310 : _vlib_init_function_list_elt_t *num_workers_change_function_registrations;
311 : _vlib_init_function_list_elt_t *api_init_function_registrations;
312 : vlib_config_function_runtime_t *config_function_registrations;
313 :
314 : /* Hash table to record which init functions have been called. */
315 : uword *init_functions_called;
316 :
317 : } vlib_global_main_t;
318 :
319 : /* Global main structure. */
320 : extern vlib_global_main_t vlib_global_main;
321 :
322 : void vlib_worker_loop (vlib_main_t * vm);
323 :
324 : always_inline f64
325 1223050733 : vlib_time_now (vlib_main_t * vm)
326 : {
327 : #if CLIB_DEBUG > 0
328 : extern __thread uword __os_thread_index;
329 : #endif
330 : /*
331 : * Make sure folks don't pass &vlib_global_main from a worker thread.
332 : */
333 1223050733 : ASSERT (vm->thread_index == __os_thread_index);
334 1223050733 : return clib_time_now (&vm->clib_time) + vm->time_offset;
335 : }
336 :
337 : always_inline f64
338 : vlib_time_now_ticks (vlib_main_t * vm, u64 n)
339 : {
340 : return clib_time_now_internal (&vm->clib_time, n);
341 : }
342 :
343 : /* Busy wait for specified time. */
344 : always_inline void
345 : vlib_time_wait (vlib_main_t * vm, f64 wait)
346 : {
347 : f64 t = vlib_time_now (vm);
348 : f64 limit = t + wait;
349 : while (t < limit)
350 : t = vlib_time_now (vm);
351 : }
352 :
353 : /* Time a piece of code. */
354 : #define vlib_time_code(vm,body) \
355 : do { \
356 : f64 _t[2]; \
357 : _t[0] = vlib_time_now (vm); \
358 : do { body; } while (0); \
359 : _t[1] = vlib_time_now (vm); \
360 : clib_warning ("%.7e", _t[1] - _t[0]); \
361 : } while (0)
362 :
363 : #define vlib_wait_with_timeout(vm,suspend_time,timeout_time,test) \
364 : ({ \
365 : uword __vlib_wait_with_timeout = 0; \
366 : f64 __vlib_wait_time = 0; \
367 : while (! (__vlib_wait_with_timeout = (test)) \
368 : && __vlib_wait_time < (timeout_time)) \
369 : { \
370 : vlib_process_suspend (vm, suspend_time); \
371 : __vlib_wait_time += suspend_time; \
372 : } \
373 : __vlib_wait_with_timeout; \
374 : })
375 :
376 : always_inline void
377 0 : vlib_panic_with_error (vlib_main_t * vm, clib_error_t * error)
378 : {
379 0 : vm->main_loop_error = error;
380 0 : if (vm->main_loop_exit_set)
381 0 : clib_longjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_PANIC);
382 : else
383 : {
384 0 : clib_warning ("panic: %U", format_clib_error, error);
385 0 : abort ();
386 : }
387 0 : }
388 :
389 : #define vlib_panic_with_msg(vm,args...) \
390 : vlib_panic_with_error (vm, clib_error_return (0, args))
391 :
392 : always_inline void
393 : vlib_panic (vlib_main_t * vm)
394 : {
395 : vlib_panic_with_error (vm, 0);
396 : }
397 :
398 : /* Asynchronously requests exit with the given status. */
399 : void vlib_exit_with_status (vlib_main_t *vm, int status);
400 :
401 : always_inline f64
402 3475 : vlib_internal_node_vector_rate (vlib_main_t * vm)
403 : {
404 : u64 vectors;
405 : u64 calls;
406 :
407 3475 : calls = vm->internal_node_calls - vm->internal_node_calls_last_clear;
408 :
409 3475 : if (PREDICT_FALSE (calls == 0))
410 1481 : return 0.0;
411 :
412 1994 : vectors = vm->internal_node_vectors - vm->internal_node_vectors_last_clear;
413 :
414 1994 : return (f64) vectors / (f64) calls;
415 : }
416 :
417 : always_inline void
418 1508 : vlib_clear_internal_node_vector_rate (vlib_main_t * vm)
419 : {
420 1508 : vm->internal_node_calls_last_clear = vm->internal_node_calls;
421 1508 : vm->internal_node_vectors_last_clear = vm->internal_node_vectors;
422 1508 : }
423 :
424 : always_inline void
425 699699000 : vlib_increment_main_loop_counter (vlib_main_t * vm)
426 : {
427 699699000 : vm->main_loop_count++;
428 699699000 : vm->internal_node_last_vectors_per_main_loop = 0;
429 :
430 699699000 : if (PREDICT_FALSE (vm->main_loop_exit_now))
431 575 : clib_longjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_CLI);
432 699699000 : }
433 :
434 : always_inline u32
435 86156489 : vlib_last_vectors_per_main_loop (vlib_main_t * vm)
436 : {
437 86156489 : return vm->internal_node_last_vectors_per_main_loop;
438 : }
439 :
440 : always_inline void
441 1594458165 : vlib_node_runtime_perf_counter (vlib_main_t * vm, vlib_node_runtime_t * node,
442 : vlib_frame_t * frame, uword n, u64 t,
443 : vlib_node_runtime_perf_call_type_t call_type)
444 : {
445 1594458165 : vlib_node_runtime_perf_callback_data_t *v =
446 1594458165 : clib_callback_data_check_and_get (&vm->vlib_node_runtime_perf_callbacks);
447 1594458165 : if (vec_len (v))
448 : {
449 7549 : vlib_node_runtime_perf_callback_args_t args = {
450 : .vm = vm,
451 : .node = node,
452 : .frame = frame,
453 : .packets = n,
454 : .cpu_time_now = t,
455 : .call_type = call_type,
456 : };
457 15098 : clib_callback_data_call_vec (v, &args);
458 : }
459 1594458165 : }
460 :
461 575 : always_inline void vlib_set_queue_signal_callback
462 : (vlib_main_t * vm, void (*fp) (vlib_main_t *))
463 : {
464 575 : vm->queue_signal_callback = fp;
465 575 : }
466 :
467 : always_inline void
468 575 : vlib_main_init ()
469 : {
470 575 : vlib_global_main_t *vgm = &vlib_global_main;
471 : vlib_main_t *vm;
472 :
473 575 : vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
474 :
475 575 : vm = clib_mem_alloc_aligned (sizeof (*vm), CLIB_CACHE_LINE_BYTES);
476 575 : vec_add1_ha (vgm->vlib_mains, vm, 0, CLIB_CACHE_LINE_BYTES);
477 575 : }
478 :
479 : /* Main routine. */
480 : int vlib_main (vlib_main_t * vm, unformat_input_t * input);
481 :
482 : /* Thread stacks, for os_get_thread_index */
483 : extern u8 **vlib_thread_stacks;
484 :
485 : /* Number of thread stacks that the application needs */
486 : u32 vlib_app_num_thread_stacks_needed (void) __attribute__ ((weak));
487 :
488 : void vlib_add_del_post_mortem_callback (void *cb, int is_add);
489 :
490 : vlib_main_t *vlib_get_main_not_inline (void);
491 : elog_main_t *vlib_get_elog_main_not_inline ();
492 :
493 : #endif /* included_vlib_main_h */
494 :
495 : /*
496 : * fd.io coding-style-patch-verification: ON
497 : *
498 : * Local Variables:
499 : * eval: (c-set-style "gnu")
500 : * End:
501 : */
|
