Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * main.c: main vector processing loop
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #include <math.h>
41 : #include <vppinfra/format.h>
42 : #include <vlib/vlib.h>
43 : #include <vlib/threads.h>
44 : #include <vlib/stats/stats.h>
45 : #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
46 :
47 : #include <vlib/unix/unix.h>
48 :
49 : #define VLIB_FRAME_MAGIC (0xabadc0ed)
50 :
51 : always_inline u32 *
52 30598700 : vlib_frame_find_magic (vlib_frame_t * f, vlib_node_t * node)
53 : {
54 30598700 : return (void *) f + node->magic_offset;
55 : }
56 :
57 : static vlib_frame_t *
58 695882 : vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index,
59 : u32 frame_flags)
60 : {
61 695882 : vlib_node_main_t *nm = &vm->node_main;
62 : vlib_frame_size_t *fs;
63 : vlib_node_t *to_node;
64 : vlib_frame_t *f;
65 : u32 l, n;
66 :
67 695882 : ASSERT (vm == vlib_get_main ());
68 :
69 695882 : to_node = vlib_get_node (vm, to_node_index);
70 :
71 695881 : vec_validate (nm->frame_sizes, to_node->frame_size_index);
72 695878 : fs = vec_elt_at_index (nm->frame_sizes, to_node->frame_size_index);
73 :
74 695877 : if (fs->frame_size == 0)
75 33 : fs->frame_size = to_node->frame_size;
76 : else
77 695844 : ASSERT (fs->frame_size == to_node->frame_size);
78 :
79 695877 : n = fs->frame_size;
80 695877 : if ((l = vec_len (fs->free_frames)) > 0)
81 : {
82 : /* Allocate from end of free list. */
83 679821 : f = fs->free_frames[l - 1];
84 679821 : vec_set_len (fs->free_frames, l - 1);
85 : }
86 : else
87 : {
88 16056 : f = clib_mem_alloc_aligned_no_fail (n, CLIB_CACHE_LINE_BYTES);
89 : }
90 :
91 : /* Poison frame when debugging. */
92 : if (CLIB_DEBUG > 0)
93 695879 : clib_memset_u8 (f, 0xfe, n);
94 :
95 : /* Insert magic number. */
96 : {
97 : u32 *magic;
98 :
99 695888 : magic = vlib_frame_find_magic (f, to_node);
100 695889 : *magic = VLIB_FRAME_MAGIC;
101 : }
102 :
103 695889 : f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
104 695889 : f->n_vectors = 0;
105 695889 : f->scalar_offset = to_node->scalar_offset;
106 695889 : f->vector_offset = to_node->vector_offset;
107 695889 : f->aux_offset = to_node->aux_offset;
108 695889 : f->flags = 0;
109 695889 : f->frame_size_index = to_node->frame_size_index;
110 :
111 695889 : fs->n_alloc_frames += 1;
112 :
113 695889 : return f;
114 : }
115 :
116 : /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
117 : Returns frame index. */
118 : static vlib_frame_t *
119 511478 : vlib_frame_alloc (vlib_main_t * vm, vlib_node_runtime_t * from_node_runtime,
120 : u32 to_next_index)
121 : {
122 : vlib_node_t *from_node;
123 :
124 511478 : from_node = vlib_get_node (vm, from_node_runtime->node_index);
125 511478 : ASSERT (to_next_index < vec_len (from_node->next_nodes));
126 :
127 511478 : return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
128 : /* frame_flags */ 0);
129 : }
130 :
131 : vlib_frame_t *
132 184404 : vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index)
133 : {
134 184404 : vlib_frame_t *f = vlib_frame_alloc_to_node (vm, to_node_index,
135 : /* frame_flags */
136 : VLIB_FRAME_FREE_AFTER_DISPATCH);
137 184410 : return vlib_get_frame (vm, f);
138 : }
139 :
140 : static inline void
141 15229700 : vlib_validate_frame_indices (vlib_frame_t * f)
142 : {
143 : if (CLIB_DEBUG > 0)
144 : {
145 : int i;
146 15229700 : u32 *from = vlib_frame_vector_args (f);
147 :
148 : /* Check for bad buffer index values */
149 54698700 : for (i = 0; i < f->n_vectors; i++)
150 : {
151 39469000 : if (from[i] == 0)
152 : {
153 0 : clib_warning ("BUG: buffer index 0 at index %d", i);
154 0 : ASSERT (0);
155 : }
156 39469000 : else if (from[i] == 0xfefefefe)
157 : {
158 0 : clib_warning ("BUG: frame poison pattern at index %d", i);
159 0 : ASSERT (0);
160 : }
161 : }
162 : }
163 15229700 : }
164 :
165 : void
166 184312 : vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f)
167 : {
168 : vlib_pending_frame_t *p;
169 : vlib_node_t *to_node;
170 :
171 184312 : if (f->n_vectors == 0)
172 0 : return;
173 :
174 184312 : ASSERT (vm == vlib_get_main ());
175 :
176 184312 : vlib_validate_frame_indices (f);
177 :
178 184315 : to_node = vlib_get_node (vm, to_node_index);
179 :
180 184314 : vec_add2 (vm->node_main.pending_frames, p, 1);
181 :
182 184313 : f->frame_flags |= VLIB_FRAME_PENDING;
183 184313 : p->frame = vlib_get_frame (vm, f);
184 184313 : p->node_runtime_index = to_node->runtime_index;
185 184313 : p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME;
186 : }
187 :
188 : /* Free given frame. */
189 : void
190 680832 : vlib_frame_free (vlib_main_t *vm, vlib_frame_t *f)
191 : {
192 680832 : vlib_node_main_t *nm = &vm->node_main;
193 : vlib_frame_size_t *fs;
194 :
195 680832 : ASSERT (vm == vlib_get_main ());
196 680835 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
197 :
198 680833 : fs = vec_elt_at_index (nm->frame_sizes, f->frame_size_index);
199 :
200 680833 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
201 :
202 : /* No next frames may point to freed frame. */
203 : if (CLIB_DEBUG > 0)
204 : {
205 : vlib_next_frame_t *nf;
206 1529400000 : vec_foreach (nf, vm->node_main.next_frames) ASSERT (nf->frame != f);
207 : }
208 :
209 681786 : f->frame_flags &= ~(VLIB_FRAME_IS_ALLOCATED | VLIB_FRAME_NO_APPEND);
210 681786 : f->flags = 0;
211 :
212 681786 : vec_add1 (fs->free_frames, f);
213 680836 : ASSERT (fs->n_alloc_frames > 0);
214 680836 : fs->n_alloc_frames -= 1;
215 680836 : }
216 :
217 : static clib_error_t *
218 2 : show_frame_stats (vlib_main_t * vm,
219 : unformat_input_t * input, vlib_cli_command_t * cmd)
220 : {
221 : vlib_frame_size_t *fs;
222 :
223 2 : vlib_cli_output (vm, "%=8s%=6s%=12s%=12s", "Thread", "Size", "# Alloc",
224 : "# Free");
225 6 : foreach_vlib_main ()
226 : {
227 4 : vlib_node_main_t *nm = &this_vlib_main->node_main;
228 14 : vec_foreach (fs, nm->frame_sizes)
229 : {
230 10 : u32 n_alloc = fs->n_alloc_frames;
231 10 : u32 n_free = vec_len (fs->free_frames);
232 :
233 10 : if (n_alloc + n_free > 0)
234 2 : vlib_cli_output (vm, "%=8d%=6d%=12d%=12d",
235 2 : this_vlib_main->thread_index, fs->frame_size,
236 : n_alloc, n_free);
237 : }
238 : }
239 :
240 2 : return 0;
241 : }
242 :
243 : /* *INDENT-OFF* */
244 272887 : VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
245 : .path = "show vlib frame-allocation",
246 : .short_help = "Show node dispatch frame statistics",
247 : .function = show_frame_stats,
248 : };
249 : /* *INDENT-ON* */
250 :
251 : /* Change ownership of enqueue rights to given next node. */
252 : static void
253 278569 : vlib_next_frame_change_ownership (vlib_main_t * vm,
254 : vlib_node_runtime_t * node_runtime,
255 : u32 next_index)
256 : {
257 278569 : vlib_node_main_t *nm = &vm->node_main;
258 : vlib_next_frame_t *next_frame;
259 : vlib_node_t *node, *next_node;
260 :
261 278569 : node = vec_elt (nm->nodes, node_runtime->node_index);
262 :
263 : /* Only internal & input nodes are allowed to call other nodes. */
264 278569 : ASSERT (node->type == VLIB_NODE_TYPE_INTERNAL
265 : || node->type == VLIB_NODE_TYPE_INPUT
266 : || node->type == VLIB_NODE_TYPE_PROCESS);
267 :
268 278569 : ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
269 :
270 : next_frame =
271 278569 : vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
272 278570 : next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
273 :
274 278570 : if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
275 : {
276 : /* Get frame from previous owner. */
277 : vlib_next_frame_t *owner_next_frame;
278 : vlib_next_frame_t tmp;
279 :
280 : owner_next_frame =
281 262160 : vlib_node_get_next_frame (vm,
282 : next_node->owner_node_index,
283 : next_node->owner_next_index);
284 :
285 : /* Swap target next frame with owner's. */
286 262160 : tmp = owner_next_frame[0];
287 262160 : owner_next_frame[0] = next_frame[0];
288 262160 : next_frame[0] = tmp;
289 :
290 : /*
291 : * If next_frame is already pending, we have to track down
292 : * all pending frames and fix their next_frame_index fields.
293 : */
294 262160 : if (next_frame->flags & VLIB_FRAME_PENDING)
295 : {
296 : vlib_pending_frame_t *p;
297 255613 : if (next_frame->frame != NULL)
298 : {
299 794189 : vec_foreach (p, nm->pending_frames)
300 : {
301 538614 : if (p->frame == next_frame->frame)
302 : {
303 90137 : p->next_frame_index =
304 90137 : next_frame - vm->node_main.next_frames;
305 : }
306 : }
307 : }
308 : }
309 : }
310 : else
311 : {
312 : /* No previous owner. Take ownership. */
313 16410 : next_frame->flags |= VLIB_FRAME_OWNER;
314 : }
315 :
316 : /* Record new owner. */
317 278570 : next_node->owner_node_index = node->index;
318 278570 : next_node->owner_next_index = next_index;
319 :
320 : /* Now we should be owner. */
321 278570 : ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
322 278570 : }
323 :
324 : /* Make sure that magic number is still there.
325 : Otherwise, it is likely that caller has overrun frame arguments. */
326 : always_inline void
327 29902800 : validate_frame_magic (vlib_main_t * vm,
328 : vlib_frame_t * f, vlib_node_t * n, uword next_index)
329 : {
330 29902800 : vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
331 29902800 : u32 *magic = vlib_frame_find_magic (f, next_node);
332 29902800 : ASSERT (VLIB_FRAME_MAGIC == magic[0]);
333 29902800 : }
334 :
335 : vlib_frame_t *
336 14857400 : vlib_get_next_frame_internal (vlib_main_t * vm,
337 : vlib_node_runtime_t * node,
338 : u32 next_index, u32 allocate_new_next_frame)
339 : {
340 : vlib_frame_t *f;
341 : vlib_next_frame_t *nf;
342 : u32 n_used;
343 :
344 14857400 : nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
345 :
346 : /* Make sure this next frame owns right to enqueue to destination frame. */
347 14857400 : if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
348 278569 : vlib_next_frame_change_ownership (vm, node, next_index);
349 :
350 : /* ??? Don't need valid flag: can use frame_index == ~0 */
351 14857400 : if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_IS_ALLOCATED)))
352 : {
353 16756 : nf->frame = vlib_frame_alloc (vm, node, next_index);
354 16757 : nf->flags |= VLIB_FRAME_IS_ALLOCATED;
355 : }
356 :
357 14857400 : f = nf->frame;
358 :
359 : /* Has frame been removed from pending vector (e.g. finished dispatching)?
360 : If so we can reuse frame. */
361 14857400 : if ((nf->flags & VLIB_FRAME_PENDING)
362 14610100 : && !(f->frame_flags & VLIB_FRAME_PENDING))
363 : {
364 13078400 : nf->flags &= ~VLIB_FRAME_PENDING;
365 13078400 : f->n_vectors = 0;
366 13078400 : f->flags = 0;
367 : }
368 :
369 : /* Allocate new frame if current one is marked as no-append or
370 : it is already full. */
371 14857400 : n_used = f->n_vectors;
372 14857400 : if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) ||
373 14362700 : (f->frame_flags & VLIB_FRAME_NO_APPEND))
374 : {
375 : /* Old frame may need to be freed after dispatch, since we'll have
376 : two redundant frames from node -> next node. */
377 494724 : if (!(nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH))
378 : {
379 494722 : vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame);
380 494722 : f_old->frame_flags |= VLIB_FRAME_FREE_AFTER_DISPATCH;
381 : }
382 :
383 : /* Allocate new frame to replace full one. */
384 494724 : f = nf->frame = vlib_frame_alloc (vm, node, next_index);
385 494722 : n_used = f->n_vectors;
386 : }
387 :
388 : /* Should have free vectors in frame now. */
389 14857400 : ASSERT (n_used < VLIB_FRAME_SIZE);
390 :
391 : if (CLIB_DEBUG > 0)
392 : {
393 14857400 : validate_frame_magic (vm, f,
394 : vlib_get_node (vm, node->node_index), next_index);
395 : }
396 :
397 14857400 : return f;
398 : }
399 :
400 : static void
401 15045400 : vlib_put_next_frame_validate (vlib_main_t * vm,
402 : vlib_node_runtime_t * rt,
403 : u32 next_index, u32 n_vectors_left)
404 : {
405 15045400 : vlib_node_main_t *nm = &vm->node_main;
406 : vlib_next_frame_t *nf;
407 : vlib_frame_t *f;
408 : vlib_node_runtime_t *next_rt;
409 : vlib_node_t *next_node;
410 : u32 n_before, n_after;
411 :
412 15045400 : nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
413 15045400 : f = vlib_get_frame (vm, nf->frame);
414 :
415 15045400 : ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
416 :
417 15045400 : vlib_validate_frame_indices (f);
418 :
419 15045400 : n_after = VLIB_FRAME_SIZE - n_vectors_left;
420 15045400 : n_before = f->n_vectors;
421 :
422 15045400 : ASSERT (n_after >= n_before);
423 :
424 15045400 : next_rt = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
425 : nf->node_runtime_index);
426 15045400 : next_node = vlib_get_node (vm, next_rt->node_index);
427 15045400 : if (n_after > 0 && next_node->validate_frame)
428 : {
429 4125 : u8 *msg = next_node->validate_frame (vm, rt, f);
430 4125 : if (msg)
431 : {
432 0 : clib_warning ("%v", msg);
433 0 : ASSERT (0);
434 : }
435 4125 : vec_free (msg);
436 : }
437 15045400 : }
438 :
439 : void
440 15045400 : vlib_put_next_frame (vlib_main_t * vm,
441 : vlib_node_runtime_t * r,
442 : u32 next_index, u32 n_vectors_left)
443 : {
444 15045400 : vlib_node_main_t *nm = &vm->node_main;
445 : vlib_next_frame_t *nf;
446 : vlib_frame_t *f;
447 : u32 n_vectors_in_frame;
448 :
449 : if (CLIB_DEBUG > 0)
450 15045400 : vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
451 :
452 15045400 : nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
453 15045400 : f = vlib_get_frame (vm, nf->frame);
454 :
455 : /* Make sure that magic number is still there. Otherwise, caller
456 : has overrun frame meta data. */
457 : if (CLIB_DEBUG > 0)
458 : {
459 15045400 : vlib_node_t *node = vlib_get_node (vm, r->node_index);
460 15045400 : validate_frame_magic (vm, f, node, next_index);
461 : }
462 :
463 : /* Convert # of vectors left -> number of vectors there. */
464 15045400 : ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
465 15045400 : n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
466 :
467 15045400 : f->n_vectors = n_vectors_in_frame;
468 :
469 : /* If vectors were added to frame, add to pending vector. */
470 15045400 : if (PREDICT_TRUE (n_vectors_in_frame > 0))
471 : {
472 : vlib_pending_frame_t *p;
473 : u32 v0, v1;
474 :
475 14810900 : r->cached_next_index = next_index;
476 :
477 14810900 : if (!(f->frame_flags & VLIB_FRAME_PENDING))
478 : {
479 : __attribute__ ((unused)) vlib_node_t *node;
480 :
481 13585900 : node = vlib_get_node (vm, r->node_index);
482 :
483 13585900 : vec_add2 (nm->pending_frames, p, 1);
484 :
485 13585900 : p->frame = nf->frame;
486 13585900 : p->node_runtime_index = nf->node_runtime_index;
487 13585900 : p->next_frame_index = nf - nm->next_frames;
488 13585900 : nf->flags |= VLIB_FRAME_PENDING;
489 13585900 : f->frame_flags |= VLIB_FRAME_PENDING;
490 : }
491 :
492 : /* Copy trace flag from next_frame and from runtime. */
493 14810900 : nf->flags |=
494 14810900 : (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
495 14810900 : flags & VLIB_NODE_FLAG_TRACE);
496 :
497 14810900 : v0 = nf->vectors_since_last_overflow;
498 14810900 : v1 = v0 + n_vectors_in_frame;
499 14810900 : nf->vectors_since_last_overflow = v1;
500 14810900 : if (PREDICT_FALSE (v1 < v0))
501 : {
502 0 : vlib_node_t *node = vlib_get_node (vm, r->node_index);
503 0 : vec_elt (node->n_vectors_by_next_node, next_index) += v0;
504 : }
505 : }
506 15045400 : }
507 :
508 : /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
509 : void
510 2626540 : vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r,
511 : uword n_calls, uword n_vectors,
512 : uword n_clocks)
513 : {
514 2626540 : n->stats_total.calls += n_calls + r->calls_since_last_overflow;
515 2626540 : n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
516 2626540 : n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
517 2626540 : n->stats_total.max_clock = r->max_clock;
518 2626540 : n->stats_total.max_clock_n = r->max_clock_n;
519 :
520 2626540 : r->calls_since_last_overflow = 0;
521 2626540 : r->vectors_since_last_overflow = 0;
522 2626540 : r->clocks_since_last_overflow = 0;
523 2626540 : }
524 :
525 : void
526 2626540 : vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r,
527 : uword n_calls, uword n_vectors, uword n_clocks)
528 : {
529 2626540 : vlib_node_t *n = vlib_get_node (vm, r->node_index);
530 2626540 : vlib_node_runtime_sync_stats_node (n, r, n_calls, n_vectors, n_clocks);
531 2626540 : }
532 :
533 : always_inline void __attribute__ ((unused))
534 : vlib_process_sync_stats (vlib_main_t * vm,
535 : vlib_process_t * p,
536 : uword n_calls, uword n_vectors, uword n_clocks)
537 : {
538 : vlib_node_runtime_t *rt = &p->node_runtime;
539 : vlib_node_t *n = vlib_get_node (vm, rt->node_index);
540 : vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks);
541 : n->stats_total.suspends += p->n_suspends;
542 : p->n_suspends = 0;
543 : }
544 :
545 : void
546 1892500 : vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n)
547 : {
548 : vlib_node_runtime_t *rt;
549 :
550 1892500 : if (n->type == VLIB_NODE_TYPE_PROCESS)
551 : {
552 : /* Nothing to do for PROCESS nodes except in main thread */
553 123088 : if (vm != vlib_get_first_main ())
554 18001 : return;
555 :
556 105087 : vlib_process_t *p = vlib_get_process_from_node (vm, n);
557 105087 : n->stats_total.suspends += p->n_suspends;
558 105087 : p->n_suspends = 0;
559 105087 : rt = &p->node_runtime;
560 : }
561 : else
562 1769410 : rt =
563 1769410 : vec_elt_at_index (vm->node_main.nodes_by_type[n->type],
564 : n->runtime_index);
565 :
566 1874490 : vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0);
567 :
568 : /* Sync up runtime next frame vector counters with main node structure. */
569 : {
570 : vlib_next_frame_t *nf;
571 : uword i;
572 7828240 : for (i = 0; i < rt->n_next_nodes; i++)
573 : {
574 5953740 : nf = vlib_node_runtime_get_next_frame (vm, rt, i);
575 5953740 : vec_elt (n->n_vectors_by_next_node, i) +=
576 5953740 : nf->vectors_since_last_overflow;
577 5953740 : nf->vectors_since_last_overflow = 0;
578 : }
579 : }
580 : }
581 :
582 : always_inline u32
583 668688000 : vlib_node_runtime_update_stats (vlib_main_t * vm,
584 : vlib_node_runtime_t * node,
585 : uword n_calls,
586 : uword n_vectors, uword n_clocks)
587 : {
588 : u32 ca0, ca1, v0, v1, cl0, cl1, r;
589 :
590 668688000 : cl0 = cl1 = node->clocks_since_last_overflow;
591 668688000 : ca0 = ca1 = node->calls_since_last_overflow;
592 668688000 : v0 = v1 = node->vectors_since_last_overflow;
593 :
594 668688000 : ca1 = ca0 + n_calls;
595 668688000 : v1 = v0 + n_vectors;
596 668688000 : cl1 = cl0 + n_clocks;
597 :
598 668688000 : node->calls_since_last_overflow = ca1;
599 668688000 : node->clocks_since_last_overflow = cl1;
600 668688000 : node->vectors_since_last_overflow = v1;
601 :
602 668688000 : node->max_clock_n = node->max_clock > n_clocks ?
603 : node->max_clock_n : n_vectors;
604 668688000 : node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
605 :
606 668688000 : r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
607 :
608 668777000 : if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0))
609 : {
610 2949 : node->calls_since_last_overflow = ca0;
611 2949 : node->clocks_since_last_overflow = cl0;
612 2949 : node->vectors_since_last_overflow = v0;
613 :
614 2949 : vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks);
615 : }
616 :
617 668778000 : return r;
618 : }
619 :
620 : always_inline void
621 1691510 : vlib_process_update_stats (vlib_main_t * vm,
622 : vlib_process_t * p,
623 : uword n_calls, uword n_vectors, uword n_clocks)
624 : {
625 1691510 : vlib_node_runtime_update_stats (vm, &p->node_runtime,
626 : n_calls, n_vectors, n_clocks);
627 1691510 : }
628 :
629 : static clib_error_t *
630 20 : vlib_cli_elog_clear (vlib_main_t * vm,
631 : unformat_input_t * input, vlib_cli_command_t * cmd)
632 : {
633 20 : elog_reset_buffer (&vlib_global_main.elog_main);
634 20 : return 0;
635 : }
636 :
637 : /* *INDENT-OFF* */
638 272887 : VLIB_CLI_COMMAND (elog_clear_cli, static) = {
639 : .path = "event-logger clear",
640 : .short_help = "Clear the event log",
641 : .function = vlib_cli_elog_clear,
642 : };
643 : /* *INDENT-ON* */
644 :
645 : #ifdef CLIB_UNIX
646 : static clib_error_t *
647 0 : elog_save_buffer (vlib_main_t * vm,
648 : unformat_input_t * input, vlib_cli_command_t * cmd)
649 : {
650 0 : elog_main_t *em = &vlib_global_main.elog_main;
651 : char *file, *chroot_file;
652 0 : clib_error_t *error = 0;
653 :
654 0 : if (!unformat (input, "%s", &file))
655 : {
656 0 : vlib_cli_output (vm, "expected file name, got `%U'",
657 : format_unformat_error, input);
658 0 : return 0;
659 : }
660 :
661 : /* It's fairly hard to get "../oopsie" through unformat; just in case */
662 0 : if (strstr (file, "..") || index (file, '/'))
663 : {
664 0 : vlib_cli_output (vm, "illegal characters in filename '%s'", file);
665 0 : return 0;
666 : }
667 :
668 0 : chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
669 :
670 0 : vec_free (file);
671 :
672 0 : vlib_cli_output (vm, "Saving %wd of %wd events to %s",
673 : elog_n_events_in_buffer (em),
674 : elog_buffer_capacity (em), chroot_file);
675 :
676 0 : vlib_worker_thread_barrier_sync (vm);
677 0 : error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
678 0 : vlib_worker_thread_barrier_release (vm);
679 0 : vec_free (chroot_file);
680 0 : return error;
681 : }
682 :
683 : void
684 0 : vlib_post_mortem_dump (void)
685 : {
686 0 : vlib_global_main_t *vgm = vlib_get_global_main ();
687 :
688 0 : for (int i = 0; i < vec_len (vgm->post_mortem_callbacks); i++)
689 0 : (vgm->post_mortem_callbacks[i]) ();
690 0 : }
691 :
692 : /* *INDENT-OFF* */
693 272887 : VLIB_CLI_COMMAND (elog_save_cli, static) = {
694 : .path = "event-logger save",
695 : .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
696 : .function = elog_save_buffer,
697 : };
698 : /* *INDENT-ON* */
699 :
700 : static clib_error_t *
701 0 : elog_stop (vlib_main_t * vm,
702 : unformat_input_t * input, vlib_cli_command_t * cmd)
703 : {
704 0 : elog_main_t *em = &vlib_global_main.elog_main;
705 :
706 0 : em->n_total_events_disable_limit = em->n_total_events;
707 :
708 0 : vlib_cli_output (vm, "Stopped the event logger...");
709 0 : return 0;
710 : }
711 :
712 : /* *INDENT-OFF* */
713 272887 : VLIB_CLI_COMMAND (elog_stop_cli, static) = {
714 : .path = "event-logger stop",
715 : .short_help = "Stop the event-logger",
716 : .function = elog_stop,
717 : };
718 : /* *INDENT-ON* */
719 :
720 : static clib_error_t *
721 0 : elog_restart (vlib_main_t * vm,
722 : unformat_input_t * input, vlib_cli_command_t * cmd)
723 : {
724 0 : elog_main_t *em = &vlib_global_main.elog_main;
725 :
726 0 : em->n_total_events_disable_limit = ~0;
727 :
728 0 : vlib_cli_output (vm, "Restarted the event logger...");
729 0 : return 0;
730 : }
731 :
732 : /* *INDENT-OFF* */
733 272887 : VLIB_CLI_COMMAND (elog_restart_cli, static) = {
734 : .path = "event-logger restart",
735 : .short_help = "Restart the event-logger",
736 : .function = elog_restart,
737 : };
738 : /* *INDENT-ON* */
739 :
740 : static clib_error_t *
741 0 : elog_resize_command_fn (vlib_main_t * vm,
742 : unformat_input_t * input, vlib_cli_command_t * cmd)
743 : {
744 0 : elog_main_t *em = &vlib_global_main.elog_main;
745 : u32 tmp;
746 :
747 : /* Stop the parade */
748 0 : elog_reset_buffer (em);
749 :
750 0 : if (unformat (input, "%d", &tmp))
751 : {
752 0 : elog_alloc (em, tmp);
753 0 : em->n_total_events_disable_limit = ~0;
754 : }
755 : else
756 0 : return clib_error_return (0, "Must specify how many events in the ring");
757 :
758 0 : vlib_cli_output (vm, "Resized ring and restarted the event logger...");
759 0 : return 0;
760 : }
761 :
762 : /* *INDENT-OFF* */
763 272887 : VLIB_CLI_COMMAND (elog_resize_cli, static) = {
764 : .path = "event-logger resize",
765 : .short_help = "event-logger resize <nnn>",
766 : .function = elog_resize_command_fn,
767 : };
768 : /* *INDENT-ON* */
769 :
770 : #endif /* CLIB_UNIX */
771 :
772 : static void
773 0 : elog_show_buffer_internal (vlib_main_t * vm, u32 n_events_to_show)
774 : {
775 0 : elog_main_t *em = &vlib_global_main.elog_main;
776 : elog_event_t *e, *es;
777 : f64 dt;
778 :
779 : /* Show events in VLIB time since log clock starts after VLIB clock. */
780 0 : dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
781 0 : * vm->clib_time.seconds_per_clock;
782 :
783 0 : es = elog_peek_events (em);
784 0 : vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
785 : em->event_ring_size,
786 0 : em->n_total_events < em->n_total_events_disable_limit ?
787 : "running" : "stopped");
788 0 : vec_foreach (e, es)
789 : {
790 0 : vlib_cli_output (vm, "%18.9f: %U",
791 0 : e->time + dt, format_elog_event, em, e);
792 0 : n_events_to_show--;
793 0 : if (n_events_to_show == 0)
794 0 : break;
795 : }
796 0 : vec_free (es);
797 :
798 0 : }
799 :
800 : static clib_error_t *
801 0 : elog_show_buffer (vlib_main_t * vm,
802 : unformat_input_t * input, vlib_cli_command_t * cmd)
803 : {
804 : u32 n_events_to_show;
805 0 : clib_error_t *error = 0;
806 :
807 0 : n_events_to_show = 250;
808 0 : while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
809 : {
810 0 : if (unformat (input, "%d", &n_events_to_show))
811 : ;
812 0 : else if (unformat (input, "all"))
813 0 : n_events_to_show = ~0;
814 : else
815 0 : return unformat_parse_error (input);
816 : }
817 0 : elog_show_buffer_internal (vm, n_events_to_show);
818 0 : return error;
819 : }
820 :
821 : /* *INDENT-OFF* */
822 272887 : VLIB_CLI_COMMAND (elog_show_cli, static) = {
823 : .path = "show event-logger",
824 : .short_help = "Show event logger info",
825 : .function = elog_show_buffer,
826 : };
827 : /* *INDENT-ON* */
828 :
829 : void
830 0 : vlib_gdb_show_event_log (void)
831 : {
832 0 : elog_show_buffer_internal (vlib_get_main (), (u32) ~ 0);
833 0 : }
834 :
835 : static inline void
836 1337170000 : vlib_elog_main_loop_event (vlib_main_t * vm,
837 : u32 node_index,
838 : u64 time, u32 n_vectors, u32 is_return)
839 : {
840 1337170000 : vlib_main_t *evm = vlib_get_first_main ();
841 1337100000 : elog_main_t *em = vlib_get_elog_main ();
842 1337050000 : int enabled = evm->elog_trace_graph_dispatch |
843 1337050000 : evm->elog_trace_graph_circuit;
844 :
845 1337050000 : if (PREDICT_FALSE (enabled && n_vectors))
846 : {
847 0 : if (PREDICT_FALSE (!elog_is_enabled (em)))
848 : {
849 0 : evm->elog_trace_graph_dispatch = 0;
850 0 : evm->elog_trace_graph_circuit = 0;
851 0 : return;
852 : }
853 0 : if (PREDICT_TRUE
854 : (evm->elog_trace_graph_dispatch ||
855 : (evm->elog_trace_graph_circuit &&
856 : node_index == evm->elog_trace_graph_circuit_node_index)))
857 : {
858 0 : elog_track (em,
859 : /* event type */
860 0 : vec_elt_at_index (is_return
861 : ? evm->node_return_elog_event_types
862 : : evm->node_call_elog_event_types,
863 : node_index),
864 : /* track */
865 0 : (vm->thread_index ?
866 0 : &vlib_worker_threads[vm->thread_index].elog_track
867 : : &em->default_track),
868 : /* data to log */ n_vectors);
869 : }
870 : }
871 : }
872 :
873 : static inline void
874 : add_trajectory_trace (vlib_buffer_t * b, u32 node_index)
875 : {
876 : #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
877 : if (PREDICT_FALSE (b->trajectory_nb >= VLIB_BUFFER_TRACE_TRAJECTORY_MAX))
878 : return;
879 : b->trajectory_trace[b->trajectory_nb] = node_index;
880 : b->trajectory_nb++;
881 : #endif
882 : }
883 :
884 : static_always_inline u64
885 16296500000 : dispatch_node (vlib_main_t * vm,
886 : vlib_node_runtime_t * node,
887 : vlib_node_type_t type,
888 : vlib_node_state_t dispatch_state,
889 : vlib_frame_t * frame, u64 last_time_stamp)
890 : {
891 : uword n, v;
892 : u64 t;
893 16296500000 : vlib_node_main_t *nm = &vm->node_main;
894 : vlib_next_frame_t *nf;
895 :
896 : if (CLIB_DEBUG > 0)
897 : {
898 16296500000 : vlib_node_t *n = vlib_get_node (vm, node->node_index);
899 16293700000 : ASSERT (n->type == type);
900 : }
901 :
902 : /* Only non-internal nodes may be disabled. */
903 16293700000 : if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
904 : {
905 15184800000 : ASSERT (type != VLIB_NODE_TYPE_INTERNAL);
906 15185500000 : return last_time_stamp;
907 : }
908 :
909 1108910000 : if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
910 1095140000 : && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
911 : {
912 1108240000 : u32 c = node->input_main_loops_per_call;
913 : /* Only call node when count reaches zero. */
914 1108240000 : if (c)
915 : {
916 455421000 : node->input_main_loops_per_call = c - 1;
917 455421000 : return last_time_stamp;
918 : }
919 : }
920 :
921 : /* Speculatively prefetch next frames. */
922 653493000 : if (node->n_next_nodes > 0)
923 : {
924 414790000 : nf = vec_elt_at_index (nm->next_frames, node->next_frame_index);
925 414790000 : CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
926 : }
927 :
928 667171000 : vm->cpu_time_last_node_dispatch = last_time_stamp;
929 :
930 680942000 : vlib_elog_main_loop_event (vm, node->node_index,
931 13770200 : last_time_stamp, frame ? frame->n_vectors : 0,
932 : /* is_after */ 0);
933 :
934 667167000 : vlib_node_runtime_perf_counter (vm, node, frame, 0, last_time_stamp,
935 : VLIB_NODE_RUNTIME_PERF_BEFORE);
936 :
937 : /*
938 : * Turn this on if you run into
939 : * "bad monkey" contexts, and you want to know exactly
940 : * which nodes they've visited... See ixge.c...
941 : */
942 : if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
943 : {
944 : int i;
945 : u32 *from;
946 : from = vlib_frame_vector_args (frame);
947 : for (i = 0; i < frame->n_vectors; i++)
948 : {
949 : vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
950 : add_trajectory_trace (b, node->node_index);
951 : }
952 : if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0))
953 : n = node->function (vm, node, frame);
954 : else
955 : n = vm->dispatch_wrapper_fn (vm, node, frame);
956 : }
957 : else
958 : {
959 667164000 : if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0))
960 667164000 : n = node->function (vm, node, frame);
961 : else
962 33 : n = vm->dispatch_wrapper_fn (vm, node, frame);
963 : }
964 :
965 667033000 : t = clib_cpu_time_now ();
966 :
967 667028000 : vlib_node_runtime_perf_counter (vm, node, frame, n, t,
968 : VLIB_NODE_RUNTIME_PERF_AFTER);
969 :
970 667015000 : vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ );
971 :
972 666985000 : vm->main_loop_vectors_processed += n;
973 666985000 : vm->main_loop_nodes_processed += n > 0;
974 :
975 666985000 : v = vlib_node_runtime_update_stats (vm, node,
976 : /* n_calls */ 1,
977 : /* n_vectors */ n,
978 : /* n_clocks */ t - last_time_stamp);
979 :
980 : /* When in adaptive mode and vector rate crosses threshold switch to
981 : polling mode and vice versa. */
982 667088000 : if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE))
983 : {
984 : /* *INDENT-OFF* */
985 : ELOG_TYPE_DECLARE (e) =
986 : {
987 : .function = (char *) __FUNCTION__,
988 : .format = "%s vector length %d, switching to %s",
989 : .format_args = "T4i4t4",
990 : .n_enum_strings = 2,
991 : .enum_strings = {
992 : "interrupt", "polling",
993 : },
994 : };
995 : /* *INDENT-ON* */
996 : struct
997 : {
998 : u32 node_name, vector_length, is_polling;
999 : } *ed;
1000 :
1001 6771 : if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1002 3959 : && v >= nm->polling_threshold_vector_length) &&
1003 17 : !(node->flags &
1004 : VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
1005 17 : {
1006 17 : vlib_node_t *n = vlib_get_node (vm, node->node_index);
1007 17 : n->state = VLIB_NODE_STATE_POLLING;
1008 17 : node->state = VLIB_NODE_STATE_POLLING;
1009 17 : node->flags &=
1010 : ~VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
1011 17 : node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
1012 17 : nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1013 17 : nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1014 :
1015 17 : if (PREDICT_FALSE (
1016 : vlib_get_first_main ()->elog_trace_graph_dispatch))
1017 : {
1018 0 : vlib_worker_thread_t *w = vlib_worker_threads
1019 0 : + vm->thread_index;
1020 :
1021 0 : ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1022 : w->elog_track);
1023 0 : ed->node_name = n->name_elog_string;
1024 0 : ed->vector_length = v;
1025 0 : ed->is_polling = 1;
1026 : }
1027 : }
1028 6754 : else if (dispatch_state == VLIB_NODE_STATE_POLLING
1029 2812 : && v <= nm->interrupt_threshold_vector_length)
1030 : {
1031 34 : vlib_node_t *n = vlib_get_node (vm, node->node_index);
1032 34 : if (node->flags &
1033 : VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)
1034 : {
1035 : /* Switch to interrupt mode after dispatch in polling one more time.
1036 : This allows driver to re-enable interrupts. */
1037 17 : n->state = VLIB_NODE_STATE_INTERRUPT;
1038 17 : node->state = VLIB_NODE_STATE_INTERRUPT;
1039 17 : node->flags &=
1040 : ~VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
1041 17 : nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1;
1042 17 : nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1;
1043 :
1044 : }
1045 : else
1046 : {
1047 17 : vlib_worker_thread_t *w = vlib_worker_threads
1048 17 : + vm->thread_index;
1049 17 : node->flags |=
1050 : VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
1051 17 : if (PREDICT_FALSE (
1052 : vlib_get_first_main ()->elog_trace_graph_dispatch))
1053 : {
1054 1820 : ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1055 : w->elog_track);
1056 0 : ed->node_name = n->name_elog_string;
1057 0 : ed->vector_length = v;
1058 0 : ed->is_polling = 0;
1059 : }
1060 : }
1061 : }
1062 : }
1063 :
1064 667086000 : return t;
1065 : }
1066 :
1067 : static u64
1068 13770200 : dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1069 : u64 last_time_stamp)
1070 : {
1071 13770200 : vlib_node_main_t *nm = &vm->node_main;
1072 : vlib_frame_t *f;
1073 : vlib_next_frame_t *nf, nf_placeholder;
1074 : vlib_node_runtime_t *n;
1075 : vlib_frame_t *restore_frame;
1076 : vlib_pending_frame_t *p;
1077 :
1078 : /* See comment below about dangling references to nm->pending_frames */
1079 13770200 : p = nm->pending_frames + pending_frame_index;
1080 :
1081 13770200 : n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
1082 : p->node_runtime_index);
1083 :
1084 13770200 : f = vlib_get_frame (vm, p->frame);
1085 13770200 : if (p->next_frame_index == VLIB_PENDING_FRAME_NO_NEXT_FRAME)
1086 : {
1087 : /* No next frame: so use placeholder on stack. */
1088 184313 : nf = &nf_placeholder;
1089 184313 : nf->flags = f->frame_flags & VLIB_NODE_FLAG_TRACE;
1090 184313 : nf->frame = NULL;
1091 : }
1092 : else
1093 13585900 : nf = vec_elt_at_index (nm->next_frames, p->next_frame_index);
1094 :
1095 13770200 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
1096 :
1097 : /* Force allocation of new frame while current frame is being
1098 : dispatched. */
1099 13770200 : restore_frame = NULL;
1100 13770200 : if (nf->frame == p->frame)
1101 : {
1102 13091200 : nf->frame = NULL;
1103 13091200 : nf->flags &= ~VLIB_FRAME_IS_ALLOCATED;
1104 13091200 : if (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH))
1105 13091000 : restore_frame = p->frame;
1106 : }
1107 :
1108 : /* Frame must be pending. */
1109 13770200 : ASSERT (f->frame_flags & VLIB_FRAME_PENDING);
1110 13770200 : ASSERT (f->n_vectors > 0);
1111 :
1112 : /* Copy trace flag from next frame to node.
1113 : Trace flag indicates that at least one vector in the dispatched
1114 : frame is traced. */
1115 13770200 : n->flags &= ~VLIB_NODE_FLAG_TRACE;
1116 13770200 : n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1117 13770200 : nf->flags &= ~VLIB_FRAME_TRACE;
1118 :
1119 13770200 : last_time_stamp = dispatch_node (vm, n,
1120 : VLIB_NODE_TYPE_INTERNAL,
1121 : VLIB_NODE_STATE_POLLING,
1122 : f, last_time_stamp);
1123 : /* Internal node vector-rate accounting, for summary stats */
1124 13770200 : vm->internal_node_vectors += f->n_vectors;
1125 13770200 : vm->internal_node_calls++;
1126 13770200 : vm->internal_node_last_vectors_per_main_loop =
1127 13770200 : (f->n_vectors > vm->internal_node_last_vectors_per_main_loop) ?
1128 13770200 : f->n_vectors : vm->internal_node_last_vectors_per_main_loop;
1129 :
1130 13770200 : f->frame_flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_NO_APPEND);
1131 :
1132 : /* Frame is ready to be used again, so restore it. */
1133 13770200 : if (restore_frame != NULL)
1134 : {
1135 : /*
1136 : * We musn't restore a frame that is flagged to be freed. This
1137 : * shouldn't happen since frames to be freed post dispatch are
1138 : * those used when the to-node frame becomes full i.e. they form a
1139 : * sort of queue of frames to a single node. If we get here then
1140 : * the to-node frame and the pending frame *were* the same, and so
1141 : * we removed the to-node frame. Therefore this frame is no
1142 : * longer part of the queue for that node and hence it cannot be
1143 : * it's overspill.
1144 : */
1145 13091000 : ASSERT (!(f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH));
1146 :
1147 : /*
1148 : * NB: dispatching node n can result in the creation and scheduling
1149 : * of new frames, and hence in the reallocation of nm->pending_frames.
1150 : * Recompute p, or no supper. This was broken for more than 10 years.
1151 : */
1152 13091000 : p = nm->pending_frames + pending_frame_index;
1153 :
1154 : /*
1155 : * p->next_frame_index can change during node dispatch if node
1156 : * function decides to change graph hook up.
1157 : */
1158 13091000 : nf = vec_elt_at_index (nm->next_frames, p->next_frame_index);
1159 13091000 : nf->flags |= VLIB_FRAME_IS_ALLOCATED;
1160 :
1161 13091000 : if (NULL == nf->frame)
1162 : {
1163 : /* no new frame has been assigned to this node, use the saved one */
1164 13090800 : nf->frame = restore_frame;
1165 13090800 : f->n_vectors = 0;
1166 13090800 : f->flags = 0;
1167 : }
1168 : else
1169 : {
1170 : /* The node has gained a frame, implying packets from the current frame
1171 : were re-queued to this same node. we don't need the saved one
1172 : anymore */
1173 176 : vlib_frame_free (vm, f);
1174 : }
1175 : }
1176 : else
1177 : {
1178 679211 : if (f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH)
1179 : {
1180 679041 : ASSERT (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH));
1181 679041 : vlib_frame_free (vm, f);
1182 : }
1183 : }
1184 :
1185 13770200 : return last_time_stamp;
1186 : }
1187 :
1188 : always_inline uword
1189 451 : vlib_process_stack_is_valid (vlib_process_t * p)
1190 : {
1191 451 : return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1192 : }
1193 :
1194 : typedef struct
1195 : {
1196 : vlib_main_t *vm;
1197 : vlib_process_t *process;
1198 : vlib_frame_t *frame;
1199 : } vlib_process_bootstrap_args_t;
1200 :
1201 : /* Called in process stack. */
1202 : static uword
1203 24692 : vlib_process_bootstrap (uword _a)
1204 : {
1205 : vlib_process_bootstrap_args_t *a;
1206 : vlib_main_t *vm;
1207 : vlib_node_runtime_t *node;
1208 : vlib_frame_t *f;
1209 : vlib_process_t *p;
1210 : uword n;
1211 :
1212 24692 : a = uword_to_pointer (_a, vlib_process_bootstrap_args_t *);
1213 :
1214 24692 : vm = a->vm;
1215 24692 : p = a->process;
1216 24692 : vlib_process_finish_switch_stack (vm);
1217 :
1218 24692 : f = a->frame;
1219 24692 : node = &p->node_runtime;
1220 :
1221 24692 : n = node->function (vm, node, f);
1222 :
1223 451 : ASSERT (vlib_process_stack_is_valid (p));
1224 :
1225 451 : vlib_process_start_switch_stack (vm, 0);
1226 451 : clib_longjmp (&p->return_longjmp, n);
1227 :
1228 0 : return n;
1229 : }
1230 :
1231 : /* Called in main stack. */
1232 : static_always_inline uword
1233 24692 : vlib_process_startup (vlib_main_t * vm, vlib_process_t * p, vlib_frame_t * f)
1234 : {
1235 : vlib_process_bootstrap_args_t a;
1236 : uword r;
1237 :
1238 24692 : a.vm = vm;
1239 24692 : a.process = p;
1240 24692 : a.frame = f;
1241 :
1242 24692 : r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN);
1243 49384 : if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN)
1244 : {
1245 24692 : vlib_process_start_switch_stack (vm, p);
1246 24692 : r = clib_calljmp (vlib_process_bootstrap, pointer_to_uword (&a),
1247 24692 : (void *) p->stack + (1 << p->log2_n_stack_bytes));
1248 : }
1249 : else
1250 24692 : vlib_process_finish_switch_stack (vm);
1251 :
1252 24692 : return r;
1253 : }
1254 :
1255 : static_always_inline uword
1256 1666820 : vlib_process_resume (vlib_main_t * vm, vlib_process_t * p)
1257 : {
1258 : uword r;
1259 1666820 : p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
1260 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
1261 : | VLIB_PROCESS_RESUME_PENDING);
1262 1666820 : r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN);
1263 3333640 : if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN)
1264 : {
1265 1666820 : vlib_process_start_switch_stack (vm, p);
1266 1666820 : clib_longjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_RESUME);
1267 : }
1268 : else
1269 1666820 : vlib_process_finish_switch_stack (vm);
1270 1666820 : return r;
1271 : }
1272 :
1273 : static u64
1274 26369 : dispatch_process (vlib_main_t * vm,
1275 : vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1276 : {
1277 26369 : vlib_node_main_t *nm = &vm->node_main;
1278 26369 : vlib_node_runtime_t *node_runtime = &p->node_runtime;
1279 26369 : vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1280 : u32 old_process_index;
1281 : u64 t;
1282 : uword n_vectors, is_suspend;
1283 :
1284 26369 : if (node->state != VLIB_NODE_STATE_POLLING
1285 24737 : || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
1286 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)))
1287 1677 : return last_time_stamp;
1288 :
1289 24692 : p->flags |= VLIB_PROCESS_IS_RUNNING;
1290 :
1291 24692 : t = last_time_stamp;
1292 24692 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1293 0 : f ? f->n_vectors : 0, /* is_after */ 0);
1294 :
1295 : /* Save away current process for suspend. */
1296 24692 : old_process_index = nm->current_process_index;
1297 24692 : nm->current_process_index = node->runtime_index;
1298 :
1299 24692 : vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp,
1300 : VLIB_NODE_RUNTIME_PERF_BEFORE);
1301 :
1302 24692 : n_vectors = vlib_process_startup (vm, p, f);
1303 :
1304 24692 : nm->current_process_index = old_process_index;
1305 :
1306 24692 : ASSERT (n_vectors != VLIB_PROCESS_RETURN_LONGJMP_RETURN);
1307 24692 : is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1308 24692 : if (is_suspend)
1309 : {
1310 : vlib_pending_frame_t *pf;
1311 :
1312 24692 : n_vectors = 0;
1313 24692 : pool_get (nm->suspended_process_frames, pf);
1314 24692 : pf->node_runtime_index = node->runtime_index;
1315 24692 : pf->frame = f;
1316 24692 : pf->next_frame_index = ~0;
1317 :
1318 24692 : p->n_suspends += 1;
1319 24692 : p->suspended_process_frame_index = pf - nm->suspended_process_frames;
1320 :
1321 24692 : if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
1322 : {
1323 13508 : TWT (tw_timer_wheel) * tw =
1324 : (TWT (tw_timer_wheel) *) nm->timing_wheel;
1325 13508 : p->stop_timer_handle =
1326 13508 : TW (tw_timer_start) (tw,
1327 : vlib_timing_wheel_data_set_suspended_process
1328 : (node->runtime_index) /* [sic] pool idex */ ,
1329 : 0 /* timer_id */ ,
1330 : p->resume_clock_interval);
1331 : }
1332 : }
1333 : else
1334 0 : p->flags &= ~VLIB_PROCESS_IS_RUNNING;
1335 :
1336 24692 : t = clib_cpu_time_now ();
1337 :
1338 24692 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1339 : /* is_after */ 1);
1340 :
1341 24692 : vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t,
1342 : VLIB_NODE_RUNTIME_PERF_AFTER);
1343 :
1344 24692 : vlib_process_update_stats (vm, p,
1345 : /* n_calls */ !is_suspend,
1346 : /* n_vectors */ n_vectors,
1347 : /* n_clocks */ t - last_time_stamp);
1348 :
1349 24692 : return t;
1350 : }
1351 :
1352 : void
1353 96 : vlib_start_process (vlib_main_t * vm, uword process_index)
1354 : {
1355 96 : vlib_node_main_t *nm = &vm->node_main;
1356 96 : vlib_process_t *p = vec_elt (nm->processes, process_index);
1357 96 : dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1358 96 : }
1359 :
1360 : static u64
1361 1666820 : dispatch_suspended_process (vlib_main_t * vm,
1362 : uword process_index, u64 last_time_stamp)
1363 : {
1364 1666820 : vlib_node_main_t *nm = &vm->node_main;
1365 : vlib_node_runtime_t *node_runtime;
1366 : vlib_node_t *node;
1367 : vlib_frame_t *f;
1368 : vlib_process_t *p;
1369 : vlib_pending_frame_t *pf;
1370 : u64 t, n_vectors, is_suspend;
1371 :
1372 1666820 : t = last_time_stamp;
1373 :
1374 1666820 : p = vec_elt (nm->processes, process_index);
1375 1666820 : if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING)))
1376 0 : return last_time_stamp;
1377 :
1378 1666820 : ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
1379 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT));
1380 :
1381 1666820 : pf = pool_elt_at_index (nm->suspended_process_frames,
1382 : p->suspended_process_frame_index);
1383 :
1384 1666820 : node_runtime = &p->node_runtime;
1385 1666820 : node = vlib_get_node (vm, node_runtime->node_index);
1386 1666820 : f = pf->frame;
1387 :
1388 1666820 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1389 0 : f ? f->n_vectors : 0, /* is_after */ 0);
1390 :
1391 : /* Save away current process for suspend. */
1392 1666820 : nm->current_process_index = node->runtime_index;
1393 :
1394 1666820 : vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp,
1395 : VLIB_NODE_RUNTIME_PERF_BEFORE);
1396 :
1397 1666820 : n_vectors = vlib_process_resume (vm, p);
1398 1666820 : t = clib_cpu_time_now ();
1399 :
1400 1666820 : nm->current_process_index = ~0;
1401 :
1402 1666820 : is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1403 1666820 : if (is_suspend)
1404 : {
1405 : /* Suspend it again. */
1406 1666370 : n_vectors = 0;
1407 1666370 : p->n_suspends += 1;
1408 1666370 : if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
1409 : {
1410 1644420 : p->stop_timer_handle =
1411 1644420 : TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1412 : vlib_timing_wheel_data_set_suspended_process
1413 : (node->runtime_index) /* [sic] pool idex */ ,
1414 : 0 /* timer_id */ ,
1415 : p->resume_clock_interval);
1416 : }
1417 : }
1418 : else
1419 : {
1420 451 : p->flags &= ~VLIB_PROCESS_IS_RUNNING;
1421 451 : pool_put_index (nm->suspended_process_frames,
1422 : p->suspended_process_frame_index);
1423 451 : p->suspended_process_frame_index = ~0;
1424 : }
1425 :
1426 1666820 : t = clib_cpu_time_now ();
1427 1666820 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1428 : /* is_after */ 1);
1429 :
1430 1666820 : vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t,
1431 : VLIB_NODE_RUNTIME_PERF_AFTER);
1432 :
1433 1666820 : vlib_process_update_stats (vm, p,
1434 : /* n_calls */ !is_suspend,
1435 : /* n_vectors */ n_vectors,
1436 : /* n_clocks */ t - last_time_stamp);
1437 :
1438 1666820 : return t;
1439 : }
1440 :
1441 : void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1442 : void
1443 0 : vl_api_send_pending_rpc_requests (vlib_main_t * vm)
1444 : {
1445 0 : }
1446 :
1447 : static_always_inline void
1448 612 : vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
1449 : {
1450 612 : vlib_node_main_t *nm = &vm->node_main;
1451 612 : vlib_thread_main_t *tm = vlib_get_thread_main ();
1452 : uword i;
1453 : u64 cpu_time_now;
1454 : f64 now;
1455 : vlib_frame_queue_main_t *fqm;
1456 612 : u32 frame_queue_check_counter = 0;
1457 :
1458 : /* Initialize pending node vector. */
1459 612 : if (is_main)
1460 : {
1461 559 : vec_resize (nm->pending_frames, 32);
1462 559 : vec_set_len (nm->pending_frames, 0);
1463 : }
1464 :
1465 : /* Mark time of main loop start. */
1466 613 : if (is_main)
1467 : {
1468 559 : cpu_time_now = vm->clib_time.last_cpu_time;
1469 559 : vm->cpu_time_main_loop_start = cpu_time_now;
1470 : }
1471 : else
1472 54 : cpu_time_now = clib_cpu_time_now ();
1473 :
1474 : /* Pre-allocate interupt runtime indices and lock. */
1475 613 : vec_validate_aligned (nm->pending_interrupts, 0, CLIB_CACHE_LINE_BYTES);
1476 :
1477 : /* Pre-allocate expired nodes. */
1478 613 : if (!nm->polling_threshold_vector_length)
1479 613 : nm->polling_threshold_vector_length = 10;
1480 613 : if (!nm->interrupt_threshold_vector_length)
1481 613 : nm->interrupt_threshold_vector_length = 5;
1482 :
1483 613 : vm->cpu_id = clib_get_current_cpu_id ();
1484 612 : vm->numa_node = clib_get_current_numa_node ();
1485 613 : os_set_numa_index (vm->numa_node);
1486 :
1487 : /* Start all processes. */
1488 613 : if (is_main)
1489 : {
1490 : uword i;
1491 :
1492 : /*
1493 : * Perform an initial barrier sync. Pays no attention to
1494 : * the barrier sync hold-down timer scheme, which won't work
1495 : * at this point in time.
1496 : */
1497 559 : vlib_worker_thread_initial_barrier_sync_and_release (vm);
1498 :
1499 559 : nm->current_process_index = ~0;
1500 26832 : for (i = 0; i < vec_len (nm->processes); i++)
1501 26273 : cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1502 : cpu_time_now);
1503 : }
1504 :
1505 : while (1)
1506 552728000 : {
1507 : vlib_node_runtime_t *n;
1508 :
1509 552729000 : if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1510 : {
1511 176 : if (!is_main)
1512 169 : vl_api_send_pending_rpc_requests (vm);
1513 : }
1514 :
1515 552323000 : if (!is_main)
1516 26596100 : vlib_worker_thread_barrier_check ();
1517 :
1518 552434000 : if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter))
1519 : {
1520 51914 : u32 processed = 0;
1521 : vlib_frame_queue_dequeue_fn_t *fn;
1522 :
1523 51914 : if (vm->check_frame_queues)
1524 : {
1525 555 : frame_queue_check_counter = 100;
1526 555 : vm->check_frame_queues = 0;
1527 : }
1528 :
1529 2111040 : vec_foreach (fqm, tm->frame_queue_mains)
1530 : {
1531 2058580 : fn = fqm->frame_queue_dequeue_fn;
1532 2058580 : processed += (fn) (vm, fqm);
1533 : }
1534 :
1535 : /* No handoff queue work found? */
1536 52160 : if (processed)
1537 578 : frame_queue_check_counter = 100;
1538 : else
1539 51582 : frame_queue_check_counter--;
1540 : }
1541 :
1542 552434000 : if (PREDICT_FALSE (vec_len (vm->worker_thread_main_loop_callbacks)))
1543 0 : clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm,
1544 : cpu_time_now);
1545 :
1546 : /* Process pre-input nodes. */
1547 552434000 : cpu_time_now = clib_cpu_time_now ();
1548 2204220000 : vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])
1549 1651700000 : cpu_time_now = dispatch_node (vm, n,
1550 : VLIB_NODE_TYPE_PRE_INPUT,
1551 : VLIB_NODE_STATE_POLLING,
1552 : /* frame */ 0,
1553 : cpu_time_now);
1554 :
1555 : /* Next process input nodes. */
1556 15220700000 : vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
1557 14667100000 : cpu_time_now = dispatch_node (vm, n,
1558 : VLIB_NODE_TYPE_INPUT,
1559 : VLIB_NODE_STATE_POLLING,
1560 : /* frame */ 0,
1561 : cpu_time_now);
1562 :
1563 551797000 : if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1564 520735000 : vm->queue_signal_callback (vm);
1565 :
1566 553679000 : if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE))
1567 : {
1568 388941 : int int_num = -1;
1569 388941 : *nm->pending_interrupts = 0;
1570 :
1571 1167040 : while ((int_num =
1572 777983 : clib_interrupt_get_next (nm->interrupts, int_num)) != -1)
1573 : {
1574 : vlib_node_runtime_t *n;
1575 389100 : clib_interrupt_clear (nm->interrupts, int_num);
1576 389099 : n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
1577 : int_num);
1578 389092 : cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,
1579 : VLIB_NODE_STATE_INTERRUPT,
1580 : /* frame */ 0, cpu_time_now);
1581 : }
1582 : }
1583 :
1584 : /* Input nodes may have added work to the pending vector.
1585 : Process pending vector until there is nothing left.
1586 : All pending vectors will be processed from input -> output. */
1587 567450000 : for (i = 0; i < _vec_len (nm->pending_frames); i++)
1588 13770200 : cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1589 : /* Reset pending vector for next iteration. */
1590 553631000 : vec_set_len (nm->pending_frames, 0);
1591 :
1592 552447000 : if (is_main)
1593 : {
1594 : /* *INDENT-OFF* */
1595 : ELOG_TYPE_DECLARE (es) =
1596 : {
1597 : .format = "process tw start",
1598 : .format_args = "",
1599 : };
1600 : ELOG_TYPE_DECLARE (ee) =
1601 : {
1602 : .format = "process tw end: %d",
1603 : .format_args = "i4",
1604 : };
1605 : /* *INDENT-ON* */
1606 :
1607 : struct
1608 : {
1609 : int nready_procs;
1610 : } *ed;
1611 :
1612 : /* Check if process nodes have expired from timing wheel. */
1613 525951000 : ASSERT (nm->data_from_advancing_timing_wheel != 0);
1614 :
1615 525951000 : if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
1616 0 : ed = ELOG_DATA (&vlib_global_main.elog_main, es);
1617 :
1618 525951000 : TW (tw_timer_expire_timers)
1619 525951000 : ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm));
1620 :
1621 525951000 : ASSERT (nm->data_from_advancing_timing_wheel != 0);
1622 :
1623 525951000 : if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
1624 : {
1625 0 : ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
1626 0 : ed->nready_procs =
1627 0 : _vec_len (nm->data_from_advancing_timing_wheel);
1628 : }
1629 :
1630 525951000 : if (PREDICT_FALSE
1631 : (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1632 : {
1633 : uword i;
1634 :
1635 3157340 : for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1636 1666820 : i++)
1637 : {
1638 1666820 : u32 d = nm->data_from_advancing_timing_wheel[i];
1639 1666820 : u32 di = vlib_timing_wheel_data_get_index (d);
1640 :
1641 1666820 : if (vlib_timing_wheel_data_is_timed_event (d))
1642 : {
1643 0 : vlib_signal_timed_event_data_t *te =
1644 0 : pool_elt_at_index (nm->signal_timed_event_data_pool,
1645 : di);
1646 : vlib_node_t *n =
1647 0 : vlib_get_node (vm, te->process_node_index);
1648 0 : vlib_process_t *p =
1649 0 : vec_elt (nm->processes, n->runtime_index);
1650 0 : p->stop_timer_handle = ~0;
1651 : void *data;
1652 : data =
1653 0 : vlib_process_signal_event_helper (nm, n, p,
1654 0 : te->event_type_index,
1655 0 : te->n_data_elts,
1656 0 : te->n_data_elt_bytes);
1657 0 : if (te->n_data_bytes < sizeof (te->inline_event_data))
1658 0 : clib_memcpy_fast (data, te->inline_event_data,
1659 0 : te->n_data_bytes);
1660 : else
1661 : {
1662 0 : clib_memcpy_fast (data, te->event_data_as_vector,
1663 0 : te->n_data_bytes);
1664 0 : vec_free (te->event_data_as_vector);
1665 : }
1666 0 : pool_put (nm->signal_timed_event_data_pool, te);
1667 : }
1668 : else
1669 : {
1670 1666820 : cpu_time_now = clib_cpu_time_now ();
1671 : cpu_time_now =
1672 1666820 : dispatch_suspended_process (vm, di, cpu_time_now);
1673 : }
1674 : }
1675 1490520 : vec_set_len (nm->data_from_advancing_timing_wheel, 0);
1676 : }
1677 : }
1678 552447000 : vlib_increment_main_loop_counter (vm);
1679 : /* Record time stamp in case there are no enabled nodes and above
1680 : calls do not update time stamp. */
1681 552507000 : cpu_time_now = clib_cpu_time_now ();
1682 552332000 : vm->loops_this_reporting_interval++;
1683 552332000 : now = clib_time_now_internal (&vm->clib_time, cpu_time_now);
1684 : /* Time to update loops_per_second? */
1685 552728000 : if (PREDICT_FALSE (now >= vm->loop_interval_end))
1686 : {
1687 : /* Next sample ends in 20ms */
1688 7480160 : if (vm->loop_interval_start)
1689 : {
1690 : f64 this_loops_per_second;
1691 :
1692 7451000 : this_loops_per_second =
1693 7451000 : ((f64) vm->loops_this_reporting_interval) / (now -
1694 7451000 : vm->loop_interval_start);
1695 :
1696 7451000 : vm->loops_per_second =
1697 7451000 : vm->loops_per_second * vm->damping_constant +
1698 7451000 : (1.0 - vm->damping_constant) * this_loops_per_second;
1699 7451000 : if (vm->loops_per_second != 0.0)
1700 7451620 : vm->seconds_per_loop = 1.0 / vm->loops_per_second;
1701 : else
1702 0 : vm->seconds_per_loop = 0.0;
1703 : }
1704 : /* New interval starts now, and ends in 20ms */
1705 7480160 : vm->loop_interval_start = now;
1706 7480160 : vm->loop_interval_end = now + 2e-4;
1707 7480160 : vm->loops_this_reporting_interval = 0;
1708 : }
1709 : }
1710 : }
1711 :
1712 : static void
1713 559 : vlib_main_loop (vlib_main_t * vm)
1714 : {
1715 559 : vlib_main_or_worker_loop (vm, /* is_main */ 1);
1716 0 : }
1717 :
1718 : void
1719 54 : vlib_worker_loop (vlib_main_t * vm)
1720 : {
1721 54 : vlib_main_or_worker_loop (vm, /* is_main */ 0);
1722 0 : }
1723 :
1724 : vlib_global_main_t vlib_global_main;
1725 :
1726 : void
1727 1 : vlib_add_del_post_mortem_callback (void *cb, int is_add)
1728 : {
1729 1 : vlib_global_main_t *vgm = vlib_get_global_main ();
1730 : int i;
1731 :
1732 1 : if (is_add == 0)
1733 : {
1734 1 : for (i = vec_len (vgm->post_mortem_callbacks) - 1; i >= 0; i--)
1735 0 : if (vgm->post_mortem_callbacks[i] == cb)
1736 0 : vec_del1 (vgm->post_mortem_callbacks, i);
1737 1 : return;
1738 : }
1739 :
1740 0 : for (i = 0; i < vec_len (vgm->post_mortem_callbacks); i++)
1741 0 : if (vgm->post_mortem_callbacks[i] == cb)
1742 0 : return;
1743 0 : vec_add1 (vgm->post_mortem_callbacks, cb);
1744 : }
1745 :
1746 : static void
1747 0 : elog_post_mortem_dump (void)
1748 : {
1749 0 : elog_main_t *em = vlib_get_elog_main ();
1750 :
1751 : u8 *filename;
1752 : clib_error_t *error;
1753 :
1754 0 : filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
1755 0 : error = elog_write_file (em, (char *) filename, 1 /* flush ring */);
1756 0 : if (error)
1757 0 : clib_error_report (error);
1758 : /*
1759 : * We're in the middle of crashing. Don't try to free the filename.
1760 : */
1761 0 : }
1762 :
1763 : static clib_error_t *
1764 559 : vlib_main_configure (vlib_main_t * vm, unformat_input_t * input)
1765 : {
1766 559 : vlib_global_main_t *vgm = vlib_get_global_main ();
1767 559 : int turn_on_mem_trace = 0;
1768 :
1769 559 : while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1770 : {
1771 0 : if (unformat (input, "memory-trace"))
1772 0 : turn_on_mem_trace = 1;
1773 :
1774 0 : else if (unformat (input, "elog-events %d",
1775 : &vgm->configured_elog_ring_size))
1776 0 : vgm->configured_elog_ring_size =
1777 0 : 1 << max_log2 (vgm->configured_elog_ring_size);
1778 0 : else if (unformat (input, "elog-post-mortem-dump"))
1779 0 : vlib_add_del_post_mortem_callback (elog_post_mortem_dump,
1780 : /* is_add */ 1);
1781 0 : else if (unformat (input, "buffer-alloc-success-rate %f",
1782 : &vm->buffer_alloc_success_rate))
1783 : {
1784 : if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0)
1785 0 : return clib_error_return
1786 : (0, "Buffer fault injection not configured");
1787 : }
1788 0 : else if (unformat (input, "buffer-alloc-success-seed %u",
1789 : &vm->buffer_alloc_success_seed))
1790 : {
1791 : if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0)
1792 0 : return clib_error_return
1793 : (0, "Buffer fault injection not configured");
1794 : }
1795 : else
1796 0 : return unformat_parse_error (input);
1797 : }
1798 :
1799 559 : unformat_free (input);
1800 :
1801 : /* Enable memory trace as early as possible. */
1802 559 : if (turn_on_mem_trace)
1803 0 : clib_mem_trace (1);
1804 :
1805 559 : return 0;
1806 : }
1807 :
1808 7306 : VLIB_EARLY_CONFIG_FUNCTION (vlib_main_configure, "vlib");
1809 :
1810 : static void
1811 0 : placeholder_queue_signal_callback (vlib_main_t * vm)
1812 : {
1813 0 : }
1814 :
1815 : #define foreach_weak_reference_stub \
1816 : _(vpe_api_init) \
1817 : _(vlibmemory_init) \
1818 : _(map_api_segment_init)
1819 :
1820 : #define _(name) \
1821 : clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
1822 : clib_error_t *name (vlib_main_t *vm) { return 0; }
1823 0 : foreach_weak_reference_stub;
1824 : #undef _
1825 :
1826 : void vl_api_set_elog_main (elog_main_t * m) __attribute__ ((weak));
1827 : void
1828 0 : vl_api_set_elog_main (elog_main_t * m)
1829 : {
1830 0 : clib_warning ("STUB");
1831 0 : }
1832 :
1833 : int vl_api_set_elog_trace_api_messages (int enable) __attribute__ ((weak));
1834 : int
1835 0 : vl_api_set_elog_trace_api_messages (int enable)
1836 : {
1837 0 : clib_warning ("STUB");
1838 0 : return 0;
1839 : }
1840 :
1841 : int vl_api_get_elog_trace_api_messages (void) __attribute__ ((weak));
1842 : int
1843 0 : vl_api_get_elog_trace_api_messages (void)
1844 : {
1845 0 : clib_warning ("STUB");
1846 0 : return 0;
1847 : }
1848 :
1849 : static void
1850 1071250 : process_expired_timer_cb (u32 *expired_timer_handles)
1851 : {
1852 1071250 : vlib_main_t *vm = vlib_get_main ();
1853 1071250 : vlib_node_main_t *nm = &vm->node_main;
1854 : u32 *handle;
1855 :
1856 2289750 : vec_foreach (handle, expired_timer_handles)
1857 : {
1858 1218500 : u32 pi = vlib_timing_wheel_data_get_index (*handle);
1859 1218500 : vlib_process_t *p = vec_elt (nm->processes, pi);
1860 :
1861 1218500 : p->stop_timer_handle = ~0;
1862 : }
1863 1071250 : vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles);
1864 1071250 : }
1865 :
1866 : /* Main function. */
1867 : int
1868 559 : vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
1869 : {
1870 559 : vlib_global_main_t *vgm = vlib_get_global_main ();
1871 : clib_error_t *volatile error;
1872 559 : vlib_node_main_t *nm = &vm->node_main;
1873 :
1874 559 : vm->queue_signal_callback = placeholder_queue_signal_callback;
1875 :
1876 : /* Reconfigure event log which is enabled very early */
1877 559 : if (vgm->configured_elog_ring_size &&
1878 559 : vgm->configured_elog_ring_size != vgm->elog_main.event_ring_size)
1879 0 : elog_resize (&vgm->elog_main, vgm->configured_elog_ring_size);
1880 559 : vl_api_set_elog_main (vlib_get_elog_main ());
1881 559 : (void) vl_api_set_elog_trace_api_messages (1);
1882 :
1883 : /* Default name. */
1884 559 : if (!vgm->name)
1885 0 : vgm->name = "VLIB";
1886 :
1887 559 : if ((error = vlib_physmem_init (vm)))
1888 : {
1889 0 : clib_error_report (error);
1890 0 : goto done;
1891 : }
1892 :
1893 559 : if ((error = vlib_log_init (vm)))
1894 : {
1895 0 : clib_error_report (error);
1896 0 : goto done;
1897 : }
1898 :
1899 559 : if ((error = vlib_stats_init (vm)))
1900 : {
1901 0 : clib_error_report (error);
1902 0 : goto done;
1903 : }
1904 :
1905 559 : if ((error = vlib_buffer_main_init (vm)))
1906 : {
1907 0 : clib_error_report (error);
1908 0 : goto done;
1909 : }
1910 :
1911 559 : if ((error = vlib_thread_init (vm)))
1912 : {
1913 0 : clib_error_report (error);
1914 0 : goto done;
1915 : }
1916 :
1917 : /* Register node ifunction variants */
1918 559 : vlib_register_all_node_march_variants (vm);
1919 :
1920 : /* Register static nodes so that init functions may use them. */
1921 559 : vlib_register_all_static_nodes (vm);
1922 :
1923 : /* Set seed for random number generator.
1924 : Allow user to specify seed to make random sequence deterministic. */
1925 559 : if (!unformat (input, "seed %wd", &vm->random_seed))
1926 559 : vm->random_seed = clib_cpu_time_now ();
1927 559 : clib_random_buffer_init (&vm->random_buffer, vm->random_seed);
1928 :
1929 : /* Initialize node graph. */
1930 559 : if ((error = vlib_node_main_init (vm)))
1931 : {
1932 : /* Arrange for graph hook up error to not be fatal when debugging. */
1933 : if (CLIB_DEBUG > 0)
1934 0 : clib_error_report (error);
1935 : else
1936 : goto done;
1937 : }
1938 :
1939 : /* Direct call / weak reference, for vlib standalone use-cases */
1940 559 : if ((error = vpe_api_init (vm)))
1941 : {
1942 0 : clib_error_report (error);
1943 0 : goto done;
1944 : }
1945 :
1946 559 : if ((error = vlibmemory_init (vm)))
1947 : {
1948 0 : clib_error_report (error);
1949 0 : goto done;
1950 : }
1951 :
1952 559 : if ((error = map_api_segment_init (vm)))
1953 : {
1954 0 : clib_error_report (error);
1955 0 : goto done;
1956 : }
1957 :
1958 : /* See unix/main.c; most likely already set up */
1959 559 : if (vgm->init_functions_called == 0)
1960 0 : vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
1961 559 : if ((error = vlib_call_all_init_functions (vm)))
1962 0 : goto done;
1963 :
1964 559 : nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
1965 : CLIB_CACHE_LINE_BYTES);
1966 :
1967 559 : vec_validate (nm->data_from_advancing_timing_wheel, 10);
1968 559 : vec_set_len (nm->data_from_advancing_timing_wheel, 0);
1969 :
1970 : /* Create the process timing wheel */
1971 559 : TW (tw_timer_wheel_init)
1972 559 : ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1973 : process_expired_timer_cb /* callback */, 10e-6 /* timer period 10us */,
1974 : ~0 /* max expirations per call */);
1975 :
1976 559 : vec_validate (vm->pending_rpc_requests, 0);
1977 559 : vec_set_len (vm->pending_rpc_requests, 0);
1978 559 : vec_validate (vm->processing_rpc_requests, 0);
1979 559 : vec_set_len (vm->processing_rpc_requests, 0);
1980 :
1981 : /* Default params for the buffer allocator fault injector, if configured */
1982 : if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
1983 : {
1984 : vm->buffer_alloc_success_seed = 0xdeaddabe;
1985 : vm->buffer_alloc_success_rate = 0.80;
1986 : }
1987 :
1988 559 : if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
1989 0 : goto done;
1990 :
1991 : /*
1992 : * Use exponential smoothing, with a half-life of 1 second
1993 : * reported_rate(t) = reported_rate(t-1) * K + rate(t)*(1-K)
1994 : *
1995 : * Sample every 20ms, aka 50 samples per second
1996 : * K = exp (-1.0/20.0);
1997 : * K = 0.95
1998 : */
1999 559 : vm->damping_constant = exp (-1.0 / 20.0);
2000 :
2001 : /* Sort per-thread init functions before we start threads */
2002 559 : vlib_sort_init_exit_functions (&vgm->worker_init_function_registrations);
2003 :
2004 : /* Call all main loop enter functions. */
2005 : {
2006 : clib_error_t *sub_error;
2007 559 : sub_error = vlib_call_all_main_loop_enter_functions (vm);
2008 559 : if (sub_error)
2009 0 : clib_error_report (sub_error);
2010 : }
2011 :
2012 559 : switch (clib_setjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_NONE))
2013 : {
2014 559 : case VLIB_MAIN_LOOP_EXIT_NONE:
2015 559 : vm->main_loop_exit_set = 1;
2016 559 : break;
2017 :
2018 559 : case VLIB_MAIN_LOOP_EXIT_CLI:
2019 559 : goto done;
2020 :
2021 0 : default:
2022 0 : error = vm->main_loop_error;
2023 0 : goto done;
2024 : }
2025 :
2026 559 : vlib_main_loop (vm);
2027 :
2028 559 : done:
2029 : /* Stop worker threads, barrier will not be released */
2030 559 : vlib_worker_thread_barrier_sync (vm);
2031 :
2032 : /* Call all exit functions. */
2033 : {
2034 : clib_error_t *sub_error;
2035 559 : sub_error = vlib_call_all_main_loop_exit_functions (vm);
2036 559 : if (sub_error)
2037 0 : clib_error_report (sub_error);
2038 : }
2039 :
2040 559 : if (error)
2041 0 : clib_error_report (error);
2042 :
2043 559 : return vm->main_loop_exit_status;
2044 : }
2045 :
2046 : vlib_main_t *
2047 0 : vlib_get_main_not_inline (void)
2048 : {
2049 0 : return vlib_get_main ();
2050 : }
2051 :
2052 : elog_main_t *
2053 0 : vlib_get_elog_main_not_inline ()
2054 : {
2055 0 : return &vlib_global_main.elog_main;
2056 : }
2057 :
2058 : void
2059 0 : vlib_exit_with_status (vlib_main_t *vm, int status)
2060 : {
2061 0 : vm->main_loop_exit_status = status;
2062 0 : __atomic_store_n (&vm->main_loop_exit_now, 1, __ATOMIC_RELEASE);
2063 0 : }
2064 :
2065 : /*
2066 : * fd.io coding-style-patch-verification: ON
2067 : *
2068 : * Local Variables:
2069 : * eval: (c-set-style "gnu")
2070 : * End:
2071 : */
|
