Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * main.c: main vector processing loop
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #include <math.h>
41 : #include <vppinfra/format.h>
42 : #include <vlib/vlib.h>
43 : #include <vlib/threads.h>
44 : #include <vlib/stats/stats.h>
45 : #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
46 :
47 : #include <vlib/unix/unix.h>
48 :
49 : #define VLIB_FRAME_MAGIC (0xabadc0ed)
50 :
51 : always_inline u32 *
52 35757800 : vlib_frame_find_magic (vlib_frame_t * f, vlib_node_t * node)
53 : {
54 35757800 : return (void *) f + node->magic_offset;
55 : }
56 :
57 : static vlib_frame_t *
58 773244 : vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index,
59 : u32 frame_flags)
60 : {
61 773244 : vlib_node_main_t *nm = &vm->node_main;
62 : vlib_frame_size_t *fs;
63 : vlib_node_t *to_node;
64 : vlib_frame_t *f;
65 : u32 l, n;
66 :
67 773244 : ASSERT (vm == vlib_get_main ());
68 :
69 773241 : to_node = vlib_get_node (vm, to_node_index);
70 :
71 773240 : vec_validate (nm->frame_sizes, to_node->frame_size_index);
72 773238 : fs = vec_elt_at_index (nm->frame_sizes, to_node->frame_size_index);
73 :
74 773238 : if (fs->frame_size == 0)
75 34 : fs->frame_size = to_node->frame_size;
76 : else
77 773204 : ASSERT (fs->frame_size == to_node->frame_size);
78 :
79 773240 : n = fs->frame_size;
80 773240 : if ((l = vec_len (fs->free_frames)) > 0)
81 : {
82 : /* Allocate from end of free list. */
83 756644 : f = fs->free_frames[l - 1];
84 756644 : vec_set_len (fs->free_frames, l - 1);
85 : }
86 : else
87 : {
88 16596 : f = clib_mem_alloc_aligned_no_fail (n, CLIB_CACHE_LINE_BYTES);
89 : }
90 :
91 : /* Poison frame when debugging. */
92 : if (CLIB_DEBUG > 0)
93 773241 : clib_memset_u8 (f, 0xfe, n);
94 :
95 : /* Insert magic number. */
96 : {
97 : u32 *magic;
98 :
99 773248 : magic = vlib_frame_find_magic (f, to_node);
100 773248 : *magic = VLIB_FRAME_MAGIC;
101 : }
102 :
103 773248 : f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
104 773248 : f->n_vectors = 0;
105 773248 : f->scalar_offset = to_node->scalar_offset;
106 773248 : f->vector_offset = to_node->vector_offset;
107 773248 : f->aux_offset = to_node->aux_offset;
108 773248 : f->flags = 0;
109 773248 : f->frame_size_index = to_node->frame_size_index;
110 :
111 773248 : fs->n_alloc_frames += 1;
112 :
113 773248 : return f;
114 : }
115 :
116 : /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
117 : Returns frame index. */
118 : static vlib_frame_t *
119 585456 : vlib_frame_alloc (vlib_main_t * vm, vlib_node_runtime_t * from_node_runtime,
120 : u32 to_next_index)
121 : {
122 : vlib_node_t *from_node;
123 :
124 585456 : from_node = vlib_get_node (vm, from_node_runtime->node_index);
125 585456 : ASSERT (to_next_index < vec_len (from_node->next_nodes));
126 :
127 585456 : return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
128 : /* frame_flags */ 0);
129 : }
130 :
131 : vlib_frame_t *
132 187788 : vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index)
133 : {
134 187788 : vlib_frame_t *f = vlib_frame_alloc_to_node (vm, to_node_index,
135 : /* frame_flags */
136 : VLIB_FRAME_FREE_AFTER_DISPATCH);
137 187791 : return vlib_get_frame (vm, f);
138 : }
139 :
140 : static inline void
141 17767300 : vlib_validate_frame_indices (vlib_frame_t * f)
142 : {
143 : if (CLIB_DEBUG > 0)
144 : {
145 : int i;
146 17767300 : u32 *from = vlib_frame_vector_args (f);
147 :
148 : /* Check for bad buffer index values */
149 60832100 : for (i = 0; i < f->n_vectors; i++)
150 : {
151 43064800 : if (from[i] == 0)
152 : {
153 0 : clib_warning ("BUG: buffer index 0 at index %d", i);
154 0 : ASSERT (0);
155 : }
156 43064800 : else if (from[i] == 0xfefefefe)
157 : {
158 0 : clib_warning ("BUG: frame poison pattern at index %d", i);
159 0 : ASSERT (0);
160 : }
161 : }
162 : }
163 17767300 : }
164 :
165 : void
166 187687 : vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f)
167 : {
168 : vlib_pending_frame_t *p;
169 : vlib_node_t *to_node;
170 :
171 187687 : if (f->n_vectors == 0)
172 0 : return;
173 :
174 187687 : ASSERT (vm == vlib_get_main ());
175 :
176 187687 : vlib_validate_frame_indices (f);
177 :
178 187686 : to_node = vlib_get_node (vm, to_node_index);
179 :
180 187686 : vec_add2 (vm->node_main.pending_frames, p, 1);
181 :
182 187683 : f->frame_flags |= VLIB_FRAME_PENDING;
183 187683 : p->frame = vlib_get_frame (vm, f);
184 187683 : p->node_runtime_index = to_node->runtime_index;
185 187683 : p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME;
186 : }
187 :
188 : /* Free given frame. */
189 : void
190 757654 : vlib_frame_free (vlib_main_t *vm, vlib_frame_t *f)
191 : {
192 757654 : vlib_node_main_t *nm = &vm->node_main;
193 : vlib_frame_size_t *fs;
194 :
195 757654 : ASSERT (vm == vlib_get_main ());
196 757654 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
197 :
198 757656 : fs = vec_elt_at_index (nm->frame_sizes, f->frame_size_index);
199 :
200 757656 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
201 :
202 : /* No next frames may point to freed frame. */
203 : if (CLIB_DEBUG > 0)
204 : {
205 : vlib_next_frame_t *nf;
206 1763480000 : vec_foreach (nf, vm->node_main.next_frames) ASSERT (nf->frame != f);
207 : }
208 :
209 756952 : f->frame_flags &= ~(VLIB_FRAME_IS_ALLOCATED | VLIB_FRAME_NO_APPEND);
210 756952 : f->flags = 0;
211 :
212 756952 : vec_add1 (fs->free_frames, f);
213 757659 : ASSERT (fs->n_alloc_frames > 0);
214 757659 : fs->n_alloc_frames -= 1;
215 757659 : }
216 :
217 : static clib_error_t *
218 2 : show_frame_stats (vlib_main_t * vm,
219 : unformat_input_t * input, vlib_cli_command_t * cmd)
220 : {
221 : vlib_frame_size_t *fs;
222 :
223 2 : vlib_cli_output (vm, "%=8s%=6s%=12s%=12s", "Thread", "Size", "# Alloc",
224 : "# Free");
225 6 : foreach_vlib_main ()
226 : {
227 4 : vlib_node_main_t *nm = &this_vlib_main->node_main;
228 14 : vec_foreach (fs, nm->frame_sizes)
229 : {
230 10 : u32 n_alloc = fs->n_alloc_frames;
231 10 : u32 n_free = vec_len (fs->free_frames);
232 :
233 10 : if (n_alloc + n_free > 0)
234 2 : vlib_cli_output (vm, "%=8d%=6d%=12d%=12d",
235 2 : this_vlib_main->thread_index, fs->frame_size,
236 : n_alloc, n_free);
237 : }
238 : }
239 :
240 2 : return 0;
241 : }
242 :
243 : /* *INDENT-OFF* */
244 285289 : VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
245 : .path = "show vlib frame-allocation",
246 : .short_help = "Show node dispatch frame statistics",
247 : .function = show_frame_stats,
248 : };
249 : /* *INDENT-ON* */
250 :
251 : /* Change ownership of enqueue rights to given next node. */
252 : static void
253 271872 : vlib_next_frame_change_ownership (vlib_main_t * vm,
254 : vlib_node_runtime_t * node_runtime,
255 : u32 next_index)
256 : {
257 271872 : vlib_node_main_t *nm = &vm->node_main;
258 : vlib_next_frame_t *next_frame;
259 : vlib_node_t *node, *next_node;
260 :
261 271872 : node = vec_elt (nm->nodes, node_runtime->node_index);
262 :
263 : /* Only internal & input nodes are allowed to call other nodes. */
264 271872 : ASSERT (node->type == VLIB_NODE_TYPE_INTERNAL
265 : || node->type == VLIB_NODE_TYPE_INPUT
266 : || node->type == VLIB_NODE_TYPE_PROCESS);
267 :
268 271872 : ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
269 :
270 : next_frame =
271 271872 : vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
272 271870 : next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
273 :
274 271872 : if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
275 : {
276 : /* Get frame from previous owner. */
277 : vlib_next_frame_t *owner_next_frame;
278 : vlib_next_frame_t tmp;
279 :
280 : owner_next_frame =
281 254914 : vlib_node_get_next_frame (vm,
282 : next_node->owner_node_index,
283 : next_node->owner_next_index);
284 :
285 : /* Swap target next frame with owner's. */
286 254914 : tmp = owner_next_frame[0];
287 254914 : owner_next_frame[0] = next_frame[0];
288 254914 : next_frame[0] = tmp;
289 :
290 : /*
291 : * If next_frame is already pending, we have to track down
292 : * all pending frames and fix their next_frame_index fields.
293 : */
294 254914 : if (next_frame->flags & VLIB_FRAME_PENDING)
295 : {
296 : vlib_pending_frame_t *p;
297 247858 : if (next_frame->frame != NULL)
298 : {
299 818976 : vec_foreach (p, nm->pending_frames)
300 : {
301 571157 : if (p->frame == next_frame->frame)
302 : {
303 86686 : p->next_frame_index =
304 86686 : next_frame - vm->node_main.next_frames;
305 : }
306 : }
307 : }
308 : }
309 : }
310 : else
311 : {
312 : /* No previous owner. Take ownership. */
313 16958 : next_frame->flags |= VLIB_FRAME_OWNER;
314 : }
315 :
316 : /* Record new owner. */
317 271872 : next_node->owner_node_index = node->index;
318 271872 : next_node->owner_next_index = next_index;
319 :
320 : /* Now we should be owner. */
321 271872 : ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
322 271872 : }
323 :
324 : /* Make sure that magic number is still there.
325 : Otherwise, it is likely that caller has overrun frame arguments. */
326 : always_inline void
327 34984500 : validate_frame_magic (vlib_main_t * vm,
328 : vlib_frame_t * f, vlib_node_t * n, uword next_index)
329 : {
330 34984500 : vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
331 34984500 : u32 *magic = vlib_frame_find_magic (f, next_node);
332 34984500 : ASSERT (VLIB_FRAME_MAGIC == magic[0]);
333 34984500 : }
334 :
335 : vlib_frame_t *
336 17404900 : vlib_get_next_frame_internal (vlib_main_t * vm,
337 : vlib_node_runtime_t * node,
338 : u32 next_index, u32 allocate_new_next_frame)
339 : {
340 : vlib_frame_t *f;
341 : vlib_next_frame_t *nf;
342 : u32 n_used;
343 :
344 17404900 : nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
345 :
346 : /* Make sure this next frame owns right to enqueue to destination frame. */
347 17404900 : if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
348 271872 : vlib_next_frame_change_ownership (vm, node, next_index);
349 :
350 : /* ??? Don't need valid flag: can use frame_index == ~0 */
351 17404900 : if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_IS_ALLOCATED)))
352 : {
353 17315 : nf->frame = vlib_frame_alloc (vm, node, next_index);
354 17316 : nf->flags |= VLIB_FRAME_IS_ALLOCATED;
355 : }
356 :
357 17404900 : f = nf->frame;
358 :
359 : /* Has frame been removed from pending vector (e.g. finished dispatching)?
360 : If so we can reuse frame. */
361 17404900 : if ((nf->flags & VLIB_FRAME_PENDING)
362 17045300 : && !(f->frame_flags & VLIB_FRAME_PENDING))
363 : {
364 14935700 : nf->flags &= ~VLIB_FRAME_PENDING;
365 14935700 : f->n_vectors = 0;
366 14935700 : f->flags = 0;
367 : }
368 :
369 : /* Allocate new frame if current one is marked as no-append or
370 : it is already full. */
371 17404900 : n_used = f->n_vectors;
372 17404900 : if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) ||
373 16836800 : (f->frame_flags & VLIB_FRAME_NO_APPEND))
374 : {
375 : /* Old frame may need to be freed after dispatch, since we'll have
376 : two redundant frames from node -> next node. */
377 568140 : if (!(nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH))
378 : {
379 568141 : vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame);
380 568141 : f_old->frame_flags |= VLIB_FRAME_FREE_AFTER_DISPATCH;
381 : }
382 :
383 : /* Allocate new frame to replace full one. */
384 568140 : f = nf->frame = vlib_frame_alloc (vm, node, next_index);
385 568141 : n_used = f->n_vectors;
386 : }
387 :
388 : /* Should have free vectors in frame now. */
389 17404900 : ASSERT (n_used < VLIB_FRAME_SIZE);
390 :
391 : if (CLIB_DEBUG > 0)
392 : {
393 17404900 : validate_frame_magic (vm, f,
394 : vlib_get_node (vm, node->node_index), next_index);
395 : }
396 :
397 17404900 : return f;
398 : }
399 :
400 : static void
401 17579600 : vlib_put_next_frame_validate (vlib_main_t * vm,
402 : vlib_node_runtime_t * rt,
403 : u32 next_index, u32 n_vectors_left)
404 : {
405 17579600 : vlib_node_main_t *nm = &vm->node_main;
406 : vlib_next_frame_t *nf;
407 : vlib_frame_t *f;
408 : vlib_node_runtime_t *next_rt;
409 : vlib_node_t *next_node;
410 : u32 n_before, n_after;
411 :
412 17579600 : nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
413 17579600 : f = vlib_get_frame (vm, nf->frame);
414 :
415 17579600 : ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
416 :
417 17579600 : vlib_validate_frame_indices (f);
418 :
419 17579600 : n_after = VLIB_FRAME_SIZE - n_vectors_left;
420 17579600 : n_before = f->n_vectors;
421 :
422 17579600 : ASSERT (n_after >= n_before);
423 :
424 17579600 : next_rt = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
425 : nf->node_runtime_index);
426 17579600 : next_node = vlib_get_node (vm, next_rt->node_index);
427 17579600 : if (n_after > 0 && next_node->validate_frame)
428 : {
429 4370 : u8 *msg = next_node->validate_frame (vm, rt, f);
430 4370 : if (msg)
431 : {
432 0 : clib_warning ("%v", msg);
433 0 : ASSERT (0);
434 : }
435 4370 : vec_free (msg);
436 : }
437 17579600 : }
438 :
439 : void
440 17579600 : vlib_put_next_frame (vlib_main_t * vm,
441 : vlib_node_runtime_t * r,
442 : u32 next_index, u32 n_vectors_left)
443 : {
444 17579600 : vlib_node_main_t *nm = &vm->node_main;
445 : vlib_next_frame_t *nf;
446 : vlib_frame_t *f;
447 : u32 n_vectors_in_frame;
448 :
449 : if (CLIB_DEBUG > 0)
450 17579600 : vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
451 :
452 17579600 : nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
453 17579600 : f = vlib_get_frame (vm, nf->frame);
454 :
455 : /* Make sure that magic number is still there. Otherwise, caller
456 : has overrun frame meta data. */
457 : if (CLIB_DEBUG > 0)
458 : {
459 17579600 : vlib_node_t *node = vlib_get_node (vm, r->node_index);
460 17579600 : validate_frame_magic (vm, f, node, next_index);
461 : }
462 :
463 : /* Convert # of vectors left -> number of vectors there. */
464 17579600 : ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
465 17579600 : n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
466 :
467 17579600 : f->n_vectors = n_vectors_in_frame;
468 :
469 : /* If vectors were added to frame, add to pending vector. */
470 17579600 : if (PREDICT_TRUE (n_vectors_in_frame > 0))
471 : {
472 : vlib_pending_frame_t *p;
473 : u32 v0, v1;
474 :
475 17233300 : r->cached_next_index = next_index;
476 :
477 17233300 : if (!(f->frame_flags & VLIB_FRAME_PENDING))
478 : {
479 : __attribute__ ((unused)) vlib_node_t *node;
480 :
481 15517000 : node = vlib_get_node (vm, r->node_index);
482 :
483 15517000 : vec_add2 (nm->pending_frames, p, 1);
484 :
485 15517000 : p->frame = nf->frame;
486 15517000 : p->node_runtime_index = nf->node_runtime_index;
487 15517000 : p->next_frame_index = nf - nm->next_frames;
488 15517000 : nf->flags |= VLIB_FRAME_PENDING;
489 15517000 : f->frame_flags |= VLIB_FRAME_PENDING;
490 : }
491 :
492 : /* Copy trace flag from next_frame and from runtime. */
493 17233300 : nf->flags |=
494 17233300 : (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
495 17233300 : flags & VLIB_NODE_FLAG_TRACE);
496 :
497 17233300 : v0 = nf->vectors_since_last_overflow;
498 17233300 : v1 = v0 + n_vectors_in_frame;
499 17233300 : nf->vectors_since_last_overflow = v1;
500 17233300 : if (PREDICT_FALSE (v1 < v0))
501 : {
502 0 : vlib_node_t *node = vlib_get_node (vm, r->node_index);
503 0 : vec_elt (node->n_vectors_by_next_node, next_index) += v0;
504 : }
505 : }
506 17579600 : }
507 :
508 : /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
509 : void
510 2663680 : vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r,
511 : uword n_calls, uword n_vectors,
512 : uword n_clocks)
513 : {
514 2663680 : n->stats_total.calls += n_calls + r->calls_since_last_overflow;
515 2663680 : n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
516 2663680 : n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
517 2663680 : n->stats_total.max_clock = r->max_clock;
518 2663680 : n->stats_total.max_clock_n = r->max_clock_n;
519 :
520 2663680 : r->calls_since_last_overflow = 0;
521 2663680 : r->vectors_since_last_overflow = 0;
522 2663680 : r->clocks_since_last_overflow = 0;
523 2663680 : }
524 :
525 : void
526 2663680 : vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r,
527 : uword n_calls, uword n_vectors, uword n_clocks)
528 : {
529 2663680 : vlib_node_t *n = vlib_get_node (vm, r->node_index);
530 2663680 : vlib_node_runtime_sync_stats_node (n, r, n_calls, n_vectors, n_clocks);
531 2663680 : }
532 :
533 : always_inline void __attribute__ ((unused))
534 : vlib_process_sync_stats (vlib_main_t * vm,
535 : vlib_process_t * p,
536 : uword n_calls, uword n_vectors, uword n_clocks)
537 : {
538 : vlib_node_runtime_t *rt = &p->node_runtime;
539 : vlib_node_t *n = vlib_get_node (vm, rt->node_index);
540 : vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks);
541 : n->stats_total.suspends += p->n_suspends;
542 : p->n_suspends = 0;
543 : }
544 :
545 : void
546 1917570 : vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n)
547 : {
548 : vlib_node_runtime_t *rt;
549 :
550 1917570 : if (n->type == VLIB_NODE_TYPE_PROCESS)
551 : {
552 : /* Nothing to do for PROCESS nodes except in main thread */
553 126859 : if (vm != vlib_get_first_main ())
554 18720 : return;
555 :
556 108139 : vlib_process_t *p = vlib_get_process_from_node (vm, n);
557 108139 : n->stats_total.suspends += p->n_suspends;
558 108139 : p->n_suspends = 0;
559 108139 : rt = &p->node_runtime;
560 : }
561 : else
562 1790710 : rt =
563 1790710 : vec_elt_at_index (vm->node_main.nodes_by_type[n->type],
564 : n->runtime_index);
565 :
566 1898850 : vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0);
567 :
568 : /* Sync up runtime next frame vector counters with main node structure. */
569 : {
570 : vlib_next_frame_t *nf;
571 : uword i;
572 8120160 : for (i = 0; i < rt->n_next_nodes; i++)
573 : {
574 6221310 : nf = vlib_node_runtime_get_next_frame (vm, rt, i);
575 6221310 : vec_elt (n->n_vectors_by_next_node, i) +=
576 6221310 : nf->vectors_since_last_overflow;
577 6221310 : nf->vectors_since_last_overflow = 0;
578 : }
579 : }
580 : }
581 :
582 : always_inline u32
583 797261000 : vlib_node_runtime_update_stats (vlib_main_t * vm,
584 : vlib_node_runtime_t * node,
585 : uword n_calls,
586 : uword n_vectors, uword n_clocks)
587 : {
588 : u32 ca0, ca1, v0, v1, cl0, cl1, r;
589 :
590 797261000 : cl0 = cl1 = node->clocks_since_last_overflow;
591 797261000 : ca0 = ca1 = node->calls_since_last_overflow;
592 797261000 : v0 = v1 = node->vectors_since_last_overflow;
593 :
594 797261000 : ca1 = ca0 + n_calls;
595 797261000 : v1 = v0 + n_vectors;
596 797261000 : cl1 = cl0 + n_clocks;
597 :
598 797261000 : node->calls_since_last_overflow = ca1;
599 797261000 : node->clocks_since_last_overflow = cl1;
600 797261000 : node->vectors_since_last_overflow = v1;
601 :
602 797261000 : node->max_clock_n = node->max_clock > n_clocks ?
603 : node->max_clock_n : n_vectors;
604 797261000 : node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
605 :
606 797261000 : r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
607 :
608 797353000 : if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0))
609 : {
610 9981 : node->calls_since_last_overflow = ca0;
611 9981 : node->clocks_since_last_overflow = cl0;
612 9981 : node->vectors_since_last_overflow = v0;
613 :
614 9981 : vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks);
615 : }
616 :
617 797348000 : return r;
618 : }
619 :
620 : always_inline void
621 2285260 : vlib_process_update_stats (vlib_main_t * vm,
622 : vlib_process_t * p,
623 : uword n_calls, uword n_vectors, uword n_clocks)
624 : {
625 2285260 : vlib_node_runtime_update_stats (vm, &p->node_runtime,
626 : n_calls, n_vectors, n_clocks);
627 2285260 : }
628 :
629 : static clib_error_t *
630 20 : vlib_cli_elog_clear (vlib_main_t * vm,
631 : unformat_input_t * input, vlib_cli_command_t * cmd)
632 : {
633 20 : elog_reset_buffer (&vlib_global_main.elog_main);
634 20 : return 0;
635 : }
636 :
637 : /* *INDENT-OFF* */
638 285289 : VLIB_CLI_COMMAND (elog_clear_cli, static) = {
639 : .path = "event-logger clear",
640 : .short_help = "Clear the event log",
641 : .function = vlib_cli_elog_clear,
642 : };
643 : /* *INDENT-ON* */
644 :
645 : #ifdef CLIB_UNIX
646 : static clib_error_t *
647 0 : elog_save_buffer (vlib_main_t * vm,
648 : unformat_input_t * input, vlib_cli_command_t * cmd)
649 : {
650 0 : elog_main_t *em = &vlib_global_main.elog_main;
651 : char *file, *chroot_file;
652 0 : clib_error_t *error = 0;
653 :
654 0 : if (!unformat (input, "%s", &file))
655 : {
656 0 : vlib_cli_output (vm, "expected file name, got `%U'",
657 : format_unformat_error, input);
658 0 : return 0;
659 : }
660 :
661 : /* It's fairly hard to get "../oopsie" through unformat; just in case */
662 0 : if (strstr (file, "..") || index (file, '/'))
663 : {
664 0 : vlib_cli_output (vm, "illegal characters in filename '%s'", file);
665 0 : return 0;
666 : }
667 :
668 0 : chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
669 :
670 0 : vec_free (file);
671 :
672 0 : vlib_cli_output (vm, "Saving %wd of %wd events to %s",
673 : elog_n_events_in_buffer (em),
674 : elog_buffer_capacity (em), chroot_file);
675 :
676 0 : vlib_worker_thread_barrier_sync (vm);
677 0 : error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
678 0 : vlib_worker_thread_barrier_release (vm);
679 0 : vec_free (chroot_file);
680 0 : return error;
681 : }
682 :
683 : void
684 0 : vlib_post_mortem_dump (void)
685 : {
686 0 : vlib_global_main_t *vgm = vlib_get_global_main ();
687 :
688 0 : for (int i = 0; i < vec_len (vgm->post_mortem_callbacks); i++)
689 0 : (vgm->post_mortem_callbacks[i]) ();
690 0 : }
691 :
692 : /* *INDENT-OFF* */
693 285289 : VLIB_CLI_COMMAND (elog_save_cli, static) = {
694 : .path = "event-logger save",
695 : .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
696 : .function = elog_save_buffer,
697 : };
698 : /* *INDENT-ON* */
699 :
700 : static clib_error_t *
701 0 : elog_stop (vlib_main_t * vm,
702 : unformat_input_t * input, vlib_cli_command_t * cmd)
703 : {
704 0 : elog_main_t *em = &vlib_global_main.elog_main;
705 :
706 0 : em->n_total_events_disable_limit = em->n_total_events;
707 :
708 0 : vlib_cli_output (vm, "Stopped the event logger...");
709 0 : return 0;
710 : }
711 :
712 : /* *INDENT-OFF* */
713 285289 : VLIB_CLI_COMMAND (elog_stop_cli, static) = {
714 : .path = "event-logger stop",
715 : .short_help = "Stop the event-logger",
716 : .function = elog_stop,
717 : };
718 : /* *INDENT-ON* */
719 :
720 : static clib_error_t *
721 0 : elog_restart (vlib_main_t * vm,
722 : unformat_input_t * input, vlib_cli_command_t * cmd)
723 : {
724 0 : elog_main_t *em = &vlib_global_main.elog_main;
725 :
726 0 : em->n_total_events_disable_limit = ~0;
727 :
728 0 : vlib_cli_output (vm, "Restarted the event logger...");
729 0 : return 0;
730 : }
731 :
732 : /* *INDENT-OFF* */
733 285289 : VLIB_CLI_COMMAND (elog_restart_cli, static) = {
734 : .path = "event-logger restart",
735 : .short_help = "Restart the event-logger",
736 : .function = elog_restart,
737 : };
738 : /* *INDENT-ON* */
739 :
740 : static clib_error_t *
741 0 : elog_resize_command_fn (vlib_main_t * vm,
742 : unformat_input_t * input, vlib_cli_command_t * cmd)
743 : {
744 0 : elog_main_t *em = &vlib_global_main.elog_main;
745 : u32 tmp;
746 :
747 : /* Stop the parade */
748 0 : elog_reset_buffer (em);
749 :
750 0 : if (unformat (input, "%d", &tmp))
751 : {
752 0 : elog_alloc (em, tmp);
753 0 : em->n_total_events_disable_limit = ~0;
754 : }
755 : else
756 0 : return clib_error_return (0, "Must specify how many events in the ring");
757 :
758 0 : vlib_cli_output (vm, "Resized ring and restarted the event logger...");
759 0 : return 0;
760 : }
761 :
762 : /* *INDENT-OFF* */
763 285289 : VLIB_CLI_COMMAND (elog_resize_cli, static) = {
764 : .path = "event-logger resize",
765 : .short_help = "event-logger resize <nnn>",
766 : .function = elog_resize_command_fn,
767 : };
768 : /* *INDENT-ON* */
769 :
770 : #endif /* CLIB_UNIX */
771 :
772 : static void
773 1 : elog_show_buffer_internal (vlib_main_t * vm, u32 n_events_to_show)
774 : {
775 1 : elog_main_t *em = &vlib_global_main.elog_main;
776 : elog_event_t *e, *es;
777 : f64 dt;
778 :
779 : /* Show events in VLIB time since log clock starts after VLIB clock. */
780 1 : dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
781 1 : * vm->clib_time.seconds_per_clock;
782 :
783 1 : es = elog_peek_events (em);
784 1 : vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
785 : em->event_ring_size,
786 1 : em->n_total_events < em->n_total_events_disable_limit ?
787 : "running" : "stopped");
788 250 : vec_foreach (e, es)
789 : {
790 250 : vlib_cli_output (vm, "%18.9f: %U",
791 250 : e->time + dt, format_elog_event, em, e);
792 250 : n_events_to_show--;
793 250 : if (n_events_to_show == 0)
794 1 : break;
795 : }
796 1 : vec_free (es);
797 :
798 1 : }
799 :
800 : static clib_error_t *
801 1 : elog_show_buffer (vlib_main_t * vm,
802 : unformat_input_t * input, vlib_cli_command_t * cmd)
803 : {
804 : u32 n_events_to_show;
805 1 : clib_error_t *error = 0;
806 :
807 1 : n_events_to_show = 250;
808 1 : while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
809 : {
810 0 : if (unformat (input, "%d", &n_events_to_show))
811 : ;
812 0 : else if (unformat (input, "all"))
813 0 : n_events_to_show = ~0;
814 : else
815 0 : return unformat_parse_error (input);
816 : }
817 1 : elog_show_buffer_internal (vm, n_events_to_show);
818 1 : return error;
819 : }
820 :
821 : /* *INDENT-OFF* */
822 285289 : VLIB_CLI_COMMAND (elog_show_cli, static) = {
823 : .path = "show event-logger",
824 : .short_help = "Show event logger info",
825 : .function = elog_show_buffer,
826 : };
827 : /* *INDENT-ON* */
828 :
829 : void
830 0 : vlib_gdb_show_event_log (void)
831 : {
832 0 : elog_show_buffer_internal (vlib_get_main (), (u32) ~ 0);
833 0 : }
834 :
835 : static inline void
836 1594490000 : vlib_elog_main_loop_event (vlib_main_t * vm,
837 : u32 node_index,
838 : u64 time, u32 n_vectors, u32 is_return)
839 : {
840 1594490000 : vlib_main_t *evm = vlib_get_first_main ();
841 1594430000 : elog_main_t *em = vlib_get_elog_main ();
842 1594400000 : int enabled = evm->elog_trace_graph_dispatch |
843 1594400000 : evm->elog_trace_graph_circuit;
844 :
845 1594400000 : if (PREDICT_FALSE (enabled && n_vectors))
846 : {
847 0 : if (PREDICT_FALSE (!elog_is_enabled (em)))
848 : {
849 0 : evm->elog_trace_graph_dispatch = 0;
850 0 : evm->elog_trace_graph_circuit = 0;
851 0 : return;
852 : }
853 0 : if (PREDICT_TRUE
854 : (evm->elog_trace_graph_dispatch ||
855 : (evm->elog_trace_graph_circuit &&
856 : node_index == evm->elog_trace_graph_circuit_node_index)))
857 : {
858 0 : elog_track (em,
859 : /* event type */
860 0 : vec_elt_at_index (is_return
861 : ? evm->node_return_elog_event_types
862 : : evm->node_call_elog_event_types,
863 : node_index),
864 : /* track */
865 0 : (vm->thread_index ?
866 0 : &vlib_worker_threads[vm->thread_index].elog_track
867 : : &em->default_track),
868 : /* data to log */ n_vectors);
869 : }
870 : }
871 : }
872 :
873 : static inline void
874 : add_trajectory_trace (vlib_buffer_t * b, u32 node_index)
875 : {
876 : #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
877 : if (PREDICT_FALSE (b->trajectory_nb >= VLIB_BUFFER_TRACE_TRAJECTORY_MAX))
878 : return;
879 : b->trajectory_trace[b->trajectory_nb] = node_index;
880 : b->trajectory_nb++;
881 : #endif
882 : }
883 :
884 : static_always_inline u64
885 21392400000 : dispatch_node (vlib_main_t * vm,
886 : vlib_node_runtime_t * node,
887 : vlib_node_type_t type,
888 : vlib_node_state_t dispatch_state,
889 : vlib_frame_t * frame, u64 last_time_stamp)
890 : {
891 : uword n, v;
892 : u64 t;
893 21392400000 : vlib_node_main_t *nm = &vm->node_main;
894 : vlib_next_frame_t *nf;
895 :
896 : if (CLIB_DEBUG > 0)
897 : {
898 21392400000 : vlib_node_t *n = vlib_get_node (vm, node->node_index);
899 21389700000 : ASSERT (n->type == type);
900 : }
901 :
902 : /* Only non-internal nodes may be disabled. */
903 21389700000 : if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
904 : {
905 19991900000 : ASSERT (type != VLIB_NODE_TYPE_INTERNAL);
906 19992000000 : return last_time_stamp;
907 : }
908 :
909 1397740000 : if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
910 1382030000 : && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
911 : {
912 1392880000 : u32 c = node->input_main_loops_per_call;
913 : /* Only call node when count reaches zero. */
914 1392880000 : if (c)
915 : {
916 614174000 : node->input_main_loops_per_call = c - 1;
917 614174000 : return last_time_stamp;
918 : }
919 : }
920 :
921 : /* Speculatively prefetch next frames. */
922 783566000 : if (node->n_next_nodes > 0)
923 : {
924 543717000 : nf = vec_elt_at_index (nm->next_frames, node->next_frame_index);
925 543709000 : CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
926 : }
927 :
928 795074000 : vm->cpu_time_last_node_dispatch = last_time_stamp;
929 :
930 810779000 : vlib_elog_main_loop_event (vm, node->node_index,
931 15704700 : last_time_stamp, frame ? frame->n_vectors : 0,
932 : /* is_after */ 0);
933 :
934 795088000 : vlib_node_runtime_perf_counter (vm, node, frame, 0, last_time_stamp,
935 : VLIB_NODE_RUNTIME_PERF_BEFORE);
936 :
937 : /*
938 : * Turn this on if you run into
939 : * "bad monkey" contexts, and you want to know exactly
940 : * which nodes they've visited... See ixge.c...
941 : */
942 : if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
943 : {
944 : int i;
945 : u32 *from;
946 : from = vlib_frame_vector_args (frame);
947 : for (i = 0; i < frame->n_vectors; i++)
948 : {
949 : vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
950 : add_trajectory_trace (b, node->node_index);
951 : }
952 : if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0))
953 : n = node->function (vm, node, frame);
954 : else
955 : n = vm->dispatch_wrapper_fn (vm, node, frame);
956 : }
957 : else
958 : {
959 795092000 : if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0))
960 795092000 : n = node->function (vm, node, frame);
961 : else
962 33 : n = vm->dispatch_wrapper_fn (vm, node, frame);
963 : }
964 :
965 794982000 : t = clib_cpu_time_now ();
966 :
967 794980000 : vlib_node_runtime_perf_counter (vm, node, frame, n, t,
968 : VLIB_NODE_RUNTIME_PERF_AFTER);
969 :
970 794983000 : vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ );
971 :
972 794972000 : vm->main_loop_vectors_processed += n;
973 794972000 : vm->main_loop_nodes_processed += n > 0;
974 :
975 794972000 : v = vlib_node_runtime_update_stats (vm, node,
976 : /* n_calls */ 1,
977 : /* n_vectors */ n,
978 : /* n_clocks */ t - last_time_stamp);
979 :
980 : /* When in adaptive mode and vector rate crosses threshold switch to
981 : polling mode and vice versa. */
982 795062000 : if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE))
983 : {
984 : /* *INDENT-OFF* */
985 : ELOG_TYPE_DECLARE (e) =
986 : {
987 : .function = (char *) __FUNCTION__,
988 : .format = "%s vector length %d, switching to %s",
989 : .format_args = "T4i4t4",
990 : .n_enum_strings = 2,
991 : .enum_strings = {
992 : "interrupt", "polling",
993 : },
994 : };
995 : /* *INDENT-ON* */
996 : struct
997 : {
998 : u32 node_name, vector_length, is_polling;
999 : } *ed;
1000 :
1001 7059 : if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1002 3945 : && v >= nm->polling_threshold_vector_length) &&
1003 18 : !(node->flags &
1004 : VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
1005 18 : {
1006 18 : vlib_node_t *n = vlib_get_node (vm, node->node_index);
1007 18 : n->state = VLIB_NODE_STATE_POLLING;
1008 18 : node->state = VLIB_NODE_STATE_POLLING;
1009 18 : node->flags &=
1010 : ~VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
1011 18 : node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
1012 18 : nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1013 18 : nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1014 :
1015 18 : if (PREDICT_FALSE (
1016 : vlib_get_first_main ()->elog_trace_graph_dispatch))
1017 : {
1018 0 : vlib_worker_thread_t *w = vlib_worker_threads
1019 0 : + vm->thread_index;
1020 :
1021 0 : ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1022 : w->elog_track);
1023 0 : ed->node_name = n->name_elog_string;
1024 0 : ed->vector_length = v;
1025 0 : ed->is_polling = 1;
1026 : }
1027 : }
1028 7041 : else if (dispatch_state == VLIB_NODE_STATE_POLLING
1029 3114 : && v <= nm->interrupt_threshold_vector_length)
1030 : {
1031 36 : vlib_node_t *n = vlib_get_node (vm, node->node_index);
1032 36 : if (node->flags &
1033 : VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)
1034 : {
1035 : /* Switch to interrupt mode after dispatch in polling one more time.
1036 : This allows driver to re-enable interrupts. */
1037 18 : n->state = VLIB_NODE_STATE_INTERRUPT;
1038 18 : node->state = VLIB_NODE_STATE_INTERRUPT;
1039 18 : node->flags &=
1040 : ~VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
1041 18 : nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1;
1042 18 : nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1;
1043 :
1044 : }
1045 : else
1046 : {
1047 18 : vlib_worker_thread_t *w = vlib_worker_threads
1048 18 : + vm->thread_index;
1049 18 : node->flags |=
1050 : VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
1051 18 : if (PREDICT_FALSE (
1052 : vlib_get_first_main ()->elog_trace_graph_dispatch))
1053 : {
1054 6335 : ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1055 : w->elog_track);
1056 0 : ed->node_name = n->name_elog_string;
1057 0 : ed->vector_length = v;
1058 0 : ed->is_polling = 0;
1059 : }
1060 : }
1061 : }
1062 : }
1063 :
1064 795055000 : return t;
1065 : }
1066 :
1067 : static u64
1068 15704700 : dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1069 : u64 last_time_stamp)
1070 : {
1071 15704700 : vlib_node_main_t *nm = &vm->node_main;
1072 : vlib_frame_t *f;
1073 : vlib_next_frame_t *nf, nf_placeholder;
1074 : vlib_node_runtime_t *n;
1075 : vlib_frame_t *restore_frame;
1076 : vlib_pending_frame_t *p;
1077 :
1078 : /* See comment below about dangling references to nm->pending_frames */
1079 15704700 : p = nm->pending_frames + pending_frame_index;
1080 :
1081 15704700 : n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
1082 : p->node_runtime_index);
1083 :
1084 15704700 : f = vlib_get_frame (vm, p->frame);
1085 15704700 : if (p->next_frame_index == VLIB_PENDING_FRAME_NO_NEXT_FRAME)
1086 : {
1087 : /* No next frame: so use placeholder on stack. */
1088 187685 : nf = &nf_placeholder;
1089 187685 : nf->flags = f->frame_flags & VLIB_NODE_FLAG_TRACE;
1090 187685 : nf->frame = NULL;
1091 : }
1092 : else
1093 15517000 : nf = vec_elt_at_index (nm->next_frames, p->next_frame_index);
1094 :
1095 15704700 : ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
1096 :
1097 : /* Force allocation of new frame while current frame is being
1098 : dispatched. */
1099 15704700 : restore_frame = NULL;
1100 15704700 : if (nf->frame == p->frame)
1101 : {
1102 14948900 : nf->frame = NULL;
1103 14948900 : nf->flags &= ~VLIB_FRAME_IS_ALLOCATED;
1104 14948900 : if (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH))
1105 14948700 : restore_frame = p->frame;
1106 : }
1107 :
1108 : /* Frame must be pending. */
1109 15704700 : ASSERT (f->frame_flags & VLIB_FRAME_PENDING);
1110 15704700 : ASSERT (f->n_vectors > 0);
1111 :
1112 : /* Copy trace flag from next frame to node.
1113 : Trace flag indicates that at least one vector in the dispatched
1114 : frame is traced. */
1115 15704700 : n->flags &= ~VLIB_NODE_FLAG_TRACE;
1116 15704700 : n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1117 15704700 : nf->flags &= ~VLIB_FRAME_TRACE;
1118 :
1119 15704700 : last_time_stamp = dispatch_node (vm, n,
1120 : VLIB_NODE_TYPE_INTERNAL,
1121 : VLIB_NODE_STATE_POLLING,
1122 : f, last_time_stamp);
1123 : /* Internal node vector-rate accounting, for summary stats */
1124 15704700 : vm->internal_node_vectors += f->n_vectors;
1125 15704700 : vm->internal_node_calls++;
1126 15704700 : vm->internal_node_last_vectors_per_main_loop =
1127 15704700 : (f->n_vectors > vm->internal_node_last_vectors_per_main_loop) ?
1128 15704700 : f->n_vectors : vm->internal_node_last_vectors_per_main_loop;
1129 :
1130 15704700 : f->frame_flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_NO_APPEND);
1131 :
1132 : /* Frame is ready to be used again, so restore it. */
1133 15704700 : if (restore_frame != NULL)
1134 : {
1135 : /*
1136 : * We musn't restore a frame that is flagged to be freed. This
1137 : * shouldn't happen since frames to be freed post dispatch are
1138 : * those used when the to-node frame becomes full i.e. they form a
1139 : * sort of queue of frames to a single node. If we get here then
1140 : * the to-node frame and the pending frame *were* the same, and so
1141 : * we removed the to-node frame. Therefore this frame is no
1142 : * longer part of the queue for that node and hence it cannot be
1143 : * it's overspill.
1144 : */
1145 14948700 : ASSERT (!(f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH));
1146 :
1147 : /*
1148 : * NB: dispatching node n can result in the creation and scheduling
1149 : * of new frames, and hence in the reallocation of nm->pending_frames.
1150 : * Recompute p, or no supper. This was broken for more than 10 years.
1151 : */
1152 14948700 : p = nm->pending_frames + pending_frame_index;
1153 :
1154 : /*
1155 : * p->next_frame_index can change during node dispatch if node
1156 : * function decides to change graph hook up.
1157 : */
1158 14948700 : nf = vec_elt_at_index (nm->next_frames, p->next_frame_index);
1159 14948700 : nf->flags |= VLIB_FRAME_IS_ALLOCATED;
1160 :
1161 14948700 : if (NULL == nf->frame)
1162 : {
1163 : /* no new frame has been assigned to this node, use the saved one */
1164 14948500 : nf->frame = restore_frame;
1165 14948500 : f->n_vectors = 0;
1166 14948500 : f->flags = 0;
1167 : }
1168 : else
1169 : {
1170 : /* The node has gained a frame, implying packets from the current frame
1171 : were re-queued to this same node. we don't need the saved one
1172 : anymore */
1173 178 : vlib_frame_free (vm, f);
1174 : }
1175 : }
1176 : else
1177 : {
1178 756013 : if (f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH)
1179 : {
1180 755832 : ASSERT (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH));
1181 755832 : vlib_frame_free (vm, f);
1182 : }
1183 : }
1184 :
1185 15704700 : return last_time_stamp;
1186 : }
1187 :
1188 : always_inline uword
1189 462 : vlib_process_stack_is_valid (vlib_process_t * p)
1190 : {
1191 462 : return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1192 : }
1193 :
1194 : typedef struct
1195 : {
1196 : vlib_main_t *vm;
1197 : vlib_process_t *process;
1198 : vlib_frame_t *frame;
1199 : } vlib_process_bootstrap_args_t;
1200 :
1201 : /* Called in process stack. */
1202 : static uword
1203 25971 : vlib_process_bootstrap (uword _a)
1204 : {
1205 : vlib_process_bootstrap_args_t *a;
1206 : vlib_main_t *vm;
1207 : vlib_node_runtime_t *node;
1208 : vlib_frame_t *f;
1209 : vlib_process_t *p;
1210 : uword n;
1211 :
1212 25971 : a = uword_to_pointer (_a, vlib_process_bootstrap_args_t *);
1213 :
1214 25971 : vm = a->vm;
1215 25971 : p = a->process;
1216 25971 : vlib_process_finish_switch_stack (vm);
1217 :
1218 25971 : f = a->frame;
1219 25971 : node = &p->node_runtime;
1220 :
1221 25971 : n = node->function (vm, node, f);
1222 :
1223 462 : ASSERT (vlib_process_stack_is_valid (p));
1224 :
1225 462 : vlib_process_start_switch_stack (vm, 0);
1226 462 : clib_longjmp (&p->return_longjmp, n);
1227 :
1228 0 : return n;
1229 : }
1230 :
1231 : /* Called in main stack. */
1232 : static_always_inline uword
1233 25971 : vlib_process_startup (vlib_main_t * vm, vlib_process_t * p, vlib_frame_t * f)
1234 : {
1235 : vlib_process_bootstrap_args_t a;
1236 : uword r;
1237 :
1238 25971 : a.vm = vm;
1239 25971 : a.process = p;
1240 25971 : a.frame = f;
1241 :
1242 25971 : r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN);
1243 51942 : if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN)
1244 : {
1245 25971 : vlib_process_start_switch_stack (vm, p);
1246 25971 : r = clib_calljmp (vlib_process_bootstrap, pointer_to_uword (&a),
1247 25971 : (void *) p->stack + (1 << p->log2_n_stack_bytes));
1248 : }
1249 : else
1250 25971 : vlib_process_finish_switch_stack (vm);
1251 :
1252 25971 : return r;
1253 : }
1254 :
1255 : static_always_inline uword
1256 2259280 : vlib_process_resume (vlib_main_t * vm, vlib_process_t * p)
1257 : {
1258 : uword r;
1259 2259280 : p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
1260 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
1261 : | VLIB_PROCESS_RESUME_PENDING);
1262 2259280 : r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN);
1263 4518570 : if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN)
1264 : {
1265 2259280 : vlib_process_start_switch_stack (vm, p);
1266 2259280 : clib_longjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_RESUME);
1267 : }
1268 : else
1269 2259280 : vlib_process_finish_switch_stack (vm);
1270 2259280 : return r;
1271 : }
1272 :
1273 : static u64
1274 27696 : dispatch_process (vlib_main_t * vm,
1275 : vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1276 : {
1277 27696 : vlib_node_main_t *nm = &vm->node_main;
1278 27696 : vlib_node_runtime_t *node_runtime = &p->node_runtime;
1279 27696 : vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1280 : u32 old_process_index;
1281 : u64 t;
1282 : uword n_vectors, is_suspend;
1283 :
1284 27696 : if (node->state != VLIB_NODE_STATE_POLLING
1285 26016 : || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
1286 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)))
1287 1725 : return last_time_stamp;
1288 :
1289 25971 : p->flags |= VLIB_PROCESS_IS_RUNNING;
1290 :
1291 25971 : t = last_time_stamp;
1292 25971 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1293 0 : f ? f->n_vectors : 0, /* is_after */ 0);
1294 :
1295 : /* Save away current process for suspend. */
1296 25971 : old_process_index = nm->current_process_index;
1297 25971 : nm->current_process_index = node->runtime_index;
1298 :
1299 25971 : vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp,
1300 : VLIB_NODE_RUNTIME_PERF_BEFORE);
1301 :
1302 25971 : n_vectors = vlib_process_startup (vm, p, f);
1303 :
1304 25971 : nm->current_process_index = old_process_index;
1305 :
1306 25971 : ASSERT (n_vectors != VLIB_PROCESS_RETURN_LONGJMP_RETURN);
1307 25971 : is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1308 25971 : if (is_suspend)
1309 : {
1310 : vlib_pending_frame_t *pf;
1311 :
1312 25971 : n_vectors = 0;
1313 25971 : pool_get (nm->suspended_process_frames, pf);
1314 25971 : pf->node_runtime_index = node->runtime_index;
1315 25971 : pf->frame = f;
1316 25971 : pf->next_frame_index = ~0;
1317 :
1318 25971 : p->n_suspends += 1;
1319 25971 : p->suspended_process_frame_index = pf - nm->suspended_process_frames;
1320 :
1321 25971 : if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
1322 : {
1323 14467 : TWT (tw_timer_wheel) * tw =
1324 : (TWT (tw_timer_wheel) *) nm->timing_wheel;
1325 14467 : p->stop_timer_handle =
1326 14467 : TW (tw_timer_start) (tw,
1327 : vlib_timing_wheel_data_set_suspended_process
1328 : (node->runtime_index) /* [sic] pool idex */ ,
1329 : 0 /* timer_id */ ,
1330 : p->resume_clock_interval);
1331 : }
1332 : }
1333 : else
1334 0 : p->flags &= ~VLIB_PROCESS_IS_RUNNING;
1335 :
1336 25971 : t = clib_cpu_time_now ();
1337 :
1338 25971 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1339 : /* is_after */ 1);
1340 :
1341 25971 : vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t,
1342 : VLIB_NODE_RUNTIME_PERF_AFTER);
1343 :
1344 25971 : vlib_process_update_stats (vm, p,
1345 : /* n_calls */ !is_suspend,
1346 : /* n_vectors */ n_vectors,
1347 : /* n_clocks */ t - last_time_stamp);
1348 :
1349 25971 : return t;
1350 : }
1351 :
1352 : void
1353 96 : vlib_start_process (vlib_main_t * vm, uword process_index)
1354 : {
1355 96 : vlib_node_main_t *nm = &vm->node_main;
1356 96 : vlib_process_t *p = vec_elt (nm->processes, process_index);
1357 96 : dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1358 96 : }
1359 :
1360 : static u64
1361 2259280 : dispatch_suspended_process (vlib_main_t * vm,
1362 : uword process_index, u64 last_time_stamp)
1363 : {
1364 2259280 : vlib_node_main_t *nm = &vm->node_main;
1365 : vlib_node_runtime_t *node_runtime;
1366 : vlib_node_t *node;
1367 : vlib_frame_t *f;
1368 : vlib_process_t *p;
1369 : vlib_pending_frame_t *pf;
1370 : u64 t, n_vectors, is_suspend;
1371 :
1372 2259280 : t = last_time_stamp;
1373 :
1374 2259280 : p = vec_elt (nm->processes, process_index);
1375 2259280 : if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING)))
1376 0 : return last_time_stamp;
1377 :
1378 2259280 : ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
1379 : | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT));
1380 :
1381 2259280 : pf = pool_elt_at_index (nm->suspended_process_frames,
1382 : p->suspended_process_frame_index);
1383 :
1384 2259280 : node_runtime = &p->node_runtime;
1385 2259280 : node = vlib_get_node (vm, node_runtime->node_index);
1386 2259280 : f = pf->frame;
1387 :
1388 2259280 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1389 0 : f ? f->n_vectors : 0, /* is_after */ 0);
1390 :
1391 : /* Save away current process for suspend. */
1392 2259280 : nm->current_process_index = node->runtime_index;
1393 :
1394 2259280 : vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp,
1395 : VLIB_NODE_RUNTIME_PERF_BEFORE);
1396 :
1397 2259280 : n_vectors = vlib_process_resume (vm, p);
1398 2259280 : t = clib_cpu_time_now ();
1399 :
1400 2259280 : nm->current_process_index = ~0;
1401 :
1402 2259280 : is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1403 2259280 : if (is_suspend)
1404 : {
1405 : /* Suspend it again. */
1406 2258820 : n_vectors = 0;
1407 2258820 : p->n_suspends += 1;
1408 2258820 : if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
1409 : {
1410 2236460 : p->stop_timer_handle =
1411 2236460 : TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1412 : vlib_timing_wheel_data_set_suspended_process
1413 : (node->runtime_index) /* [sic] pool idex */ ,
1414 : 0 /* timer_id */ ,
1415 : p->resume_clock_interval);
1416 : }
1417 : }
1418 : else
1419 : {
1420 462 : p->flags &= ~VLIB_PROCESS_IS_RUNNING;
1421 462 : pool_put_index (nm->suspended_process_frames,
1422 : p->suspended_process_frame_index);
1423 462 : p->suspended_process_frame_index = ~0;
1424 : }
1425 :
1426 2259280 : t = clib_cpu_time_now ();
1427 2259280 : vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1428 : /* is_after */ 1);
1429 :
1430 2259280 : vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t,
1431 : VLIB_NODE_RUNTIME_PERF_AFTER);
1432 :
1433 2259280 : vlib_process_update_stats (vm, p,
1434 : /* n_calls */ !is_suspend,
1435 : /* n_vectors */ n_vectors,
1436 : /* n_clocks */ t - last_time_stamp);
1437 :
1438 2259280 : return t;
1439 : }
1440 :
1441 : void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1442 : void
1443 0 : vl_api_send_pending_rpc_requests (vlib_main_t * vm)
1444 : {
1445 0 : }
1446 :
1447 : static_always_inline void
1448 630 : vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
1449 : {
1450 630 : vlib_node_main_t *nm = &vm->node_main;
1451 630 : vlib_thread_main_t *tm = vlib_get_thread_main ();
1452 : uword i;
1453 : u64 cpu_time_now;
1454 : f64 now;
1455 : vlib_frame_queue_main_t *fqm;
1456 630 : u32 frame_queue_check_counter = 0;
1457 :
1458 : /* Initialize pending node vector. */
1459 630 : if (is_main)
1460 : {
1461 575 : vec_resize (nm->pending_frames, 32);
1462 575 : vec_set_len (nm->pending_frames, 0);
1463 : }
1464 :
1465 : /* Mark time of main loop start. */
1466 630 : if (is_main)
1467 : {
1468 575 : cpu_time_now = vm->clib_time.last_cpu_time;
1469 575 : vm->cpu_time_main_loop_start = cpu_time_now;
1470 : }
1471 : else
1472 55 : cpu_time_now = clib_cpu_time_now ();
1473 :
1474 630 : nm->pending_interrupts = 0;
1475 :
1476 : /* Pre-allocate expired nodes. */
1477 630 : if (!nm->polling_threshold_vector_length)
1478 630 : nm->polling_threshold_vector_length = 10;
1479 630 : if (!nm->interrupt_threshold_vector_length)
1480 630 : nm->interrupt_threshold_vector_length = 5;
1481 :
1482 630 : vm->cpu_id = clib_get_current_cpu_id ();
1483 630 : vm->numa_node = clib_get_current_numa_node ();
1484 630 : os_set_numa_index (vm->numa_node);
1485 :
1486 : /* Start all processes. */
1487 630 : if (is_main)
1488 : {
1489 : uword i;
1490 :
1491 : /*
1492 : * Perform an initial barrier sync. Pays no attention to
1493 : * the barrier sync hold-down timer scheme, which won't work
1494 : * at this point in time.
1495 : */
1496 575 : vlib_worker_thread_initial_barrier_sync_and_release (vm);
1497 :
1498 575 : nm->current_process_index = ~0;
1499 28175 : for (i = 0; i < vec_len (nm->processes); i++)
1500 27600 : cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1501 : cpu_time_now);
1502 : }
1503 :
1504 : while (1)
1505 699783000 : {
1506 : vlib_node_runtime_t *n;
1507 : u8 pending_interrupts;
1508 :
1509 699784000 : if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1510 : {
1511 171 : if (!is_main)
1512 166 : vl_api_send_pending_rpc_requests (vm);
1513 : }
1514 :
1515 699603000 : if (!is_main)
1516 33380900 : vlib_worker_thread_barrier_check ();
1517 :
1518 699727000 : if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter))
1519 : {
1520 51990 : u32 processed = 0;
1521 : vlib_frame_queue_dequeue_fn_t *fn;
1522 :
1523 51990 : if (vm->check_frame_queues)
1524 : {
1525 559 : frame_queue_check_counter = 100;
1526 559 : vm->check_frame_queues = 0;
1527 : }
1528 :
1529 2111360 : vec_foreach (fqm, tm->frame_queue_mains)
1530 : {
1531 2058840 : fn = fqm->frame_queue_dequeue_fn;
1532 2058840 : processed += (fn) (vm, fqm);
1533 : }
1534 :
1535 : /* No handoff queue work found? */
1536 51939 : if (processed)
1537 579 : frame_queue_check_counter = 100;
1538 : else
1539 51360 : frame_queue_check_counter--;
1540 : }
1541 :
1542 699727000 : if (PREDICT_FALSE (vec_len (vm->worker_thread_main_loop_callbacks)))
1543 0 : clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm,
1544 : cpu_time_now);
1545 :
1546 : /* Process pre-input nodes. */
1547 699727000 : cpu_time_now = clib_cpu_time_now ();
1548 2795640000 : vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])
1549 2094830000 : cpu_time_now = dispatch_node (vm, n,
1550 : VLIB_NODE_TYPE_PRE_INPUT,
1551 : VLIB_NODE_STATE_POLLING,
1552 : /* frame */ 0,
1553 : cpu_time_now);
1554 :
1555 : pending_interrupts =
1556 700696000 : __atomic_load_n (&nm->pending_interrupts, __ATOMIC_ACQUIRE);
1557 :
1558 700696000 : if (pending_interrupts)
1559 : {
1560 478968 : int int_num = -1;
1561 478968 : nm->pending_interrupts = 0;
1562 :
1563 478968 : while ((int_num = clib_interrupt_get_next (
1564 : nm->pre_input_node_interrupts, int_num)) != -1)
1565 : {
1566 : vlib_node_runtime_t *n;
1567 963016 : clib_interrupt_clear (nm->pre_input_node_interrupts, int_num);
1568 0 : n = vec_elt_at_index (
1569 : nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num);
1570 0 : cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT,
1571 : VLIB_NODE_STATE_INTERRUPT,
1572 : /* frame */ 0, cpu_time_now);
1573 : }
1574 : }
1575 :
1576 : /* Next process input nodes. */
1577 20015900000 : vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
1578 19317100000 : cpu_time_now = dispatch_node (vm, n,
1579 : VLIB_NODE_TYPE_INPUT,
1580 : VLIB_NODE_STATE_POLLING,
1581 : /* frame */ 0,
1582 : cpu_time_now);
1583 :
1584 696730000 : if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1585 638307000 : vm->queue_signal_callback (vm);
1586 :
1587 699638000 : if (pending_interrupts)
1588 : {
1589 478988 : int int_num = -1;
1590 :
1591 958117 : while ((int_num = clib_interrupt_get_next (nm->input_node_interrupts,
1592 : int_num)) != -1)
1593 : {
1594 : vlib_node_runtime_t *n;
1595 479175 : clib_interrupt_clear (nm->input_node_interrupts, int_num);
1596 479186 : n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
1597 : int_num);
1598 479180 : cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,
1599 : VLIB_NODE_STATE_INTERRUPT,
1600 : /* frame */ 0, cpu_time_now);
1601 : }
1602 : }
1603 :
1604 : /* Input nodes may have added work to the pending vector.
1605 : Process pending vector until there is nothing left.
1606 : All pending vectors will be processed from input -> output. */
1607 715343000 : for (i = 0; i < _vec_len (nm->pending_frames); i++)
1608 15704700 : cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1609 : /* Reset pending vector for next iteration. */
1610 699498000 : vec_set_len (nm->pending_frames, 0);
1611 :
1612 699554000 : if (is_main)
1613 : {
1614 : /* *INDENT-OFF* */
1615 : ELOG_TYPE_DECLARE (es) =
1616 : {
1617 : .format = "process tw start",
1618 : .format_args = "",
1619 : };
1620 : ELOG_TYPE_DECLARE (ee) =
1621 : {
1622 : .format = "process tw end: %d",
1623 : .format_args = "i4",
1624 : };
1625 : /* *INDENT-ON* */
1626 :
1627 : struct
1628 : {
1629 : int nready_procs;
1630 : } *ed;
1631 :
1632 : /* Check if process nodes have expired from timing wheel. */
1633 666376000 : ASSERT (nm->data_from_advancing_timing_wheel != 0);
1634 :
1635 666376000 : if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
1636 0 : ed = ELOG_DATA (&vlib_global_main.elog_main, es);
1637 :
1638 666376000 : TW (tw_timer_expire_timers)
1639 666376000 : ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm));
1640 :
1641 666376000 : ASSERT (nm->data_from_advancing_timing_wheel != 0);
1642 :
1643 666376000 : if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
1644 : {
1645 0 : ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
1646 0 : ed->nready_procs =
1647 0 : _vec_len (nm->data_from_advancing_timing_wheel);
1648 : }
1649 :
1650 666376000 : if (PREDICT_FALSE
1651 : (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1652 : {
1653 : uword i;
1654 :
1655 4332880 : for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1656 2259280 : i++)
1657 : {
1658 2259280 : u32 d = nm->data_from_advancing_timing_wheel[i];
1659 2259280 : u32 di = vlib_timing_wheel_data_get_index (d);
1660 :
1661 2259280 : if (vlib_timing_wheel_data_is_timed_event (d))
1662 : {
1663 0 : vlib_signal_timed_event_data_t *te =
1664 0 : pool_elt_at_index (nm->signal_timed_event_data_pool,
1665 : di);
1666 : vlib_node_t *n =
1667 0 : vlib_get_node (vm, te->process_node_index);
1668 0 : vlib_process_t *p =
1669 0 : vec_elt (nm->processes, n->runtime_index);
1670 0 : p->stop_timer_handle = ~0;
1671 : void *data;
1672 : data =
1673 0 : vlib_process_signal_event_helper (nm, n, p,
1674 0 : te->event_type_index,
1675 0 : te->n_data_elts,
1676 0 : te->n_data_elt_bytes);
1677 0 : if (te->n_data_bytes < sizeof (te->inline_event_data))
1678 0 : clib_memcpy_fast (data, te->inline_event_data,
1679 0 : te->n_data_bytes);
1680 : else
1681 : {
1682 0 : clib_memcpy_fast (data, te->event_data_as_vector,
1683 0 : te->n_data_bytes);
1684 0 : vec_free (te->event_data_as_vector);
1685 : }
1686 0 : pool_put (nm->signal_timed_event_data_pool, te);
1687 : }
1688 : else
1689 : {
1690 2259280 : cpu_time_now = clib_cpu_time_now ();
1691 : cpu_time_now =
1692 2259280 : dispatch_suspended_process (vm, di, cpu_time_now);
1693 : }
1694 : }
1695 2073590 : vec_set_len (nm->data_from_advancing_timing_wheel, 0);
1696 : }
1697 : }
1698 699554000 : vlib_increment_main_loop_counter (vm);
1699 : /* Record time stamp in case there are no enabled nodes and above
1700 : calls do not update time stamp. */
1701 699649000 : cpu_time_now = clib_cpu_time_now ();
1702 699548000 : vm->loops_this_reporting_interval++;
1703 699548000 : now = clib_time_now_internal (&vm->clib_time, cpu_time_now);
1704 : /* Time to update loops_per_second? */
1705 699783000 : if (PREDICT_FALSE (now >= vm->loop_interval_end))
1706 : {
1707 : /* Next sample ends in 20ms */
1708 8178360 : if (vm->loop_interval_start)
1709 : {
1710 : f64 this_loops_per_second;
1711 :
1712 8165260 : this_loops_per_second =
1713 8165260 : ((f64) vm->loops_this_reporting_interval) / (now -
1714 8165260 : vm->loop_interval_start);
1715 :
1716 8165260 : vm->loops_per_second =
1717 8165260 : vm->loops_per_second * vm->damping_constant +
1718 8165260 : (1.0 - vm->damping_constant) * this_loops_per_second;
1719 8165260 : if (vm->loops_per_second != 0.0)
1720 8166980 : vm->seconds_per_loop = 1.0 / vm->loops_per_second;
1721 : else
1722 0 : vm->seconds_per_loop = 0.0;
1723 : }
1724 : /* New interval starts now, and ends in 20ms */
1725 8178360 : vm->loop_interval_start = now;
1726 8178360 : vm->loop_interval_end = now + 2e-4;
1727 8178360 : vm->loops_this_reporting_interval = 0;
1728 : }
1729 : }
1730 : }
1731 :
1732 : static void
1733 575 : vlib_main_loop (vlib_main_t * vm)
1734 : {
1735 575 : vlib_main_or_worker_loop (vm, /* is_main */ 1);
1736 0 : }
1737 :
1738 : void
1739 55 : vlib_worker_loop (vlib_main_t * vm)
1740 : {
1741 55 : vlib_main_or_worker_loop (vm, /* is_main */ 0);
1742 0 : }
1743 :
1744 : vlib_global_main_t vlib_global_main;
1745 :
1746 : void
1747 1 : vlib_add_del_post_mortem_callback (void *cb, int is_add)
1748 : {
1749 1 : vlib_global_main_t *vgm = vlib_get_global_main ();
1750 : int i;
1751 :
1752 1 : if (is_add == 0)
1753 : {
1754 1 : for (i = vec_len (vgm->post_mortem_callbacks) - 1; i >= 0; i--)
1755 0 : if (vgm->post_mortem_callbacks[i] == cb)
1756 0 : vec_del1 (vgm->post_mortem_callbacks, i);
1757 1 : return;
1758 : }
1759 :
1760 0 : for (i = 0; i < vec_len (vgm->post_mortem_callbacks); i++)
1761 0 : if (vgm->post_mortem_callbacks[i] == cb)
1762 0 : return;
1763 0 : vec_add1 (vgm->post_mortem_callbacks, cb);
1764 : }
1765 :
1766 : static void
1767 0 : elog_post_mortem_dump (void)
1768 : {
1769 0 : elog_main_t *em = vlib_get_elog_main ();
1770 :
1771 : u8 *filename;
1772 : clib_error_t *error;
1773 :
1774 0 : filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
1775 0 : error = elog_write_file (em, (char *) filename, 1 /* flush ring */);
1776 0 : if (error)
1777 0 : clib_error_report (error);
1778 : /*
1779 : * We're in the middle of crashing. Don't try to free the filename.
1780 : */
1781 0 : }
1782 :
1783 : static clib_error_t *
1784 575 : vlib_main_configure (vlib_main_t * vm, unformat_input_t * input)
1785 : {
1786 575 : vlib_global_main_t *vgm = vlib_get_global_main ();
1787 575 : int turn_on_mem_trace = 0;
1788 :
1789 575 : while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1790 : {
1791 0 : if (unformat (input, "memory-trace"))
1792 0 : turn_on_mem_trace = 1;
1793 :
1794 0 : else if (unformat (input, "elog-events %d",
1795 : &vgm->configured_elog_ring_size))
1796 0 : vgm->configured_elog_ring_size =
1797 0 : 1 << max_log2 (vgm->configured_elog_ring_size);
1798 0 : else if (unformat (input, "elog-post-mortem-dump"))
1799 0 : vlib_add_del_post_mortem_callback (elog_post_mortem_dump,
1800 : /* is_add */ 1);
1801 0 : else if (unformat (input, "buffer-alloc-success-rate %f",
1802 : &vm->buffer_alloc_success_rate))
1803 : {
1804 : if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0)
1805 0 : return clib_error_return
1806 : (0, "Buffer fault injection not configured");
1807 : }
1808 0 : else if (unformat (input, "buffer-alloc-success-seed %u",
1809 : &vm->buffer_alloc_success_seed))
1810 : {
1811 : if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0)
1812 0 : return clib_error_return
1813 : (0, "Buffer fault injection not configured");
1814 : }
1815 : else
1816 0 : return unformat_parse_error (input);
1817 : }
1818 :
1819 575 : unformat_free (input);
1820 :
1821 : /* Enable memory trace as early as possible. */
1822 575 : if (turn_on_mem_trace)
1823 0 : clib_mem_trace (1);
1824 :
1825 575 : return 0;
1826 : }
1827 :
1828 7514 : VLIB_EARLY_CONFIG_FUNCTION (vlib_main_configure, "vlib");
1829 :
1830 : static void
1831 0 : placeholder_queue_signal_callback (vlib_main_t * vm)
1832 : {
1833 0 : }
1834 :
1835 : #define foreach_weak_reference_stub \
1836 : _(vpe_api_init) \
1837 : _(vlibmemory_init) \
1838 : _(map_api_segment_init)
1839 :
1840 : #define _(name) \
1841 : clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
1842 : clib_error_t *name (vlib_main_t *vm) { return 0; }
1843 0 : foreach_weak_reference_stub;
1844 : #undef _
1845 :
1846 : void vl_api_set_elog_main (elog_main_t * m) __attribute__ ((weak));
1847 : void
1848 0 : vl_api_set_elog_main (elog_main_t * m)
1849 : {
1850 0 : clib_warning ("STUB");
1851 0 : }
1852 :
1853 : int vl_api_set_elog_trace_api_messages (int enable) __attribute__ ((weak));
1854 : int
1855 0 : vl_api_set_elog_trace_api_messages (int enable)
1856 : {
1857 0 : clib_warning ("STUB");
1858 0 : return 0;
1859 : }
1860 :
1861 : int vl_api_get_elog_trace_api_messages (void) __attribute__ ((weak));
1862 : int
1863 0 : vl_api_get_elog_trace_api_messages (void)
1864 : {
1865 0 : clib_warning ("STUB");
1866 0 : return 0;
1867 : }
1868 :
1869 : static void
1870 1638410 : process_expired_timer_cb (u32 *expired_timer_handles)
1871 : {
1872 1638410 : vlib_main_t *vm = vlib_get_main ();
1873 1638410 : vlib_node_main_t *nm = &vm->node_main;
1874 : u32 *handle;
1875 :
1876 3432760 : vec_foreach (handle, expired_timer_handles)
1877 : {
1878 1794350 : u32 pi = vlib_timing_wheel_data_get_index (*handle);
1879 1794350 : vlib_process_t *p = vec_elt (nm->processes, pi);
1880 :
1881 1794350 : p->stop_timer_handle = ~0;
1882 : }
1883 1638410 : vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles);
1884 1638410 : }
1885 :
1886 : /* Main function. */
1887 : int
1888 575 : vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
1889 : {
1890 575 : vlib_global_main_t *vgm = vlib_get_global_main ();
1891 : clib_error_t *volatile error;
1892 575 : vlib_node_main_t *nm = &vm->node_main;
1893 :
1894 575 : vm->queue_signal_callback = placeholder_queue_signal_callback;
1895 :
1896 : /* Reconfigure event log which is enabled very early */
1897 575 : if (vgm->configured_elog_ring_size &&
1898 575 : vgm->configured_elog_ring_size != vgm->elog_main.event_ring_size)
1899 0 : elog_resize (&vgm->elog_main, vgm->configured_elog_ring_size);
1900 575 : vl_api_set_elog_main (vlib_get_elog_main ());
1901 575 : (void) vl_api_set_elog_trace_api_messages (1);
1902 :
1903 : /* Default name. */
1904 575 : if (!vgm->name)
1905 0 : vgm->name = "VLIB";
1906 :
1907 575 : if ((error = vlib_physmem_init (vm)))
1908 : {
1909 0 : clib_error_report (error);
1910 0 : goto done;
1911 : }
1912 :
1913 575 : if ((error = vlib_log_init (vm)))
1914 : {
1915 0 : clib_error_report (error);
1916 0 : goto done;
1917 : }
1918 :
1919 575 : if ((error = vlib_stats_init (vm)))
1920 : {
1921 0 : clib_error_report (error);
1922 0 : goto done;
1923 : }
1924 :
1925 575 : if ((error = vlib_buffer_main_init (vm)))
1926 : {
1927 0 : clib_error_report (error);
1928 0 : goto done;
1929 : }
1930 :
1931 575 : if ((error = vlib_thread_init (vm)))
1932 : {
1933 0 : clib_error_report (error);
1934 0 : goto done;
1935 : }
1936 :
1937 : /* Register node ifunction variants */
1938 575 : vlib_register_all_node_march_variants (vm);
1939 :
1940 : /* Register static nodes so that init functions may use them. */
1941 575 : vlib_register_all_static_nodes (vm);
1942 :
1943 : /* Set seed for random number generator.
1944 : Allow user to specify seed to make random sequence deterministic. */
1945 575 : if (!unformat (input, "seed %wd", &vm->random_seed))
1946 575 : vm->random_seed = clib_cpu_time_now ();
1947 575 : clib_random_buffer_init (&vm->random_buffer, vm->random_seed);
1948 :
1949 : /* Initialize node graph. */
1950 575 : if ((error = vlib_node_main_init (vm)))
1951 : {
1952 : /* Arrange for graph hook up error to not be fatal when debugging. */
1953 : if (CLIB_DEBUG > 0)
1954 0 : clib_error_report (error);
1955 : else
1956 : goto done;
1957 : }
1958 :
1959 : /* Direct call / weak reference, for vlib standalone use-cases */
1960 575 : if ((error = vpe_api_init (vm)))
1961 : {
1962 0 : clib_error_report (error);
1963 0 : goto done;
1964 : }
1965 :
1966 575 : if ((error = vlibmemory_init (vm)))
1967 : {
1968 0 : clib_error_report (error);
1969 0 : goto done;
1970 : }
1971 :
1972 575 : if ((error = map_api_segment_init (vm)))
1973 : {
1974 0 : clib_error_report (error);
1975 0 : goto done;
1976 : }
1977 :
1978 : /* See unix/main.c; most likely already set up */
1979 575 : if (vgm->init_functions_called == 0)
1980 0 : vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
1981 575 : if ((error = vlib_call_all_init_functions (vm)))
1982 0 : goto done;
1983 :
1984 575 : nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
1985 : CLIB_CACHE_LINE_BYTES);
1986 :
1987 575 : vec_validate (nm->data_from_advancing_timing_wheel, 10);
1988 575 : vec_set_len (nm->data_from_advancing_timing_wheel, 0);
1989 :
1990 : /* Create the process timing wheel */
1991 575 : TW (tw_timer_wheel_init)
1992 575 : ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1993 : process_expired_timer_cb /* callback */, 10e-6 /* timer period 10us */,
1994 : ~0 /* max expirations per call */);
1995 :
1996 575 : vec_validate (vm->pending_rpc_requests, 0);
1997 575 : vec_set_len (vm->pending_rpc_requests, 0);
1998 575 : vec_validate (vm->processing_rpc_requests, 0);
1999 575 : vec_set_len (vm->processing_rpc_requests, 0);
2000 :
2001 : /* Default params for the buffer allocator fault injector, if configured */
2002 : if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
2003 : {
2004 : vm->buffer_alloc_success_seed = 0xdeaddabe;
2005 : vm->buffer_alloc_success_rate = 0.80;
2006 : }
2007 :
2008 575 : if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
2009 0 : goto done;
2010 :
2011 : /*
2012 : * Use exponential smoothing, with a half-life of 1 second
2013 : * reported_rate(t) = reported_rate(t-1) * K + rate(t)*(1-K)
2014 : *
2015 : * Sample every 20ms, aka 50 samples per second
2016 : * K = exp (-1.0/20.0);
2017 : * K = 0.95
2018 : */
2019 575 : vm->damping_constant = exp (-1.0 / 20.0);
2020 :
2021 : /* Sort per-thread init functions before we start threads */
2022 575 : vlib_sort_init_exit_functions (&vgm->worker_init_function_registrations);
2023 :
2024 : /* Call all main loop enter functions. */
2025 : {
2026 : clib_error_t *sub_error;
2027 575 : sub_error = vlib_call_all_main_loop_enter_functions (vm);
2028 575 : if (sub_error)
2029 0 : clib_error_report (sub_error);
2030 : }
2031 :
2032 575 : switch (clib_setjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_NONE))
2033 : {
2034 575 : case VLIB_MAIN_LOOP_EXIT_NONE:
2035 575 : vm->main_loop_exit_set = 1;
2036 575 : break;
2037 :
2038 575 : case VLIB_MAIN_LOOP_EXIT_CLI:
2039 575 : goto done;
2040 :
2041 0 : default:
2042 0 : error = vm->main_loop_error;
2043 0 : goto done;
2044 : }
2045 :
2046 575 : vlib_main_loop (vm);
2047 :
2048 575 : done:
2049 : /* Stop worker threads, barrier will not be released */
2050 575 : vlib_worker_thread_barrier_sync (vm);
2051 :
2052 : /* Call all exit functions. */
2053 : {
2054 : clib_error_t *sub_error;
2055 575 : sub_error = vlib_call_all_main_loop_exit_functions (vm);
2056 575 : if (sub_error)
2057 0 : clib_error_report (sub_error);
2058 : }
2059 :
2060 575 : if (error)
2061 0 : clib_error_report (error);
2062 :
2063 575 : return vm->main_loop_exit_status;
2064 : }
2065 :
2066 : vlib_main_t *
2067 0 : vlib_get_main_not_inline (void)
2068 : {
2069 0 : return vlib_get_main ();
2070 : }
2071 :
2072 : elog_main_t *
2073 0 : vlib_get_elog_main_not_inline ()
2074 : {
2075 0 : return &vlib_global_main.elog_main;
2076 : }
2077 :
2078 : void
2079 0 : vlib_exit_with_status (vlib_main_t *vm, int status)
2080 : {
2081 0 : vm->main_loop_exit_status = status;
2082 0 : __atomic_store_n (&vm->main_loop_exit_now, 1, __ATOMIC_RELEASE);
2083 0 : }
2084 :
2085 : /*
2086 : * fd.io coding-style-patch-verification: ON
2087 : *
2088 : * Local Variables:
2089 : * eval: (c-set-style "gnu")
2090 : * End:
2091 : */
|
