Line data Source code
1 : /*
2 : * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #include <vnet/session/segment_manager.h>
17 : #include <vnet/session/session.h>
18 : #include <vnet/session/application.h>
19 :
20 : typedef struct segment_manager_main_
21 : {
22 : segment_manager_t *segment_managers; /**< Pool of segment managers */
23 : u32 seg_name_counter; /**< Counter for segment names */
24 :
25 : /*
26 : * Configuration
27 : */
28 : u32 default_fifo_size; /**< default rx/tx fifo size */
29 : u32 default_segment_size; /**< default fifo segment size */
30 : u32 default_app_mq_size; /**< default app msg q size */
31 : u32 default_max_fifo_size; /**< default max fifo size */
32 : u8 default_high_watermark; /**< default high watermark % */
33 : u8 default_low_watermark; /**< default low watermark % */
34 : } segment_manager_main_t;
35 :
36 : static segment_manager_main_t sm_main;
37 :
38 : #define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \
39 : do { \
40 : clib_rwlock_reader_lock (&(SM)->segments_rwlock); \
41 : pool_foreach((VAR), ((SM)->segments)) (BODY); \
42 : clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \
43 : } while (0)
44 :
45 : static segment_manager_props_t *
46 1217 : segment_manager_properties_get (segment_manager_t * sm)
47 : {
48 1217 : app_worker_t *app_wrk = app_worker_get (sm->app_wrk_index);
49 1217 : return application_get_segment_manager_properties (app_wrk->app_index);
50 : }
51 :
52 : segment_manager_props_t *
53 216 : segment_manager_props_init (segment_manager_props_t * props)
54 : {
55 216 : props->add_segment_size = sm_main.default_segment_size;
56 216 : props->rx_fifo_size = sm_main.default_fifo_size;
57 216 : props->tx_fifo_size = sm_main.default_fifo_size;
58 216 : props->evt_q_size = sm_main.default_app_mq_size;
59 216 : props->max_fifo_size = sm_main.default_max_fifo_size;
60 216 : props->high_watermark = sm_main.default_high_watermark;
61 216 : props->low_watermark = sm_main.default_low_watermark;
62 216 : props->n_slices = vlib_num_workers () + 1;
63 216 : return props;
64 : }
65 :
66 : u8
67 270 : segment_manager_app_detached (segment_manager_t * sm)
68 : {
69 270 : return (sm->flags & SEG_MANAGER_F_DETACHED);
70 : }
71 :
72 : void
73 172 : segment_manager_app_detach (segment_manager_t * sm)
74 : {
75 172 : sm->flags |= SEG_MANAGER_F_DETACHED;
76 172 : }
77 :
78 : always_inline u32
79 301 : segment_manager_segment_index (segment_manager_t * sm, fifo_segment_t * seg)
80 : {
81 301 : return (seg - sm->segments);
82 : }
83 :
84 : /**
85 : * Adds segment to segment manager's pool
86 : *
87 : * If needed a writer's lock is acquired before allocating a new segment
88 : * to avoid affecting any of the segments pool readers.
89 : */
90 : static inline int
91 296 : segment_manager_add_segment_inline (segment_manager_t *sm, uword segment_size,
92 : u8 notify_app, u8 flags, u8 need_lock)
93 : {
94 296 : segment_manager_main_t *smm = &sm_main;
95 : segment_manager_props_t *props;
96 : app_worker_t *app_wrk;
97 : fifo_segment_t *fs;
98 296 : u32 fs_index = ~0;
99 : u8 *seg_name;
100 : int rv;
101 :
102 296 : props = segment_manager_properties_get (sm);
103 296 : app_wrk = app_worker_get (sm->app_wrk_index);
104 :
105 : /* Not configured for addition of new segments and not first */
106 296 : if (!props->add_segment && !segment_size)
107 : {
108 : SESSION_DBG ("cannot allocate new segment");
109 0 : return SESSION_E_INVALID;
110 : }
111 :
112 : /*
113 : * Allocate fifo segment and grab lock if needed
114 : */
115 296 : if (need_lock)
116 0 : clib_rwlock_writer_lock (&sm->segments_rwlock);
117 :
118 296 : pool_get_zero (sm->segments, fs);
119 :
120 : /*
121 : * Allocate ssvm segment
122 : */
123 296 : segment_size = segment_size ? segment_size : props->add_segment_size;
124 : /* add overhead to ensure the result segment size is at least
125 : * of that requested */
126 296 : segment_size +=
127 : sizeof (fifo_segment_header_t) +
128 592 : vlib_thread_main.n_vlib_mains * sizeof (fifo_segment_slice_t) +
129 296 : FIFO_SEGMENT_ALLOC_OVERHEAD;
130 :
131 296 : if (props->huge_page)
132 : {
133 0 : uword hugepage_size = clib_mem_get_default_hugepage_size ();
134 0 : segment_size = round_pow2 (segment_size, hugepage_size);
135 0 : fs->ssvm.huge_page = 1;
136 : }
137 : else
138 296 : segment_size = round_pow2 (segment_size, clib_mem_get_page_size ());
139 :
140 296 : seg_name = format (0, "seg-%u-%u-%u%c", app_wrk->app_index,
141 296 : app_wrk->wrk_index, smm->seg_name_counter++, 0);
142 :
143 296 : fs->ssvm.ssvm_size = segment_size;
144 296 : fs->ssvm.name = seg_name;
145 296 : fs->ssvm.requested_va = 0;
146 :
147 296 : if ((rv = ssvm_server_init (&fs->ssvm, props->segment_type)))
148 : {
149 0 : clib_warning ("svm_master_init ('%v', %u) failed", seg_name,
150 : segment_size);
151 0 : pool_put (sm->segments, fs);
152 0 : goto done;
153 : }
154 :
155 : /*
156 : * Initialize fifo segment
157 : */
158 296 : fs->n_slices = props->n_slices;
159 296 : fifo_segment_init (fs);
160 :
161 : /*
162 : * Save segment index before dropping lock, if any held
163 : */
164 296 : fs_index = fs - sm->segments;
165 296 : fs->fs_index = fs_index;
166 296 : fs->sm_index = segment_manager_index (sm);
167 :
168 : /*
169 : * Set watermarks in segment
170 : */
171 296 : fs->high_watermark = sm->high_watermark;
172 296 : fs->low_watermark = sm->low_watermark;
173 296 : fs->flags = flags;
174 296 : fs->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT;
175 296 : fs->h->pct_first_alloc = props->pct_first_alloc;
176 :
177 296 : if (notify_app)
178 : {
179 : app_worker_t *app_wrk;
180 : u64 fs_handle;
181 70 : fs_handle = segment_manager_segment_handle (sm, fs);
182 70 : app_wrk = app_worker_get (sm->app_wrk_index);
183 70 : rv = app_worker_add_segment_notify (app_wrk, fs_handle);
184 70 : if (rv)
185 : {
186 0 : fs_index = rv;
187 0 : goto done;
188 : }
189 : }
190 296 : done:
191 :
192 296 : if (need_lock)
193 0 : clib_rwlock_writer_unlock (&sm->segments_rwlock);
194 :
195 296 : return fs_index;
196 : }
197 :
198 : int
199 287 : segment_manager_add_segment (segment_manager_t *sm, uword segment_size,
200 : u8 notify_app)
201 : {
202 287 : return segment_manager_add_segment_inline (sm, segment_size, notify_app,
203 : 0 /* flags */, 0 /* need_lock */);
204 : }
205 :
206 : int
207 9 : segment_manager_add_segment2 (segment_manager_t *sm, uword segment_size,
208 : u8 flags)
209 : {
210 9 : return segment_manager_add_segment_inline (sm, segment_size, 0, flags,
211 9 : vlib_num_workers ());
212 : }
213 :
214 : /**
215 : * Remove segment without lock
216 : */
217 : void
218 114 : segment_manager_del_segment (segment_manager_t * sm, fifo_segment_t * fs)
219 : {
220 114 : if (ssvm_type (&fs->ssvm) != SSVM_SEGMENT_PRIVATE)
221 : {
222 58 : if (!segment_manager_app_detached (sm))
223 : {
224 : app_worker_t *app_wrk;
225 : u64 segment_handle;
226 2 : app_wrk = app_worker_get (sm->app_wrk_index);
227 2 : segment_handle = segment_manager_segment_handle (sm, fs);
228 2 : app_worker_del_segment_notify (app_wrk, segment_handle);
229 : }
230 : }
231 :
232 114 : fifo_segment_cleanup (fs);
233 114 : ssvm_delete (&fs->ssvm);
234 :
235 : if (CLIB_DEBUG)
236 114 : clib_memset (fs, 0xfb, sizeof (*fs));
237 114 : pool_put (sm->segments, fs);
238 114 : }
239 :
240 : static fifo_segment_t *
241 51 : segment_manager_get_segment_if_valid (segment_manager_t * sm,
242 : u32 segment_index)
243 : {
244 51 : if (pool_is_free_index (sm->segments, segment_index))
245 0 : return 0;
246 51 : return pool_elt_at_index (sm->segments, segment_index);
247 : }
248 :
249 : /**
250 : * Removes segment after acquiring writer lock
251 : */
252 : static inline void
253 51 : sm_lock_and_del_segment_inline (segment_manager_t *sm, u32 fs_index,
254 : u8 check_if_empty)
255 : {
256 : fifo_segment_t *fs;
257 : u8 is_prealloc;
258 :
259 51 : clib_rwlock_writer_lock (&sm->segments_rwlock);
260 :
261 51 : fs = segment_manager_get_segment_if_valid (sm, fs_index);
262 51 : if (!fs)
263 0 : goto done;
264 :
265 51 : if (check_if_empty && fifo_segment_has_fifos (fs))
266 0 : goto done;
267 :
268 51 : is_prealloc = fifo_segment_flags (fs) & FIFO_SEGMENT_F_IS_PREALLOCATED;
269 51 : if (is_prealloc && !segment_manager_app_detached (sm))
270 0 : goto done;
271 :
272 51 : segment_manager_del_segment (sm, fs);
273 :
274 51 : done:
275 51 : clib_rwlock_writer_unlock (&sm->segments_rwlock);
276 51 : }
277 :
278 : void
279 8 : segment_manager_lock_and_del_segment (segment_manager_t * sm, u32 fs_index)
280 : {
281 8 : sm_lock_and_del_segment_inline (sm, fs_index, 0 /* check_if_empty */);
282 8 : }
283 :
284 : /**
285 : * Reads a segment from the segment manager's pool without lock
286 : */
287 : fifo_segment_t *
288 296 : segment_manager_get_segment (segment_manager_t * sm, u32 segment_index)
289 : {
290 296 : return pool_elt_at_index (sm->segments, segment_index);
291 : }
292 :
293 : u64
294 301 : segment_manager_segment_handle (segment_manager_t * sm,
295 : fifo_segment_t * segment)
296 : {
297 301 : u32 segment_index = segment_manager_segment_index (sm, segment);
298 301 : return (((u64) segment_manager_index (sm) << 32) | segment_index);
299 : }
300 :
301 : u64
302 122 : segment_manager_make_segment_handle (u32 segment_manager_index,
303 : u32 segment_index)
304 : {
305 122 : return (((u64) segment_manager_index << 32) | segment_index);
306 : }
307 :
308 : fifo_segment_t *
309 40 : segment_manager_get_segment_w_handle (u64 segment_handle)
310 : {
311 : u32 sm_index, segment_index;
312 : segment_manager_t *sm;
313 :
314 40 : segment_manager_parse_segment_handle (segment_handle, &sm_index,
315 : &segment_index);
316 40 : sm = segment_manager_get (sm_index);
317 40 : if (!sm || pool_is_free_index (sm->segments, segment_index))
318 0 : return 0;
319 40 : return pool_elt_at_index (sm->segments, segment_index);
320 : }
321 :
322 : /**
323 : * Reads a segment from the segment manager's pool and acquires reader lock
324 : *
325 : * Caller must drop the reader's lock by calling
326 : * @ref segment_manager_segment_reader_unlock once it finishes working with
327 : * the segment.
328 : */
329 : fifo_segment_t *
330 521 : segment_manager_get_segment_w_lock (segment_manager_t * sm, u32 segment_index)
331 : {
332 521 : clib_rwlock_reader_lock (&sm->segments_rwlock);
333 521 : return pool_elt_at_index (sm->segments, segment_index);
334 : }
335 :
336 : void
337 411 : segment_manager_segment_reader_lock (segment_manager_t * sm)
338 : {
339 411 : clib_rwlock_reader_lock (&sm->segments_rwlock);
340 411 : }
341 :
342 : void
343 1015 : segment_manager_segment_reader_unlock (segment_manager_t * sm)
344 : {
345 1015 : clib_rwlock_reader_unlock (&sm->segments_rwlock);
346 1015 : }
347 :
348 : segment_manager_t *
349 294 : segment_manager_alloc (void)
350 : {
351 294 : segment_manager_main_t *smm = &sm_main;
352 : segment_manager_t *sm;
353 :
354 294 : pool_get_zero (smm->segment_managers, sm);
355 294 : clib_rwlock_init (&sm->segments_rwlock);
356 294 : return sm;
357 : }
358 :
359 : int
360 294 : segment_manager_init (segment_manager_t * sm)
361 : {
362 : segment_manager_props_t *props;
363 :
364 294 : props = segment_manager_properties_get (sm);
365 :
366 588 : sm->max_fifo_size = props->max_fifo_size ?
367 294 : props->max_fifo_size : sm_main.default_max_fifo_size;
368 294 : sm->max_fifo_size = clib_max (sm->max_fifo_size, 4096);
369 :
370 294 : segment_manager_set_watermarks (sm,
371 294 : props->high_watermark,
372 294 : props->low_watermark);
373 294 : return 0;
374 : }
375 :
376 : /**
377 : * Initializes segment manager based on options provided.
378 : * Returns error if ssvm segment(s) allocation fails.
379 : */
380 : int
381 216 : segment_manager_init_first (segment_manager_t * sm)
382 : {
383 : segment_manager_props_t *props;
384 : uword first_seg_size;
385 : fifo_segment_t *fs;
386 : int fs_index, i;
387 :
388 216 : segment_manager_init (sm);
389 216 : props = segment_manager_properties_get (sm);
390 216 : first_seg_size = clib_max (props->segment_size,
391 : sm_main.default_segment_size);
392 :
393 216 : if (props->prealloc_fifos)
394 : {
395 72 : u64 approx_total_size, max_seg_size = ((u64) 1 << 32) - (128 << 10);
396 : u32 rx_rounded_data_size, tx_rounded_data_size;
397 72 : u32 prealloc_fifo_pairs = props->prealloc_fifos;
398 : u32 rx_fifo_size, tx_fifo_size, pair_size;
399 : u32 approx_segment_count;
400 :
401 : /* Figure out how many segments should be preallocated */
402 72 : rx_rounded_data_size = (1 << (max_log2 (props->rx_fifo_size)));
403 72 : tx_rounded_data_size = (1 << (max_log2 (props->tx_fifo_size)));
404 :
405 72 : rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size;
406 72 : tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size;
407 72 : pair_size = rx_fifo_size + tx_fifo_size;
408 :
409 72 : approx_total_size = (u64) prealloc_fifo_pairs *pair_size;
410 72 : if (first_seg_size > approx_total_size)
411 72 : max_seg_size = first_seg_size;
412 72 : approx_segment_count = (approx_total_size + (max_seg_size - 1))
413 72 : / max_seg_size;
414 :
415 : /* Allocate the segments */
416 72 : for (i = 0; i < approx_segment_count + 1; i++)
417 : {
418 72 : fs_index = segment_manager_add_segment (sm, max_seg_size, 0);
419 72 : if (fs_index < 0)
420 : {
421 : SESSION_DBG ("Failed to preallocate segment %d", i);
422 0 : return fs_index;
423 : }
424 :
425 72 : fs = segment_manager_get_segment (sm, fs_index);
426 72 : if (i == 0)
427 72 : sm->event_queue = segment_manager_alloc_queue (fs, props);
428 :
429 72 : fifo_segment_preallocate_fifo_pairs (fs,
430 : props->rx_fifo_size,
431 : props->tx_fifo_size,
432 : &prealloc_fifo_pairs);
433 72 : fifo_segment_flags (fs) = FIFO_SEGMENT_F_IS_PREALLOCATED;
434 72 : if (prealloc_fifo_pairs == 0)
435 72 : break;
436 : }
437 72 : return 0;
438 : }
439 :
440 144 : fs_index = segment_manager_add_segment (sm, first_seg_size, 0);
441 144 : if (fs_index < 0)
442 : {
443 : SESSION_DBG ("Failed to allocate segment");
444 0 : return fs_index;
445 : }
446 :
447 144 : fs = segment_manager_get_segment (sm, fs_index);
448 144 : sm->event_queue = segment_manager_alloc_queue (fs, props);
449 :
450 144 : if (props->prealloc_fifo_hdrs)
451 : {
452 : u32 hdrs_per_slice;
453 :
454 : /* Do not preallocate on slice associated to main thread */
455 1 : i = (vlib_num_workers ()? 1 : 0);
456 1 : hdrs_per_slice = props->prealloc_fifo_hdrs / (fs->n_slices - i);
457 :
458 2 : for (; i < fs->n_slices; i++)
459 : {
460 1 : if (fifo_segment_prealloc_fifo_hdrs (fs, i, hdrs_per_slice))
461 0 : return SESSION_E_SEG_CREATE;
462 : }
463 : }
464 :
465 144 : return 0;
466 : }
467 :
468 : void
469 16 : segment_manager_cleanup_detached_listener (segment_manager_t * sm)
470 : {
471 : app_worker_t *app_wrk;
472 :
473 16 : app_wrk = app_worker_get_if_valid (sm->app_wrk_index);
474 16 : if (!app_wrk)
475 12 : return;
476 :
477 4 : app_worker_del_detached_sm (app_wrk, segment_manager_index (sm));
478 : }
479 :
480 : /**
481 : * Cleanup segment manager.
482 : */
483 : void
484 124 : segment_manager_free (segment_manager_t * sm)
485 : {
486 124 : segment_manager_main_t *smm = &sm_main;
487 : fifo_segment_t *fifo_segment;
488 :
489 124 : ASSERT (vlib_get_thread_index () == 0
490 : && !segment_manager_has_fifos (sm)
491 : && segment_manager_app_detached (sm));
492 :
493 124 : if (sm->flags & SEG_MANAGER_F_DETACHED_LISTENER)
494 16 : segment_manager_cleanup_detached_listener (sm);
495 :
496 : /* If we have empty preallocated segments that haven't been removed, remove
497 : * them now. Apart from that, the first segment in the first segment manager
498 : * is not removed when all fifos are removed. It can only be removed when
499 : * the manager is explicitly deleted/detached by the app. */
500 124 : clib_rwlock_writer_lock (&sm->segments_rwlock);
501 :
502 : /* *INDENT-OFF* */
503 187 : pool_foreach (fifo_segment, sm->segments) {
504 63 : segment_manager_del_segment (sm, fifo_segment);
505 : }
506 : /* *INDENT-ON* */
507 :
508 124 : pool_free (sm->segments);
509 124 : clib_rwlock_writer_unlock (&sm->segments_rwlock);
510 :
511 124 : clib_rwlock_free (&sm->segments_rwlock);
512 : if (CLIB_DEBUG)
513 124 : clib_memset (sm, 0xfe, sizeof (*sm));
514 124 : pool_put (smm->segment_managers, sm);
515 124 : }
516 :
517 : static void
518 2 : sm_free_w_index_helper (void *arg)
519 : {
520 2 : u32 sm_index = *(u32 *) arg;
521 : segment_manager_t *sm;
522 :
523 2 : ASSERT (vlib_get_thread_index () == 0);
524 :
525 2 : if ((sm = segment_manager_get_if_valid (sm_index)))
526 2 : segment_manager_free (sm);
527 2 : }
528 :
529 : void
530 35 : segment_manager_free_safe (segment_manager_t *sm)
531 : {
532 35 : if (!vlib_thread_is_main_w_barrier ())
533 : {
534 2 : u32 sm_index = segment_manager_index (sm);
535 2 : vlib_rpc_call_main_thread (sm_free_w_index_helper, (u8 *) & sm_index,
536 : sizeof (sm_index));
537 : }
538 : else
539 : {
540 33 : segment_manager_free (sm);
541 : }
542 35 : }
543 :
544 : void
545 110 : segment_manager_init_free (segment_manager_t * sm)
546 : {
547 110 : ASSERT (vlib_get_thread_index () == 0);
548 :
549 110 : segment_manager_app_detach (sm);
550 110 : if (segment_manager_has_fifos (sm))
551 56 : segment_manager_del_sessions (sm);
552 : else
553 : {
554 54 : ASSERT (!sm->first_is_protected || segment_manager_app_detached (sm));
555 54 : segment_manager_free (sm);
556 : }
557 110 : }
558 :
559 : segment_manager_t *
560 902 : segment_manager_get (u32 index)
561 : {
562 902 : return pool_elt_at_index (sm_main.segment_managers, index);
563 : }
564 :
565 : segment_manager_t *
566 296 : segment_manager_get_if_valid (u32 index)
567 : {
568 296 : if (pool_is_free_index (sm_main.segment_managers, index))
569 12 : return 0;
570 284 : return pool_elt_at_index (sm_main.segment_managers, index);
571 : }
572 :
573 : u32
574 919 : segment_manager_index (segment_manager_t * sm)
575 : {
576 919 : return sm - sm_main.segment_managers;
577 : }
578 :
579 : u8
580 331 : segment_manager_has_fifos (segment_manager_t * sm)
581 : {
582 : fifo_segment_t *seg;
583 331 : u8 first = 1;
584 :
585 : /* *INDENT-OFF* */
586 457 : segment_manager_foreach_segment_w_lock (seg, sm, ({
587 : if (CLIB_DEBUG && !first && !fifo_segment_has_fifos (seg)
588 : && !(fifo_segment_flags (seg) & FIFO_SEGMENT_F_IS_PREALLOCATED))
589 : {
590 : clib_warning ("segment %d has no fifos!",
591 : segment_manager_segment_index (sm, seg));
592 : first = 0;
593 : }
594 : if (fifo_segment_has_fifos (seg))
595 : {
596 : segment_manager_segment_reader_unlock (sm);
597 : return 1;
598 : }
599 : }));
600 : /* *INDENT-ON* */
601 :
602 248 : return 0;
603 : }
604 :
605 : /**
606 : * Initiate disconnects for all sessions 'owned' by a segment manager
607 : */
608 : void
609 56 : segment_manager_del_sessions (segment_manager_t * sm)
610 : {
611 56 : session_handle_t *handles = 0, *handle;
612 : fifo_segment_t *fs;
613 : session_t *session;
614 : int slice_index;
615 : svm_fifo_t *f;
616 :
617 56 : ASSERT (pool_elts (sm->segments) != 0);
618 :
619 : /* Across all fifo segments used by the server */
620 : /* *INDENT-OFF* */
621 405 : segment_manager_foreach_segment_w_lock (fs, sm, ({
622 : for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
623 : {
624 : f = fifo_segment_get_slice_fifo_list (fs, slice_index);
625 :
626 : /*
627 : * Remove any residual sessions from the session lookup table
628 : * Don't bother deleting the individual fifos, we're going to
629 : * throw away the fifo segment in a minute.
630 : */
631 : while (f)
632 : {
633 : session = session_get_if_valid (f->shr->master_session_index,
634 : f->master_thread_index);
635 : if (session)
636 : vec_add1 (handles, session_handle (session));
637 : f = f->next;
638 : }
639 : }
640 :
641 : /* Instead of removing the segment, test when cleaning up disconnected
642 : * sessions if the segment can be removed.
643 : */
644 : }));
645 : /* *INDENT-ON* */
646 :
647 251 : vec_foreach (handle, handles)
648 : {
649 195 : session = session_get_from_handle (*handle);
650 195 : session_close (session);
651 : /* Avoid propagating notifications back to the app */
652 195 : session->app_wrk_index = APP_INVALID_INDEX;
653 : }
654 56 : vec_free (handles);
655 56 : }
656 :
657 : /**
658 : * Initiate disconnects for sessions in specified state 'owned' by a segment
659 : * manager
660 : */
661 : void
662 27 : segment_manager_del_sessions_filter (segment_manager_t *sm,
663 : session_state_t *states)
664 : {
665 27 : session_handle_t *handles = 0, *handle;
666 : fifo_segment_t *fs;
667 : session_t *session;
668 : int slice_index;
669 : svm_fifo_t *f;
670 :
671 27 : ASSERT (pool_elts (sm->segments) != 0);
672 :
673 : /* Across all fifo segments used by the server */
674 145 : segment_manager_foreach_segment_w_lock (
675 : fs, sm, ({
676 : for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
677 : {
678 : f = fifo_segment_get_slice_fifo_list (fs, slice_index);
679 : while (f)
680 : {
681 : session = session_get_if_valid (f->shr->master_session_index,
682 : f->master_thread_index);
683 : if (session)
684 : {
685 : session_state_t *state;
686 : vec_foreach (state, states)
687 : {
688 : if (session->session_state == *state)
689 : {
690 : vec_add1 (handles, session_handle (session));
691 : break;
692 : }
693 : }
694 : }
695 : f = f->next;
696 : }
697 : }
698 : }));
699 :
700 27 : vec_foreach (handle, handles)
701 : {
702 0 : session = session_get_from_handle (*handle);
703 0 : session_close (session);
704 : /* Avoid propagating notifications back to the app */
705 0 : session->app_wrk_index = APP_INVALID_INDEX;
706 : }
707 27 : vec_free (handles);
708 27 : }
709 :
710 : int
711 424 : segment_manager_try_alloc_fifos (fifo_segment_t *fs, u32 thread_index,
712 : u32 rx_fifo_size, u32 tx_fifo_size,
713 : svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo)
714 : {
715 424 : rx_fifo_size = clib_max (rx_fifo_size, sm_main.default_fifo_size);
716 424 : *rx_fifo = fifo_segment_alloc_fifo_w_slice (fs, thread_index, rx_fifo_size,
717 : FIFO_SEGMENT_RX_FIFO);
718 :
719 424 : tx_fifo_size = clib_max (tx_fifo_size, sm_main.default_fifo_size);
720 424 : *tx_fifo = fifo_segment_alloc_fifo_w_slice (fs, thread_index, tx_fifo_size,
721 : FIFO_SEGMENT_TX_FIFO);
722 :
723 424 : if (*rx_fifo == 0)
724 : {
725 : /* This would be very odd, but handle it... */
726 0 : if (*tx_fifo != 0)
727 : {
728 0 : fifo_segment_free_fifo (fs, *tx_fifo);
729 0 : *tx_fifo = 0;
730 : }
731 0 : return SESSION_E_SEG_NO_SPACE;
732 : }
733 424 : if (*tx_fifo == 0)
734 : {
735 0 : if (*rx_fifo != 0)
736 : {
737 0 : fifo_segment_free_fifo (fs, *rx_fifo);
738 0 : *rx_fifo = 0;
739 : }
740 0 : return SESSION_E_SEG_NO_SPACE;
741 : }
742 :
743 424 : return 0;
744 : }
745 :
746 : static inline int
747 481 : sm_lookup_segment_and_alloc_fifos (segment_manager_t *sm,
748 : segment_manager_props_t *props,
749 : u32 thread_index, svm_fifo_t **rx_fifo,
750 : svm_fifo_t **tx_fifo)
751 : {
752 : uword free_bytes, max_free_bytes;
753 481 : fifo_segment_t *cur, *fs = 0;
754 :
755 481 : max_free_bytes = props->rx_fifo_size + props->tx_fifo_size - 1;
756 :
757 2365 : pool_foreach (cur, sm->segments)
758 : {
759 1884 : if (fifo_segment_flags (cur) & FIFO_SEGMENT_F_CUSTOM_USE)
760 0 : continue;
761 1884 : free_bytes = fifo_segment_available_bytes (cur);
762 1884 : if (free_bytes > max_free_bytes)
763 : {
764 343 : max_free_bytes = free_bytes;
765 343 : fs = cur;
766 : }
767 : }
768 :
769 481 : if (PREDICT_FALSE (!fs))
770 140 : return SESSION_E_SEG_NO_SPACE;
771 :
772 341 : return segment_manager_try_alloc_fifos (
773 : fs, thread_index, props->rx_fifo_size, props->tx_fifo_size, rx_fifo,
774 : tx_fifo);
775 : }
776 :
777 : static int
778 70 : sm_lock_and_alloc_segment_and_fifos (segment_manager_t *sm,
779 : segment_manager_props_t *props,
780 : u32 thread_index, svm_fifo_t **rx_fifo,
781 : svm_fifo_t **tx_fifo)
782 : {
783 : int new_fs_index, rv;
784 : fifo_segment_t *fs;
785 :
786 70 : if (!props->add_segment)
787 0 : return SESSION_E_SEG_NO_SPACE;
788 :
789 70 : clib_rwlock_writer_lock (&sm->segments_rwlock);
790 :
791 : /* Make sure there really is no free space. Another worker might've freed
792 : * some fifos or allocated a segment */
793 70 : rv = sm_lookup_segment_and_alloc_fifos (sm, props, thread_index, rx_fifo,
794 : tx_fifo);
795 70 : if (!rv)
796 0 : goto done;
797 :
798 : new_fs_index =
799 70 : segment_manager_add_segment (sm, 0 /* segment_size*/, 1 /* notify_app */);
800 70 : if (new_fs_index < 0)
801 : {
802 0 : rv = SESSION_E_SEG_CREATE;
803 0 : goto done;
804 : }
805 70 : fs = segment_manager_get_segment (sm, new_fs_index);
806 70 : rv = segment_manager_try_alloc_fifos (fs, thread_index, props->rx_fifo_size,
807 : props->tx_fifo_size, rx_fifo, tx_fifo);
808 70 : if (rv)
809 : {
810 : SESSION_DBG ("Added a segment, still can't allocate a fifo");
811 0 : rv = SESSION_E_SEG_NO_SPACE2;
812 0 : goto done;
813 : }
814 :
815 70 : done:
816 :
817 70 : clib_rwlock_writer_unlock (&sm->segments_rwlock);
818 :
819 70 : return rv;
820 : }
821 :
822 : int
823 411 : segment_manager_alloc_session_fifos (segment_manager_t * sm,
824 : u32 thread_index,
825 : svm_fifo_t ** rx_fifo,
826 : svm_fifo_t ** tx_fifo)
827 : {
828 : segment_manager_props_t *props;
829 : int rv;
830 :
831 411 : props = segment_manager_properties_get (sm);
832 :
833 : /*
834 : * Fast path: find the first segment with enough free space and
835 : * try to allocate the fifos. Done with reader lock
836 : */
837 :
838 411 : segment_manager_segment_reader_lock (sm);
839 :
840 411 : rv = sm_lookup_segment_and_alloc_fifos (sm, props, thread_index, rx_fifo,
841 : tx_fifo);
842 :
843 411 : segment_manager_segment_reader_unlock (sm);
844 :
845 : /*
846 : * Slow path: if no fifo segment or alloc fail grab writer lock and try
847 : * to allocate new segment
848 : */
849 411 : if (PREDICT_FALSE (rv < 0))
850 70 : return sm_lock_and_alloc_segment_and_fifos (sm, props, thread_index,
851 : rx_fifo, tx_fifo);
852 :
853 341 : return 0;
854 : }
855 :
856 : void
857 268 : segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo)
858 : {
859 : segment_manager_t *sm;
860 : fifo_segment_t *fs;
861 : u32 segment_index;
862 268 : u8 try_delete = 0;
863 :
864 268 : if (!rx_fifo || !tx_fifo)
865 0 : return;
866 :
867 : /* Thread that allocated the fifos must be the one to clean them up */
868 268 : ASSERT (rx_fifo->master_thread_index == vlib_get_thread_index () ||
869 : rx_fifo->refcnt > 1 || vlib_thread_is_main_w_barrier ());
870 :
871 : /* It's possible to have no segment manager if the session was removed
872 : * as result of a detach. */
873 268 : if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager)))
874 0 : return;
875 :
876 268 : segment_index = rx_fifo->segment_index;
877 268 : fs = segment_manager_get_segment_w_lock (sm, segment_index);
878 268 : fifo_segment_free_fifo (fs, rx_fifo);
879 268 : fifo_segment_free_fifo (fs, tx_fifo);
880 :
881 : /*
882 : * Try to remove fifo segment if it has no fifos. This can be done only if
883 : * the segment is not the first in the segment manager or if it is first
884 : * and it is not protected. Moreover, if the segment is first and the app
885 : * has detached from the segment manager, remove the segment manager.
886 : */
887 268 : if (!fifo_segment_has_fifos (fs))
888 : {
889 : /* If first, remove only if not protected */
890 78 : try_delete = segment_index != 0 || !sm->first_is_protected;
891 : }
892 :
893 268 : segment_manager_segment_reader_unlock (sm);
894 :
895 268 : if (PREDICT_FALSE (try_delete))
896 : {
897 : /* Only remove if empty after writer lock acquired */
898 43 : sm_lock_and_del_segment_inline (sm, segment_index,
899 : 1 /* check_if_empty */);
900 :
901 : /* Remove segment manager if no sessions and detached from app */
902 43 : if (segment_manager_app_detached (sm)
903 29 : && !segment_manager_has_fifos (sm))
904 29 : segment_manager_free_safe (sm);
905 : }
906 : }
907 :
908 : void
909 0 : segment_manager_detach_fifo (segment_manager_t *sm, svm_fifo_t **f)
910 : {
911 : fifo_segment_t *fs;
912 :
913 0 : fs = segment_manager_get_segment_w_lock (sm, (*f)->segment_index);
914 0 : fifo_segment_detach_fifo (fs, f);
915 0 : segment_manager_segment_reader_unlock (sm);
916 0 : }
917 :
918 : void
919 0 : segment_manager_attach_fifo (segment_manager_t *sm, svm_fifo_t **f,
920 : session_t *s)
921 : {
922 : fifo_segment_t *fs;
923 :
924 0 : fs = segment_manager_get_segment_w_lock (sm, (*f)->segment_index);
925 0 : fifo_segment_attach_fifo (fs, f, s->thread_index);
926 0 : segment_manager_segment_reader_unlock (sm);
927 :
928 0 : (*f)->shr->master_session_index = s->session_index;
929 0 : (*f)->master_thread_index = s->thread_index;
930 0 : }
931 :
932 : u32
933 0 : segment_manager_evt_q_expected_size (u32 q_len)
934 : {
935 : u32 fifo_evt_size, notif_q_size, q_hdrs;
936 : u32 msg_q_sz, fifo_evt_ring_sz, session_ntf_ring_sz;
937 :
938 0 : fifo_evt_size = 1 << max_log2 (sizeof (session_event_t));
939 0 : notif_q_size = clib_max (16, q_len >> 4);
940 :
941 0 : msg_q_sz = q_len * sizeof (svm_msg_q_msg_t);
942 0 : fifo_evt_ring_sz = q_len * fifo_evt_size;
943 0 : session_ntf_ring_sz = notif_q_size * 256;
944 0 : q_hdrs = sizeof (svm_queue_t) + sizeof (svm_msg_q_t);
945 :
946 0 : return (msg_q_sz + fifo_evt_ring_sz + session_ntf_ring_sz + q_hdrs);
947 : }
948 :
949 : /**
950 : * Allocates shm queue in the first segment
951 : *
952 : * Must be called with lock held
953 : */
954 : svm_msg_q_t *
955 216 : segment_manager_alloc_queue (fifo_segment_t * segment,
956 : segment_manager_props_t * props)
957 : {
958 216 : u32 fifo_evt_size, session_evt_size = 256, notif_q_size;
959 216 : svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
960 : svm_msg_q_t *q;
961 :
962 216 : fifo_evt_size = sizeof (session_event_t);
963 216 : notif_q_size = clib_max (16, props->evt_q_size >> 4);
964 : /* *INDENT-OFF* */
965 216 : svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = {
966 216 : {props->evt_q_size, fifo_evt_size, 0},
967 : {notif_q_size, session_evt_size, 0}
968 : };
969 : /* *INDENT-ON* */
970 216 : cfg->consumer_pid = 0;
971 216 : cfg->n_rings = 2;
972 216 : cfg->q_nitems = props->evt_q_size;
973 216 : cfg->ring_cfgs = rc;
974 :
975 216 : q = fifo_segment_msg_q_alloc (segment, 0, cfg);
976 :
977 216 : if (props->use_mq_eventfd)
978 : {
979 0 : if (svm_msg_q_alloc_eventfd (q))
980 0 : clib_warning ("failed to alloc eventfd");
981 : }
982 216 : return q;
983 : }
984 :
985 : svm_msg_q_t *
986 216 : segment_manager_event_queue (segment_manager_t * sm)
987 : {
988 216 : return sm->event_queue;
989 : }
990 :
991 : /**
992 : * Frees shm queue allocated in the first segment
993 : */
994 : void
995 0 : segment_manager_dealloc_queue (segment_manager_t * sm, svm_queue_t * q)
996 : {
997 : fifo_segment_t *segment;
998 : ssvm_shared_header_t *sh;
999 : void *oldheap;
1000 :
1001 0 : ASSERT (!pool_is_free_index (sm->segments, 0));
1002 :
1003 0 : segment = segment_manager_get_segment_w_lock (sm, 0);
1004 0 : sh = segment->ssvm.sh;
1005 :
1006 0 : oldheap = ssvm_push_heap (sh);
1007 0 : svm_queue_free (q);
1008 0 : ssvm_pop_heap (oldheap);
1009 0 : segment_manager_segment_reader_unlock (sm);
1010 0 : }
1011 :
1012 : /*
1013 : * Init segment vm address allocator
1014 : */
1015 : void
1016 49 : segment_manager_main_init (void)
1017 : {
1018 49 : segment_manager_main_t *sm = &sm_main;
1019 :
1020 49 : sm->default_fifo_size = 1 << 12;
1021 49 : sm->default_segment_size = 1 << 20;
1022 49 : sm->default_app_mq_size = 128;
1023 49 : sm->default_max_fifo_size = 4 << 20;
1024 49 : sm->default_high_watermark = 80;
1025 49 : sm->default_low_watermark = 50;
1026 49 : }
1027 :
1028 : static u8 *
1029 0 : format_segment_manager (u8 *s, va_list *args)
1030 : {
1031 0 : segment_manager_t *sm = va_arg (*args, segment_manager_t *);
1032 0 : int verbose = va_arg (*args, int);
1033 : app_worker_t *app_wrk;
1034 : uword max_fifo_size;
1035 : fifo_segment_t *seg;
1036 : application_t *app;
1037 : u8 custom_logic;
1038 :
1039 0 : app_wrk = app_worker_get_if_valid (sm->app_wrk_index);
1040 0 : app = app_wrk ? application_get (app_wrk->app_index) : 0;
1041 0 : custom_logic = (app && (app->cb_fns.fifo_tuning_callback)) ? 1 : 0;
1042 0 : max_fifo_size = sm->max_fifo_size;
1043 :
1044 0 : s = format (s,
1045 : "[%u] %v app-wrk: %u segs: %u max-fifo-sz: %U "
1046 : "wmarks: %u %u %s flags: 0x%x",
1047 : segment_manager_index (sm), app ? app->name : 0,
1048 0 : sm->app_wrk_index, pool_elts (sm->segments), format_memory_size,
1049 0 : max_fifo_size, sm->high_watermark, sm->low_watermark,
1050 0 : custom_logic ? "custom-tuning" : "no-tuning", sm->flags);
1051 :
1052 0 : if (!verbose || !pool_elts (sm->segments))
1053 0 : return s;
1054 :
1055 0 : s = format (s, "\n\n");
1056 :
1057 0 : segment_manager_foreach_segment_w_lock (
1058 : seg, sm, ({ s = format (s, " *%U", format_fifo_segment, seg, verbose); }));
1059 :
1060 0 : return s;
1061 : }
1062 :
1063 : static clib_error_t *
1064 0 : segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input,
1065 : vlib_cli_command_t * cmd)
1066 : {
1067 0 : unformat_input_t _line_input, *line_input = &_line_input;
1068 0 : segment_manager_main_t *smm = &sm_main;
1069 0 : u8 show_segments = 0, verbose = 0;
1070 : segment_manager_t *sm;
1071 0 : u32 sm_index = ~0;
1072 :
1073 0 : if (!unformat_user (input, unformat_line_input, line_input))
1074 : {
1075 0 : vlib_cli_output (vm, "%d segment managers allocated",
1076 0 : pool_elts (smm->segment_managers));
1077 0 : return 0;
1078 : }
1079 :
1080 0 : while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
1081 : {
1082 0 : if (unformat (line_input, "segments"))
1083 0 : show_segments = 1;
1084 0 : else if (unformat (line_input, "verbose"))
1085 0 : verbose = 1;
1086 0 : else if (unformat (line_input, "index %u", &sm_index))
1087 : ;
1088 : else
1089 : {
1090 0 : vlib_cli_output (vm, "unknown input [%U]", format_unformat_error,
1091 : line_input);
1092 0 : goto done;
1093 : }
1094 : }
1095 :
1096 0 : if (!pool_elts (smm->segment_managers))
1097 0 : goto done;
1098 :
1099 0 : if (sm_index != ~0)
1100 : {
1101 0 : sm = segment_manager_get_if_valid (sm_index);
1102 0 : if (!sm)
1103 : {
1104 0 : vlib_cli_output (vm, "segment manager %u not allocated", sm_index);
1105 0 : goto done;
1106 : }
1107 0 : vlib_cli_output (vm, "%U", format_segment_manager, sm, 1 /* verbose */);
1108 0 : goto done;
1109 : }
1110 :
1111 0 : if (verbose || show_segments)
1112 : {
1113 0 : pool_foreach (sm, smm->segment_managers) {
1114 0 : vlib_cli_output (vm, "%U", format_segment_manager, sm,
1115 : show_segments);
1116 : }
1117 :
1118 0 : vlib_cli_output (vm, "\n");
1119 : }
1120 :
1121 0 : done:
1122 :
1123 0 : unformat_free (line_input);
1124 :
1125 0 : return 0;
1126 : }
1127 :
1128 : /* *INDENT-OFF* */
1129 285289 : VLIB_CLI_COMMAND (segment_manager_show_command, static) = {
1130 : .path = "show segment-manager",
1131 : .short_help = "show segment-manager [segments][verbose][index <nn>]",
1132 : .function = segment_manager_show_fn,
1133 : };
1134 : /* *INDENT-ON* */
1135 :
1136 : void
1137 0 : segment_manager_format_sessions (segment_manager_t * sm, int verbose)
1138 : {
1139 0 : vlib_main_t *vm = vlib_get_main ();
1140 : app_worker_t *app_wrk;
1141 : fifo_segment_t *fs;
1142 : const u8 *app_name;
1143 : int slice_index;
1144 0 : u8 *s = 0, *str;
1145 : svm_fifo_t *f;
1146 :
1147 0 : if (!sm)
1148 : {
1149 0 : if (verbose)
1150 0 : vlib_cli_output (vm, "%-" SESSION_CLI_ID_LEN "s%-20s%-15s%-10s",
1151 : "Connection", "App", "API Client", "SegManager");
1152 : else
1153 0 : vlib_cli_output (vm, "%-" SESSION_CLI_ID_LEN "s%-20s", "Connection",
1154 : "App");
1155 0 : return;
1156 : }
1157 :
1158 0 : app_wrk = app_worker_get (sm->app_wrk_index);
1159 0 : app_name = application_name_from_index (app_wrk->app_index);
1160 :
1161 0 : clib_rwlock_reader_lock (&sm->segments_rwlock);
1162 :
1163 : /* *INDENT-OFF* */
1164 0 : pool_foreach (fs, sm->segments) {
1165 0 : for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
1166 : {
1167 0 : f = fifo_segment_get_slice_fifo_list (fs, slice_index);
1168 0 : while (f)
1169 : {
1170 : u32 session_index, thread_index;
1171 : session_t *session;
1172 :
1173 0 : session_index = f->shr->master_session_index;
1174 0 : thread_index = f->master_thread_index;
1175 :
1176 0 : session = session_get (session_index, thread_index);
1177 0 : str = format (0, "%U", format_session, session, verbose);
1178 :
1179 0 : if (verbose)
1180 0 : s = format (s, "%-" SESSION_CLI_ID_LEN "v%-20v%-15u%-10u", str,
1181 : app_name, app_wrk->api_client_index,
1182 : app_wrk->connects_seg_manager);
1183 : else
1184 0 : s = format (s, "%-" SESSION_CLI_ID_LEN "v%-20v", str, app_name);
1185 :
1186 0 : vlib_cli_output (vm, "%v", s);
1187 0 : vec_reset_length (s);
1188 0 : vec_free (str);
1189 :
1190 0 : f = f->next;
1191 : }
1192 0 : vec_free (s);
1193 : }
1194 : }
1195 : /* *INDENT-ON* */
1196 :
1197 0 : clib_rwlock_reader_unlock (&sm->segments_rwlock);
1198 : }
1199 :
1200 : void
1201 294 : segment_manager_set_watermarks (segment_manager_t * sm,
1202 : u8 high_watermark, u8 low_watermark)
1203 : {
1204 294 : ASSERT (high_watermark <= 100 && low_watermark <= 100 &&
1205 : low_watermark <= high_watermark);
1206 :
1207 294 : sm->high_watermark = high_watermark;
1208 294 : sm->low_watermark = low_watermark;
1209 294 : }
1210 :
1211 : /*
1212 : * fd.io coding-style-patch-verification: ON
1213 : *
1214 : * Local Variables:
1215 : * eval: (c-set-style "gnu")
1216 : * End:
1217 : */
|
