Line data Source code
1 : /*
2 : * Copyright (c) 2021 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : /**
17 : * VCL Locked Sessions (VLS) is a wrapper that synchronizes access to VCL APIs
18 : * which are, by construction, not thread safe. To this end, VLS uses
19 : * configuration and heuristics to detect how applications use sessions in
20 : * an attempt to optimize the locking strategy. The modes of operation
21 : * currently supported are the following:
22 : *
23 : * 1) per-process workers
24 : *
25 : * +----------+ +----------+
26 : * | | | |
27 : * | process0 | | process1 |
28 : * | | | |
29 : * +-----+----+ +-----+----+
30 : * | |
31 : * | |
32 : * +-----+----+ +-----+----+
33 : * | | | |
34 : * | vls_wrk0 | | vls_wrk1 |
35 : * | | | |
36 : * +-----+----+ +-----+----+
37 : * | |
38 : * | |
39 : * +-----+----+ +-----+----+
40 : * | | | |
41 : * | vcl_wrk0 | | vcl_wrk1 |
42 : * | | | |
43 : * +----------+ +----------+
44 : *
45 : * 2) per-thread workers 3) single-worker multi-thread
46 : *
47 : * +---------+ +---------+ +---------+ +---------+
48 : * | | | | | | | |
49 : * | thread0 | | thread1 | | thread0 | | thread1 |
50 : * | | | | | | | |
51 : * +--------++ +-+-------+ +--------++ +-+-------+
52 : * | | | |
53 : * | | | |
54 : * +-+------+-+ +-+------+-+
55 : * | | | |
56 : * | vls_wrk0 | | vls_wrk0 |
57 : * | | | |
58 : * +-+------+-+ +----+-----+
59 : * | | |
60 : * | | |
61 : * +--------+-+ +-+--------+ +----+-----+
62 : * | | | | | |
63 : * | vcl_wrk0 | | vcl_wrk1 | | vcl_wrk0 |
64 : * | | | | | |
65 : * +----------+ +----------+ +----------+
66 : *
67 : * 1) per-process workers: intercept fork calls and assume all children
68 : * processes are new workers that must be registered with vcl. VLS
69 : * sessions are cloned and shared between workers. Only shared sessions
70 : * are locked on use and thereby only one process can interact with
71 : * them at a time (explicit sharing).
72 : *
73 : * 2) per-thread workers: each newly detected pthread is assumed to be a new
74 : * worker and is registered with vcl. Enabled via configuration.
75 : * When a thread tries to access a session it does not own, a clone and
76 : * share rpc request is sent to the owning thread via vcl and vpp.
77 : * Consequently, a vls session can map to multiple vcl sessions, one per
78 : * vcl worker. VLS sessions are locked on use (implicit sharing).
79 : *
80 : * 3) single-worker multi-thread: vls does not make any assumptions about
81 : * application threads and therefore implements an aggressive locking
82 : * strategy that limits access to underlying vcl resources based on type
83 : * of interaction and locks vls session on use (implicit sharing).
84 : */
85 :
86 : #include <vcl/vcl_locked.h>
87 : #include <vcl/vcl_private.h>
88 :
89 : typedef struct vls_shared_data_
90 : {
91 : clib_spinlock_t lock; /**< shared data lock */
92 : u32 owner_wrk_index; /**< vcl wrk that owns session */
93 : u32 *workers_subscribed; /**< vec of wrks subscribed to session */
94 : clib_bitmap_t *listeners; /**< bitmap of wrks actively listening */
95 : } vls_shared_data_t;
96 :
97 : typedef struct vcl_locked_session_
98 : {
99 : clib_spinlock_t lock; /**< vls lock when in use */
100 : u32 session_index; /**< vcl session index */
101 : u32 vcl_wrk_index; /**< vcl worker index */
102 : u32 vls_index; /**< index in vls pool */
103 : u32 shared_data_index; /**< shared data index if any */
104 : u32 owner_vcl_wrk_index; /**< vcl wrk of the vls wrk at alloc */
105 : uword *vcl_wrk_index_to_session_index; /**< map vcl wrk to session */
106 : } vcl_locked_session_t;
107 :
108 : typedef struct vls_worker_
109 : {
110 : clib_rwlock_t sh_to_vlsh_table_lock; /**< ht rwlock with mt workers */
111 : vcl_locked_session_t *vls_pool; /**< pool of vls session */
112 : uword *sh_to_vlsh_table; /**< map from vcl sh to vls sh */
113 : u32 *pending_vcl_wrk_cleanup; /**< child vcl wrks to cleanup */
114 : u32 vcl_wrk_index; /**< if 1:1 map vls to vcl wrk */
115 : } vls_worker_t;
116 :
117 : typedef struct vls_local_
118 : {
119 : int vls_wrk_index; /**< vls wrk index, 1 per process */
120 : volatile int vls_mt_n_threads; /**< number of threads detected */
121 : clib_rwlock_t vls_pool_lock; /**< per process/wrk vls pool locks */
122 : pthread_mutex_t vls_mt_mq_mlock; /**< vcl mq lock */
123 : pthread_mutex_t vls_mt_spool_mlock; /**< vcl select or pool lock */
124 : volatile u8 select_mp_check; /**< flag set if select checks done */
125 : } vls_process_local_t;
126 :
127 : static vls_process_local_t vls_local;
128 : static vls_process_local_t *vlsl = &vls_local;
129 :
130 : typedef struct vls_main_
131 : {
132 : vls_worker_t *workers; /**< pool of vls workers */
133 : vls_shared_data_t *shared_data_pool; /**< inter proc pool of shared data */
134 : clib_rwlock_t shared_data_lock; /**< shared data pool lock */
135 : clib_spinlock_t worker_rpc_lock; /**< lock for inter-worker rpcs */
136 : } vls_main_t;
137 :
138 : vls_main_t *vlsm;
139 :
140 : typedef enum
141 : {
142 : VLS_RPC_STATE_INIT,
143 : VLS_RPC_STATE_SUCCESS,
144 : VLS_RPC_STATE_SESSION_NOT_EXIST,
145 : } vls_rpc_state_e;
146 :
147 : typedef enum vls_rpc_msg_type_
148 : {
149 : VLS_RPC_CLONE_AND_SHARE,
150 : VLS_RPC_SESS_CLEANUP,
151 : } vls_rpc_msg_type_e;
152 :
153 : typedef struct vls_rpc_msg_
154 : {
155 : u8 type;
156 : u8 data[0];
157 : } vls_rpc_msg_t;
158 :
159 : typedef struct vls_clone_and_share_msg_
160 : {
161 : u32 vls_index; /**< vls to be shared */
162 : u32 session_index; /**< vcl session to be shared */
163 : u32 origin_vls_wrk; /**< vls worker that initiated the rpc */
164 : u32 origin_vls_index; /**< vls session of the originator */
165 : u32 origin_vcl_wrk; /**< vcl worker that initiated the rpc */
166 : u32 origin_session_index; /**< vcl session of the originator */
167 : } vls_clone_and_share_msg_t;
168 :
169 : typedef struct vls_sess_cleanup_msg_
170 : {
171 : u32 session_index; /**< vcl session to be cleaned */
172 : u32 origin_vcl_wrk; /**< worker that initiated the rpc */
173 : } vls_sess_cleanup_msg_t;
174 :
175 : void vls_send_session_cleanup_rpc (vcl_worker_t * wrk,
176 : u32 dst_wrk_index, u32 dst_session_index);
177 : void vls_send_clone_and_share_rpc (vcl_worker_t *wrk, u32 origin_vls_index,
178 : u32 session_index, u32 vls_wrk_index,
179 : u32 dst_wrk_index, u32 dst_vls_index,
180 : u32 dst_session_index);
181 : static void vls_cleanup_forked_child (vcl_worker_t *wrk,
182 : vcl_worker_t *child_wrk);
183 : static void vls_handle_pending_wrk_cleanup (void);
184 :
185 : static inline u32
186 35938900 : vls_get_worker_index (void)
187 : {
188 35938900 : return vlsl->vls_wrk_index;
189 : }
190 :
191 : static u32
192 0 : vls_shared_data_alloc (void)
193 : {
194 : vls_shared_data_t *vls_shd;
195 : u32 shd_index;
196 :
197 0 : clib_rwlock_writer_lock (&vlsm->shared_data_lock);
198 0 : pool_get_zero (vlsm->shared_data_pool, vls_shd);
199 0 : clib_spinlock_init (&vls_shd->lock);
200 0 : shd_index = vls_shd - vlsm->shared_data_pool;
201 0 : clib_rwlock_writer_unlock (&vlsm->shared_data_lock);
202 :
203 0 : return shd_index;
204 : }
205 :
206 : static u32
207 0 : vls_shared_data_index (vls_shared_data_t * vls_shd)
208 : {
209 0 : return vls_shd - vlsm->shared_data_pool;
210 : }
211 :
212 : vls_shared_data_t *
213 0 : vls_shared_data_get (u32 shd_index)
214 : {
215 0 : if (pool_is_free_index (vlsm->shared_data_pool, shd_index))
216 0 : return 0;
217 0 : return pool_elt_at_index (vlsm->shared_data_pool, shd_index);
218 : }
219 :
220 : static void
221 0 : vls_shared_data_free (u32 shd_index)
222 : {
223 : vls_shared_data_t *vls_shd;
224 :
225 0 : clib_rwlock_writer_lock (&vlsm->shared_data_lock);
226 0 : vls_shd = vls_shared_data_get (shd_index);
227 0 : clib_spinlock_free (&vls_shd->lock);
228 0 : clib_bitmap_free (vls_shd->listeners);
229 0 : vec_free (vls_shd->workers_subscribed);
230 0 : pool_put (vlsm->shared_data_pool, vls_shd);
231 0 : clib_rwlock_writer_unlock (&vlsm->shared_data_lock);
232 0 : }
233 :
234 : static inline void
235 0 : vls_shared_data_pool_rlock (void)
236 : {
237 0 : clib_rwlock_reader_lock (&vlsm->shared_data_lock);
238 0 : }
239 :
240 : static inline void
241 0 : vls_shared_data_pool_runlock (void)
242 : {
243 0 : clib_rwlock_reader_unlock (&vlsm->shared_data_lock);
244 0 : }
245 :
246 : static inline void
247 8233700 : vls_mt_pool_rlock (void)
248 : {
249 8233700 : if (vlsl->vls_mt_n_threads > 1)
250 0 : clib_rwlock_reader_lock (&vlsl->vls_pool_lock);
251 8233700 : }
252 :
253 : static inline void
254 8233700 : vls_mt_pool_runlock (void)
255 : {
256 8233700 : if (vlsl->vls_mt_n_threads > 1)
257 0 : clib_rwlock_reader_unlock (&vlsl->vls_pool_lock);
258 8233700 : }
259 :
260 : static inline void
261 82 : vls_mt_pool_wlock (void)
262 : {
263 82 : if (vlsl->vls_mt_n_threads > 1)
264 0 : clib_rwlock_writer_lock (&vlsl->vls_pool_lock);
265 82 : }
266 :
267 : static inline void
268 82 : vls_mt_pool_wunlock (void)
269 : {
270 82 : if (vlsl->vls_mt_n_threads > 1)
271 0 : clib_rwlock_writer_unlock (&vlsl->vls_pool_lock);
272 82 : }
273 :
274 : typedef enum
275 : {
276 : VLS_MT_OP_READ,
277 : VLS_MT_OP_WRITE,
278 : VLS_MT_OP_SPOOL,
279 : VLS_MT_OP_XPOLL,
280 : } vls_mt_ops_t;
281 :
282 : typedef enum
283 : {
284 : VLS_MT_LOCK_MQ = 1 << 0,
285 : VLS_MT_LOCK_SPOOL = 1 << 1
286 : } vls_mt_lock_type_t;
287 :
288 : static void
289 0 : vls_mt_add (void)
290 : {
291 0 : vlsl->vls_mt_n_threads += 1;
292 :
293 : /* If multi-thread workers are supported, for each new thread register a new
294 : * vcl worker with vpp. Otherwise, all threads use the same vcl worker, so
295 : * update the vcl worker's thread local worker index variable */
296 0 : if (vls_mt_wrk_supported ())
297 : {
298 0 : if (vppcom_worker_register () != VPPCOM_OK)
299 0 : VERR ("failed to register worker");
300 : }
301 : else
302 0 : vcl_set_worker_index (vlsl->vls_wrk_index);
303 0 : }
304 :
305 : static inline void
306 0 : vls_mt_mq_lock (void)
307 : {
308 0 : pthread_mutex_lock (&vlsl->vls_mt_mq_mlock);
309 0 : }
310 :
311 : static inline void
312 0 : vls_mt_mq_unlock (void)
313 : {
314 0 : pthread_mutex_unlock (&vlsl->vls_mt_mq_mlock);
315 0 : }
316 :
317 : static inline void
318 0 : vls_mt_spool_lock (void)
319 : {
320 0 : pthread_mutex_lock (&vlsl->vls_mt_spool_mlock);
321 0 : }
322 :
323 : static inline void
324 0 : vls_mt_create_unlock (void)
325 : {
326 0 : pthread_mutex_unlock (&vlsl->vls_mt_spool_mlock);
327 0 : }
328 :
329 : static void
330 13 : vls_mt_locks_init (void)
331 : {
332 13 : pthread_mutex_init (&vlsl->vls_mt_mq_mlock, NULL);
333 13 : pthread_mutex_init (&vlsl->vls_mt_spool_mlock, NULL);
334 13 : }
335 :
336 : u8
337 9101210 : vls_is_shared (vcl_locked_session_t * vls)
338 : {
339 9101210 : return (vls->shared_data_index != ~0);
340 : }
341 :
342 : static inline void
343 4550600 : vls_lock (vcl_locked_session_t * vls)
344 : {
345 4550600 : if ((vlsl->vls_mt_n_threads > 1) || vls_is_shared (vls))
346 0 : clib_spinlock_lock (&vls->lock);
347 4550600 : }
348 :
349 : static inline void
350 4550570 : vls_unlock (vcl_locked_session_t * vls)
351 : {
352 4550570 : if ((vlsl->vls_mt_n_threads > 1) || vls_is_shared (vls))
353 0 : clib_spinlock_unlock (&vls->lock);
354 4550570 : }
355 :
356 : static inline vcl_session_handle_t
357 4550600 : vls_to_sh (vcl_locked_session_t * vls)
358 : {
359 4550600 : return vcl_session_handle_from_index (vls->session_index);
360 : }
361 :
362 : static inline vcl_session_handle_t
363 3210590 : vls_to_sh_tu (vcl_locked_session_t * vls)
364 : {
365 : vcl_session_handle_t sh;
366 3210590 : sh = vls_to_sh (vls);
367 3210590 : vls_mt_pool_runlock ();
368 3210590 : return sh;
369 : }
370 :
371 : static vls_worker_t *
372 35938900 : vls_worker_get_current (void)
373 : {
374 35938900 : return pool_elt_at_index (vlsm->workers, vls_get_worker_index ());
375 : }
376 :
377 : static void
378 13 : vls_worker_alloc (void)
379 : {
380 : vls_worker_t *wrk;
381 :
382 13 : pool_get_zero (vlsm->workers, wrk);
383 13 : if (vls_mt_wrk_supported ())
384 0 : clib_rwlock_init (&wrk->sh_to_vlsh_table_lock);
385 13 : wrk->vcl_wrk_index = vcl_get_worker_index ();
386 13 : vec_validate (wrk->pending_vcl_wrk_cleanup, 16);
387 13 : vec_reset_length (wrk->pending_vcl_wrk_cleanup);
388 13 : }
389 :
390 : static void
391 13 : vls_worker_free (vls_worker_t * wrk)
392 : {
393 13 : hash_free (wrk->sh_to_vlsh_table);
394 13 : if (vls_mt_wrk_supported ())
395 0 : clib_rwlock_free (&wrk->sh_to_vlsh_table_lock);
396 13 : pool_free (wrk->vls_pool);
397 13 : pool_put (vlsm->workers, wrk);
398 13 : }
399 :
400 : static vls_worker_t *
401 0 : vls_worker_get (u32 wrk_index)
402 : {
403 0 : if (pool_is_free_index (vlsm->workers, wrk_index))
404 0 : return 0;
405 0 : return pool_elt_at_index (vlsm->workers, wrk_index);
406 : }
407 :
408 : static void
409 47 : vls_sh_to_vlsh_table_add (vls_worker_t *wrk, vcl_session_handle_t sh, u32 vlsh)
410 : {
411 47 : if (vls_mt_wrk_supported ())
412 0 : clib_rwlock_writer_lock (&wrk->sh_to_vlsh_table_lock);
413 47 : hash_set (wrk->sh_to_vlsh_table, sh, vlsh);
414 47 : if (vls_mt_wrk_supported ())
415 0 : clib_rwlock_writer_unlock (&wrk->sh_to_vlsh_table_lock);
416 47 : }
417 :
418 : static void
419 35 : vls_sh_to_vlsh_table_del (vls_worker_t *wrk, vcl_session_handle_t sh)
420 : {
421 35 : if (vls_mt_wrk_supported ())
422 0 : clib_rwlock_writer_lock (&wrk->sh_to_vlsh_table_lock);
423 35 : hash_unset (wrk->sh_to_vlsh_table, sh);
424 35 : if (vls_mt_wrk_supported ())
425 0 : clib_rwlock_writer_unlock (&wrk->sh_to_vlsh_table_lock);
426 35 : }
427 :
428 : static uword *
429 472543 : vls_sh_to_vlsh_table_get (vls_worker_t *wrk, vcl_session_handle_t sh)
430 : {
431 472543 : if (vls_mt_wrk_supported ())
432 0 : clib_rwlock_reader_lock (&wrk->sh_to_vlsh_table_lock);
433 472543 : uword *vlshp = hash_get (wrk->sh_to_vlsh_table, sh);
434 472543 : if (vls_mt_wrk_supported ())
435 0 : clib_rwlock_reader_unlock (&wrk->sh_to_vlsh_table_lock);
436 472543 : return vlshp;
437 : }
438 :
439 : static vls_handle_t
440 47 : vls_alloc (vcl_session_handle_t sh)
441 : {
442 47 : vls_worker_t *wrk = vls_worker_get_current ();
443 : vcl_locked_session_t *vls;
444 :
445 47 : vls_mt_pool_wlock ();
446 :
447 47 : pool_get_zero (wrk->vls_pool, vls);
448 47 : vls->session_index = vppcom_session_index (sh);
449 47 : vls->vcl_wrk_index = vppcom_session_worker (sh);
450 47 : vls->vls_index = vls - wrk->vls_pool;
451 47 : vls->shared_data_index = ~0;
452 47 : vls_sh_to_vlsh_table_add (wrk, sh, vls->vls_index);
453 47 : if (vls_mt_wrk_supported ())
454 : {
455 0 : hash_set (vls->vcl_wrk_index_to_session_index, vls->vcl_wrk_index,
456 : vls->session_index);
457 0 : vls->owner_vcl_wrk_index = vls->vcl_wrk_index;
458 : }
459 47 : clib_spinlock_init (&vls->lock);
460 :
461 47 : vls_mt_pool_wunlock ();
462 47 : return vls->vls_index;
463 : }
464 :
465 : static vcl_locked_session_t *
466 3210600 : vls_get (vls_handle_t vlsh)
467 : {
468 3210600 : vls_worker_t *wrk = vls_worker_get_current ();
469 3210600 : if (pool_is_free_index (wrk->vls_pool, vlsh))
470 0 : return 0;
471 3210600 : return pool_elt_at_index (wrk->vls_pool, vlsh);
472 : }
473 :
474 : static void
475 35 : vls_free (vcl_locked_session_t * vls)
476 : {
477 35 : vls_worker_t *wrk = vls_worker_get_current ();
478 :
479 35 : ASSERT (vls != 0);
480 35 : vls_sh_to_vlsh_table_del (
481 : wrk, vcl_session_handle_from_index (vls->session_index));
482 35 : clib_spinlock_free (&vls->lock);
483 35 : pool_put (wrk->vls_pool, vls);
484 35 : }
485 :
486 : static vcl_locked_session_t *
487 4550600 : vls_get_and_lock (vls_handle_t vlsh)
488 : {
489 4550600 : vls_worker_t *wrk = vls_worker_get_current ();
490 : vcl_locked_session_t *vls;
491 4550600 : if (pool_is_free_index (wrk->vls_pool, vlsh))
492 0 : return 0;
493 4550600 : vls = pool_elt_at_index (wrk->vls_pool, vlsh);
494 4550600 : vls_lock (vls);
495 4550600 : return vls;
496 : }
497 :
498 : static vcl_locked_session_t *
499 4550560 : vls_get_w_dlock (vls_handle_t vlsh)
500 : {
501 : vcl_locked_session_t *vls;
502 4550560 : vls_mt_pool_rlock ();
503 4550560 : vls = vls_get_and_lock (vlsh);
504 4550560 : if (!vls)
505 0 : vls_mt_pool_runlock ();
506 4550560 : return vls;
507 : }
508 :
509 : static inline void
510 3210590 : vls_get_and_unlock (vls_handle_t vlsh)
511 : {
512 : vcl_locked_session_t *vls;
513 3210590 : vls_mt_pool_rlock ();
514 3210590 : vls = vls_get (vlsh);
515 3210590 : vls_unlock (vls);
516 3210590 : vls_mt_pool_runlock ();
517 3210590 : }
518 :
519 : static inline void
520 1339970 : vls_dunlock (vcl_locked_session_t * vls)
521 : {
522 1339970 : vls_unlock (vls);
523 1339970 : vls_mt_pool_runlock ();
524 1339970 : }
525 :
526 : static vcl_locked_session_t *
527 0 : vls_session_get (vls_worker_t * wrk, u32 vls_index)
528 : {
529 0 : if (pool_is_free_index (wrk->vls_pool, vls_index))
530 0 : return 0;
531 0 : return pool_elt_at_index (wrk->vls_pool, vls_index);
532 : }
533 :
534 : vcl_session_handle_t
535 1339970 : vlsh_to_sh (vls_handle_t vlsh)
536 : {
537 : vcl_locked_session_t *vls;
538 : int rv;
539 :
540 1339970 : vls = vls_get_w_dlock (vlsh);
541 1339970 : if (!vls)
542 0 : return INVALID_SESSION_ID;
543 1339970 : rv = vls_to_sh (vls);
544 1339970 : vls_dunlock (vls);
545 1339970 : return rv;
546 : }
547 :
548 : vcl_session_handle_t
549 1339970 : vlsh_to_session_index (vls_handle_t vlsh)
550 : {
551 : vcl_session_handle_t sh;
552 1339970 : sh = vlsh_to_sh (vlsh);
553 1339970 : return vppcom_session_index (sh);
554 : }
555 :
556 : vls_handle_t
557 472543 : vls_si_wi_to_vlsh (u32 session_index, u32 vcl_wrk_index)
558 : {
559 472543 : vls_worker_t *wrk = vls_worker_get_current ();
560 472543 : uword *vlshp = vls_sh_to_vlsh_table_get (
561 : wrk,
562 : vcl_session_handle_from_wrk_session_index (session_index, vcl_wrk_index));
563 :
564 472543 : return vlshp ? *vlshp : VLS_INVALID_HANDLE;
565 : }
566 :
567 : vls_handle_t
568 472543 : vls_session_index_to_vlsh (uint32_t session_index)
569 : {
570 : vls_handle_t vlsh;
571 :
572 472543 : vls_mt_pool_rlock ();
573 472543 : vlsh = vls_si_wi_to_vlsh (session_index, vcl_get_worker_index ());
574 472543 : vls_mt_pool_runlock ();
575 :
576 472543 : return vlsh;
577 : }
578 :
579 : u8
580 0 : vls_is_shared_by_wrk (vcl_locked_session_t * vls, u32 wrk_index)
581 : {
582 : vls_shared_data_t *vls_shd;
583 : int i;
584 :
585 0 : if (vls->shared_data_index == ~0)
586 0 : return 0;
587 :
588 0 : vls_shared_data_pool_rlock ();
589 :
590 0 : vls_shd = vls_shared_data_get (vls->shared_data_index);
591 0 : clib_spinlock_lock (&vls_shd->lock);
592 :
593 0 : for (i = 0; i < vec_len (vls_shd->workers_subscribed); i++)
594 0 : if (vls_shd->workers_subscribed[i] == wrk_index)
595 : {
596 0 : clib_spinlock_unlock (&vls_shd->lock);
597 0 : vls_shared_data_pool_runlock ();
598 0 : return 1;
599 : }
600 0 : clib_spinlock_unlock (&vls_shd->lock);
601 :
602 0 : vls_shared_data_pool_runlock ();
603 0 : return 0;
604 : }
605 :
606 : static void
607 0 : vls_listener_wrk_set (vcl_locked_session_t * vls, u32 wrk_index, u8 is_active)
608 : {
609 : vls_shared_data_t *vls_shd;
610 :
611 0 : if (vls->shared_data_index == ~0)
612 : {
613 0 : clib_warning ("not a shared session");
614 0 : return;
615 : }
616 :
617 0 : vls_shared_data_pool_rlock ();
618 :
619 0 : vls_shd = vls_shared_data_get (vls->shared_data_index);
620 :
621 0 : clib_spinlock_lock (&vls_shd->lock);
622 0 : vls_shd->listeners =
623 0 : clib_bitmap_set (vls_shd->listeners, wrk_index, is_active);
624 0 : clib_spinlock_unlock (&vls_shd->lock);
625 :
626 0 : vls_shared_data_pool_runlock ();
627 : }
628 :
629 : static u32
630 0 : vls_shared_get_owner (vcl_locked_session_t * vls)
631 : {
632 : vls_shared_data_t *vls_shd;
633 : u32 owner_wrk;
634 :
635 0 : vls_shared_data_pool_rlock ();
636 :
637 0 : vls_shd = vls_shared_data_get (vls->shared_data_index);
638 0 : owner_wrk = vls_shd->owner_wrk_index;
639 :
640 0 : vls_shared_data_pool_runlock ();
641 :
642 0 : return owner_wrk;
643 : }
644 :
645 : static u8
646 0 : vls_listener_wrk_is_active (vcl_locked_session_t * vls, u32 wrk_index)
647 : {
648 : vls_shared_data_t *vls_shd;
649 : u8 is_set;
650 :
651 0 : if (vls->shared_data_index == ~0)
652 : {
653 0 : clib_warning ("not a shared session");
654 0 : return 0;
655 : }
656 :
657 0 : vls_shared_data_pool_rlock ();
658 :
659 0 : vls_shd = vls_shared_data_get (vls->shared_data_index);
660 :
661 0 : clib_spinlock_lock (&vls_shd->lock);
662 0 : is_set = clib_bitmap_get (vls_shd->listeners, wrk_index);
663 0 : clib_spinlock_unlock (&vls_shd->lock);
664 :
665 0 : vls_shared_data_pool_runlock ();
666 :
667 0 : return (is_set == 1);
668 : }
669 :
670 : static void
671 0 : vls_listener_wrk_start_listen (vcl_locked_session_t * vls, u32 wrk_index)
672 : {
673 : vcl_worker_t *wrk;
674 : vcl_session_t *ls;
675 :
676 0 : wrk = vcl_worker_get (wrk_index);
677 0 : ls = vcl_session_get (wrk, vls->session_index);
678 :
679 : /* Listen request already sent */
680 0 : if (ls->flags & VCL_SESSION_F_PENDING_LISTEN)
681 0 : return;
682 :
683 0 : vcl_send_session_listen (wrk, ls);
684 :
685 0 : vls_listener_wrk_set (vls, wrk_index, 1 /* is_active */);
686 : }
687 :
688 : static void
689 0 : vls_listener_wrk_stop_listen (vcl_locked_session_t * vls, u32 wrk_index)
690 : {
691 : vcl_worker_t *wrk;
692 : vcl_session_t *s;
693 :
694 0 : wrk = vcl_worker_get (wrk_index);
695 0 : s = vcl_session_get (wrk, vls->session_index);
696 0 : if (s->session_state != VCL_STATE_LISTEN)
697 0 : return;
698 0 : vcl_send_session_unlisten (wrk, s);
699 0 : s->session_state = VCL_STATE_LISTEN_NO_MQ;
700 0 : vls_listener_wrk_set (vls, wrk_index, 0 /* is_active */ );
701 : }
702 :
703 : static int
704 0 : vls_shared_data_subscriber_position (vls_shared_data_t * vls_shd,
705 : u32 wrk_index)
706 : {
707 : int i;
708 :
709 0 : for (i = 0; i < vec_len (vls_shd->workers_subscribed); i++)
710 : {
711 0 : if (vls_shd->workers_subscribed[i] == wrk_index)
712 0 : return i;
713 : }
714 0 : return -1;
715 : }
716 :
717 : int
718 0 : vls_unshare_session (vcl_locked_session_t * vls, vcl_worker_t * wrk)
719 : {
720 : vls_shared_data_t *vls_shd;
721 : int do_disconnect, pos;
722 : u32 n_subscribers;
723 : vcl_session_t *s;
724 :
725 0 : if (vls->shared_data_index == ~0)
726 0 : return 0;
727 :
728 0 : s = vcl_session_get (wrk, vls->session_index);
729 0 : if (s->session_state == VCL_STATE_LISTEN)
730 0 : vls_listener_wrk_set (vls, wrk->wrk_index, 0 /* is_active */ );
731 :
732 0 : vls_shared_data_pool_rlock ();
733 :
734 0 : vls_shd = vls_shared_data_get (vls->shared_data_index);
735 0 : clib_spinlock_lock (&vls_shd->lock);
736 :
737 0 : pos = vls_shared_data_subscriber_position (vls_shd, wrk->wrk_index);
738 0 : if (pos < 0)
739 : {
740 0 : clib_warning ("worker %u not subscribed for vls %u", wrk->wrk_index,
741 : vls->vcl_wrk_index);
742 0 : goto done;
743 : }
744 :
745 : /*
746 : * Unsubscribe from share data and fifos
747 : */
748 0 : if (s->rx_fifo)
749 : {
750 0 : svm_fifo_del_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
751 0 : svm_fifo_del_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
752 : }
753 0 : vec_del1 (vls_shd->workers_subscribed, pos);
754 :
755 : /*
756 : * Cleanup vcl state
757 : */
758 0 : n_subscribers = vec_len (vls_shd->workers_subscribed);
759 0 : do_disconnect = s->session_state == VCL_STATE_LISTEN || !n_subscribers;
760 0 : vcl_session_cleanup (wrk, s, vcl_session_handle (s), do_disconnect);
761 :
762 : /*
763 : * No subscriber left, cleanup shared data
764 : */
765 0 : if (!n_subscribers)
766 : {
767 0 : u32 shd_index = vls_shared_data_index (vls_shd);
768 :
769 0 : clib_spinlock_unlock (&vls_shd->lock);
770 0 : vls_shared_data_pool_runlock ();
771 :
772 0 : vls_shared_data_free (shd_index);
773 :
774 : /* All locks have been dropped */
775 0 : return 0;
776 : }
777 :
778 : /* Return, if this is not the owning worker */
779 0 : if (vls_shd->owner_wrk_index != wrk->wrk_index)
780 0 : goto done;
781 :
782 0 : ASSERT (vec_len (vls_shd->workers_subscribed));
783 :
784 : /*
785 : * Check if we can change owner or close
786 : */
787 0 : vls_shd->owner_wrk_index = vls_shd->workers_subscribed[0];
788 0 : if (s->vpp_evt_q)
789 0 : vcl_send_session_worker_update (wrk, s, vls_shd->owner_wrk_index);
790 :
791 : /* XXX is this still needed? */
792 0 : if (vec_len (vls_shd->workers_subscribed) > 1)
793 0 : clib_warning ("more workers need to be updated");
794 :
795 0 : done:
796 :
797 0 : clib_spinlock_unlock (&vls_shd->lock);
798 0 : vls_shared_data_pool_runlock ();
799 :
800 0 : return 0;
801 : }
802 :
803 : void
804 0 : vls_init_share_session (vls_worker_t * vls_wrk, vcl_locked_session_t * vls)
805 : {
806 : vls_shared_data_t *vls_shd;
807 :
808 0 : u32 vls_shd_index = vls_shared_data_alloc ();
809 :
810 0 : vls_shared_data_pool_rlock ();
811 :
812 0 : vls_shd = vls_shared_data_get (vls_shd_index);
813 0 : vls_shd->owner_wrk_index = vls_wrk->vcl_wrk_index;
814 0 : vls->shared_data_index = vls_shd_index;
815 0 : vec_add1 (vls_shd->workers_subscribed, vls_wrk->vcl_wrk_index);
816 :
817 0 : vls_shared_data_pool_runlock ();
818 0 : }
819 :
820 : void
821 0 : vls_share_session (vls_worker_t * vls_wrk, vcl_locked_session_t * vls)
822 : {
823 0 : vcl_worker_t *vcl_wrk = vcl_worker_get (vls_wrk->vcl_wrk_index);
824 : vls_shared_data_t *vls_shd;
825 : vcl_session_t *s;
826 :
827 0 : s = vcl_session_get (vcl_wrk, vls->session_index);
828 0 : if (!s)
829 : {
830 0 : clib_warning ("wrk %u session %u vls %u NOT AVAILABLE",
831 : vcl_wrk->wrk_index, vls->session_index, vls->vls_index);
832 0 : return;
833 : }
834 :
835 0 : ASSERT (vls->shared_data_index != ~0);
836 :
837 : /* Reinit session lock */
838 0 : clib_spinlock_init (&vls->lock);
839 :
840 0 : vls_shared_data_pool_rlock ();
841 :
842 0 : vls_shd = vls_shared_data_get (vls->shared_data_index);
843 :
844 0 : clib_spinlock_lock (&vls_shd->lock);
845 0 : vec_add1 (vls_shd->workers_subscribed, vls_wrk->vcl_wrk_index);
846 0 : clib_spinlock_unlock (&vls_shd->lock);
847 :
848 0 : vls_shared_data_pool_runlock ();
849 :
850 0 : if (s->rx_fifo)
851 : {
852 0 : vcl_session_share_fifos (s, s->rx_fifo, s->tx_fifo);
853 : }
854 0 : else if (s->session_state == VCL_STATE_LISTEN)
855 : {
856 0 : s->session_state = VCL_STATE_LISTEN_NO_MQ;
857 : }
858 : }
859 :
860 : static void
861 0 : vls_share_sessions (vls_worker_t * vls_parent_wrk, vls_worker_t * vls_wrk)
862 : {
863 : vcl_locked_session_t *vls, *parent_vls;
864 :
865 : /* *INDENT-OFF* */
866 0 : pool_foreach (vls, vls_wrk->vls_pool) {
867 : /* Initialize sharing on parent session */
868 0 : if (vls->shared_data_index == ~0)
869 : {
870 0 : parent_vls = vls_session_get (vls_parent_wrk, vls->vls_index);
871 0 : vls_init_share_session (vls_parent_wrk, parent_vls);
872 0 : vls->shared_data_index = parent_vls->shared_data_index;
873 : }
874 0 : vls_share_session (vls_wrk, vls);
875 : }
876 : /* *INDENT-ON* */
877 0 : }
878 :
879 : static void
880 0 : vls_validate_veps (vcl_worker_t *wrk)
881 : {
882 : vcl_session_t *s;
883 : u32 session_index, wrk_index;
884 :
885 0 : pool_foreach (s, wrk->sessions)
886 : {
887 0 : if (s->vep.vep_sh != ~0)
888 : {
889 0 : vcl_session_handle_parse (s->vep.vep_sh, &wrk_index, &session_index);
890 0 : s->vep.vep_sh = vcl_session_handle_from_index (session_index);
891 : }
892 0 : if (s->vep.next_sh != ~0)
893 : {
894 0 : vcl_session_handle_parse (s->vep.next_sh, &wrk_index,
895 : &session_index);
896 0 : s->vep.next_sh = vcl_session_handle_from_index (session_index);
897 : }
898 0 : if (s->vep.prev_sh != ~0)
899 : {
900 0 : vcl_session_handle_parse (s->vep.prev_sh, &wrk_index,
901 : &session_index);
902 0 : s->vep.prev_sh = vcl_session_handle_from_index (session_index);
903 : }
904 : }
905 0 : }
906 :
907 : void
908 0 : vls_worker_copy_on_fork (vcl_worker_t * parent_wrk)
909 : {
910 0 : vls_worker_t *vls_wrk = vls_worker_get_current (), *vls_parent_wrk;
911 0 : vcl_worker_t *vcl_wrk = vcl_worker_get_current ();
912 : u32 vls_index, session_index, wrk_index;
913 : vcl_session_handle_t sh;
914 : vcl_locked_session_t *vls;
915 :
916 : /*
917 : * init vcl worker
918 : */
919 0 : vcl_wrk->sessions = pool_dup (parent_wrk->sessions);
920 0 : vcl_wrk->session_index_by_vpp_handles =
921 0 : hash_dup (parent_wrk->session_index_by_vpp_handles);
922 :
923 : /*
924 : * init vls worker
925 : */
926 0 : vls_parent_wrk = vls_worker_get (parent_wrk->wrk_index);
927 :
928 : /* clang-format off */
929 0 : hash_foreach (sh, vls_index, vls_parent_wrk->sh_to_vlsh_table, ({
930 : vcl_session_handle_parse (sh, &wrk_index, &session_index);
931 : hash_set (vls_wrk->sh_to_vlsh_table,
932 : vcl_session_handle_from_index (session_index), vls_index);
933 : }));
934 : /* clang-format on */
935 0 : vls_wrk->vls_pool = pool_dup (vls_parent_wrk->vls_pool);
936 :
937 : /*
938 : * Detach vls from parent vcl worker and attach them to child.
939 : */
940 0 : pool_foreach (vls, vls_wrk->vls_pool)
941 : {
942 0 : vls->vcl_wrk_index = vcl_wrk->wrk_index;
943 : }
944 :
945 : /* Validate vep's handle */
946 0 : vls_validate_veps (vcl_wrk);
947 :
948 0 : vls_share_sessions (vls_parent_wrk, vls_wrk);
949 0 : }
950 :
951 : static void
952 0 : vls_mt_acq_locks (vcl_locked_session_t * vls, vls_mt_ops_t op, int *locks_acq)
953 : {
954 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
955 0 : vcl_session_t *s = 0;
956 0 : int is_nonblk = 0;
957 :
958 0 : if (vls)
959 : {
960 0 : s = vcl_session_get (wrk, vls->session_index);
961 0 : if (PREDICT_FALSE (!s))
962 0 : return;
963 0 : is_nonblk = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
964 : }
965 :
966 0 : switch (op)
967 : {
968 0 : case VLS_MT_OP_READ:
969 0 : if (!is_nonblk)
970 0 : is_nonblk = vcl_session_read_ready (s) != 0;
971 0 : if (!is_nonblk)
972 : {
973 0 : vls_mt_mq_lock ();
974 0 : *locks_acq |= VLS_MT_LOCK_MQ;
975 : }
976 0 : break;
977 0 : case VLS_MT_OP_WRITE:
978 0 : ASSERT (s);
979 0 : if (!is_nonblk)
980 0 : is_nonblk = vcl_session_write_ready (s) != 0;
981 0 : if (!is_nonblk)
982 : {
983 0 : vls_mt_mq_lock ();
984 0 : *locks_acq |= VLS_MT_LOCK_MQ;
985 : }
986 0 : break;
987 0 : case VLS_MT_OP_XPOLL:
988 0 : vls_mt_mq_lock ();
989 0 : *locks_acq |= VLS_MT_LOCK_MQ;
990 0 : break;
991 0 : case VLS_MT_OP_SPOOL:
992 0 : vls_mt_spool_lock ();
993 0 : *locks_acq |= VLS_MT_LOCK_SPOOL;
994 0 : break;
995 0 : default:
996 0 : break;
997 : }
998 : }
999 :
1000 : static void
1001 0 : vls_mt_rel_locks (int locks_acq)
1002 : {
1003 0 : if (locks_acq & VLS_MT_LOCK_MQ)
1004 0 : vls_mt_mq_unlock ();
1005 0 : if (locks_acq & VLS_MT_LOCK_SPOOL)
1006 0 : vls_mt_create_unlock ();
1007 0 : }
1008 :
1009 : static inline u8
1010 660 : vls_mt_session_should_migrate (vcl_locked_session_t * vls)
1011 : {
1012 660 : return (vls_mt_wrk_supported () &&
1013 0 : vls->vcl_wrk_index != vcl_get_worker_index ());
1014 : }
1015 :
1016 : static vcl_locked_session_t *
1017 0 : vls_mt_session_migrate (vcl_locked_session_t *vls)
1018 : {
1019 0 : u32 wrk_index = vcl_get_worker_index ();
1020 : vcl_worker_t *wrk;
1021 0 : vls_worker_t *vls_wrk = vls_worker_get_current ();
1022 : u32 src_sid, sid, vls_index, own_vcl_wrk_index;
1023 : vcl_session_t *session;
1024 : uword *p;
1025 :
1026 0 : ASSERT (vls_mt_wrk_supported () && vls->vcl_wrk_index != wrk_index);
1027 :
1028 : /*
1029 : * VCL session on current vcl worker already allocated. Update current
1030 : * owner worker and index and return
1031 : */
1032 0 : if ((p = hash_get (vls->vcl_wrk_index_to_session_index, wrk_index)))
1033 : {
1034 0 : vls->vcl_wrk_index = wrk_index;
1035 0 : vls->session_index = (u32) p[0];
1036 0 : return vls;
1037 : }
1038 :
1039 : /*
1040 : * Ask vcl worker that owns the original vcl session to clone it into
1041 : * current vcl worker session pool
1042 : */
1043 :
1044 0 : if (!(p = hash_get (vls->vcl_wrk_index_to_session_index,
1045 : vls->owner_vcl_wrk_index)))
1046 : {
1047 0 : VERR ("session in owner worker(%u) is free", vls->owner_vcl_wrk_index);
1048 0 : ASSERT (0);
1049 0 : vls_unlock (vls);
1050 0 : vls_mt_pool_runlock ();
1051 0 : return 0;
1052 : }
1053 :
1054 0 : src_sid = (u32) p[0];
1055 0 : wrk = vcl_worker_get_current ();
1056 0 : session = vcl_session_alloc (wrk);
1057 0 : sid = session->session_index;
1058 0 : VDBG (1, "migrate session of worker (session): %u (%u) -> %u (%u)",
1059 : vls->owner_vcl_wrk_index, src_sid, wrk_index, sid);
1060 :
1061 : /* Drop lock to prevent dead lock when dst wrk trying to get lock. */
1062 0 : vls_index = vls->vls_index;
1063 0 : own_vcl_wrk_index = vls->owner_vcl_wrk_index;
1064 0 : vls_unlock (vls);
1065 0 : vls_mt_pool_runlock ();
1066 0 : vls_send_clone_and_share_rpc (wrk, vls_index, sid, vls_get_worker_index (),
1067 : own_vcl_wrk_index, vls_index, src_sid);
1068 :
1069 0 : if (PREDICT_FALSE (wrk->rpc_done == VLS_RPC_STATE_SESSION_NOT_EXIST))
1070 : {
1071 0 : VWRN ("session %u not exist", src_sid);
1072 0 : goto err;
1073 : }
1074 0 : else if (PREDICT_FALSE (wrk->rpc_done == VLS_RPC_STATE_INIT))
1075 : {
1076 0 : VWRN ("failed to wait rpc response");
1077 0 : goto err;
1078 : }
1079 0 : else if (PREDICT_FALSE ((session->flags & VCL_SESSION_F_IS_VEP) &&
1080 : session->vep.next_sh != ~0))
1081 : {
1082 0 : VERR ("can't migrate nonempty epoll session");
1083 0 : ASSERT (0);
1084 0 : goto err;
1085 : }
1086 0 : else if (PREDICT_FALSE (!(session->flags & VCL_SESSION_F_IS_VEP) &&
1087 : session->session_state != VCL_STATE_CLOSED))
1088 : {
1089 0 : VERR ("migrate NOT supported, session_status (%u)",
1090 : session->session_state);
1091 0 : ASSERT (0);
1092 0 : goto err;
1093 : }
1094 :
1095 0 : vls = vls_get_w_dlock (vls_index);
1096 0 : if (PREDICT_FALSE (!vls))
1097 : {
1098 0 : VWRN ("failed to get vls %u", vls_index);
1099 0 : goto err;
1100 : }
1101 :
1102 0 : session->session_index = sid;
1103 0 : vls->vcl_wrk_index = wrk_index;
1104 0 : vls->session_index = sid;
1105 0 : hash_set (vls->vcl_wrk_index_to_session_index, wrk_index, sid);
1106 0 : vls_sh_to_vlsh_table_add (vls_wrk, vcl_session_handle (session),
1107 : vls->vls_index);
1108 0 : return vls;
1109 :
1110 0 : err:
1111 0 : vcl_session_free (wrk, session);
1112 0 : return 0;
1113 : }
1114 :
1115 : static inline void
1116 29541700 : vls_mt_detect (void)
1117 : {
1118 29541700 : if (PREDICT_FALSE (vcl_get_worker_index () == ~0))
1119 0 : vls_mt_add ();
1120 29541700 : }
1121 :
1122 : #define vls_mt_guard(_vls, _op) \
1123 : int _locks_acq = 0; \
1124 : if (vls_mt_wrk_supported ()) \
1125 : { \
1126 : if (PREDICT_FALSE (_vls && \
1127 : ((vcl_locked_session_t *) _vls)->vcl_wrk_index != \
1128 : vcl_get_worker_index ())) \
1129 : { \
1130 : _vls = vls_mt_session_migrate (_vls); \
1131 : if (PREDICT_FALSE (!_vls)) \
1132 : return VPPCOM_EBADFD; \
1133 : } \
1134 : } \
1135 : else \
1136 : { \
1137 : if (PREDICT_FALSE (vlsl->vls_mt_n_threads > 1)) \
1138 : vls_mt_acq_locks (_vls, _op, &_locks_acq); \
1139 : }
1140 :
1141 : #define vls_mt_unguard() \
1142 : if (PREDICT_FALSE (_locks_acq)) \
1143 : vls_mt_rel_locks (_locks_acq)
1144 :
1145 : int
1146 0 : vls_write (vls_handle_t vlsh, void *buf, size_t nbytes)
1147 : {
1148 : vcl_locked_session_t *vls;
1149 : int rv;
1150 :
1151 0 : vls_mt_detect ();
1152 0 : if (!(vls = vls_get_w_dlock (vlsh)))
1153 0 : return VPPCOM_EBADFD;
1154 :
1155 0 : vls_mt_guard (vls, VLS_MT_OP_WRITE);
1156 0 : rv = vppcom_session_write (vls_to_sh_tu (vls), buf, nbytes);
1157 0 : vls_mt_unguard ();
1158 0 : vls_get_and_unlock (vlsh);
1159 0 : return rv;
1160 : }
1161 :
1162 : int
1163 1109770 : vls_write_msg (vls_handle_t vlsh, void *buf, size_t nbytes)
1164 : {
1165 : vcl_locked_session_t *vls;
1166 : int rv;
1167 :
1168 1109770 : vls_mt_detect ();
1169 1109770 : if (!(vls = vls_get_w_dlock (vlsh)))
1170 0 : return VPPCOM_EBADFD;
1171 1109770 : vls_mt_guard (vls, VLS_MT_OP_WRITE);
1172 1109770 : rv = vppcom_session_write_msg (vls_to_sh_tu (vls), buf, nbytes);
1173 1109770 : vls_mt_unguard ();
1174 1109770 : vls_get_and_unlock (vlsh);
1175 1109770 : return rv;
1176 : }
1177 :
1178 : int
1179 0 : vls_sendto (vls_handle_t vlsh, void *buf, int buflen, int flags,
1180 : vppcom_endpt_t * ep)
1181 : {
1182 : vcl_locked_session_t *vls;
1183 : int rv;
1184 :
1185 0 : vls_mt_detect ();
1186 0 : if (!(vls = vls_get_w_dlock (vlsh)))
1187 0 : return VPPCOM_EBADFD;
1188 0 : vls_mt_guard (vls, VLS_MT_OP_WRITE);
1189 0 : rv = vppcom_session_sendto (vls_to_sh_tu (vls), buf, buflen, flags, ep);
1190 0 : vls_mt_unguard ();
1191 0 : vls_get_and_unlock (vlsh);
1192 0 : return rv;
1193 : }
1194 :
1195 : ssize_t
1196 726053 : vls_read (vls_handle_t vlsh, void *buf, size_t nbytes)
1197 : {
1198 : vcl_locked_session_t *vls;
1199 : int rv;
1200 :
1201 726053 : vls_mt_detect ();
1202 726053 : if (!(vls = vls_get_w_dlock (vlsh)))
1203 0 : return VPPCOM_EBADFD;
1204 726053 : vls_mt_guard (vls, VLS_MT_OP_READ);
1205 726053 : rv = vppcom_session_read (vls_to_sh_tu (vls), buf, nbytes);
1206 726053 : vls_mt_unguard ();
1207 726053 : vls_get_and_unlock (vlsh);
1208 726053 : return rv;
1209 : }
1210 :
1211 : ssize_t
1212 4 : vls_recvfrom (vls_handle_t vlsh, void *buffer, uint32_t buflen, int flags,
1213 : vppcom_endpt_t * ep)
1214 : {
1215 : vcl_locked_session_t *vls;
1216 : int rv;
1217 :
1218 4 : vls_mt_detect ();
1219 4 : if (!(vls = vls_get_w_dlock (vlsh)))
1220 0 : return VPPCOM_EBADFD;
1221 4 : vls_mt_guard (vls, VLS_MT_OP_READ);
1222 4 : rv = vppcom_session_recvfrom (vls_to_sh_tu (vls), buffer, buflen, flags,
1223 : ep);
1224 4 : vls_mt_unguard ();
1225 4 : vls_get_and_unlock (vlsh);
1226 4 : return rv;
1227 : }
1228 :
1229 : int
1230 657 : vls_attr (vls_handle_t vlsh, uint32_t op, void *buffer, uint32_t * buflen)
1231 : {
1232 : vcl_locked_session_t *vls;
1233 : int rv;
1234 :
1235 657 : vls_mt_detect ();
1236 657 : if (!(vls = vls_get_w_dlock (vlsh)))
1237 0 : return VPPCOM_EBADFD;
1238 657 : if (vls_mt_session_should_migrate (vls))
1239 : {
1240 0 : vls = vls_mt_session_migrate (vls);
1241 0 : if (PREDICT_FALSE (!vls))
1242 0 : return VPPCOM_EBADFD;
1243 : }
1244 657 : rv = vppcom_session_attr (vls_to_sh_tu (vls), op, buffer, buflen);
1245 657 : vls_get_and_unlock (vlsh);
1246 657 : return rv;
1247 : }
1248 :
1249 : int
1250 16 : vls_bind (vls_handle_t vlsh, vppcom_endpt_t * ep)
1251 : {
1252 : vcl_locked_session_t *vls;
1253 : int rv;
1254 :
1255 16 : vls_mt_detect ();
1256 16 : if (!(vls = vls_get_w_dlock (vlsh)))
1257 0 : return VPPCOM_EBADFD;
1258 16 : rv = vppcom_session_bind (vls_to_sh_tu (vls), ep);
1259 16 : vls_get_and_unlock (vlsh);
1260 16 : return rv;
1261 : }
1262 :
1263 : int
1264 13 : vls_listen (vls_handle_t vlsh, int q_len)
1265 : {
1266 : vcl_locked_session_t *vls;
1267 : int rv;
1268 :
1269 13 : vls_mt_detect ();
1270 13 : if (!(vls = vls_get_w_dlock (vlsh)))
1271 0 : return VPPCOM_EBADFD;
1272 13 : vls_mt_guard (vls, VLS_MT_OP_XPOLL);
1273 13 : rv = vppcom_session_listen (vls_to_sh_tu (vls), q_len);
1274 13 : vls_mt_unguard ();
1275 13 : vls_get_and_unlock (vlsh);
1276 13 : return rv;
1277 : }
1278 :
1279 : int
1280 21 : vls_connect (vls_handle_t vlsh, vppcom_endpt_t * server_ep)
1281 : {
1282 : vcl_locked_session_t *vls;
1283 : int rv;
1284 :
1285 21 : vls_mt_detect ();
1286 21 : if (!(vls = vls_get_w_dlock (vlsh)))
1287 0 : return VPPCOM_EBADFD;
1288 21 : vls_mt_guard (vls, VLS_MT_OP_XPOLL);
1289 21 : rv = vppcom_session_connect (vls_to_sh_tu (vls), server_ep);
1290 21 : vls_mt_unguard ();
1291 21 : vls_get_and_unlock (vlsh);
1292 21 : return rv;
1293 : }
1294 :
1295 : static inline void
1296 0 : vls_mp_checks (vcl_locked_session_t * vls, int is_add)
1297 : {
1298 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
1299 : vcl_session_t *s;
1300 : u32 owner_wrk;
1301 :
1302 0 : if (vls_mt_wrk_supported ())
1303 0 : return;
1304 :
1305 0 : ASSERT (wrk->wrk_index == vls->vcl_wrk_index);
1306 0 : s = vcl_session_get (wrk, vls->session_index);
1307 0 : switch (s->session_state)
1308 : {
1309 0 : case VCL_STATE_LISTEN:
1310 0 : if (is_add)
1311 : {
1312 0 : vls_listener_wrk_set (vls, vls->vcl_wrk_index, 1 /* is_active */);
1313 0 : break;
1314 : }
1315 : /* Although removal from epoll means listener no longer accepts new
1316 : * sessions, the accept queue built by vpp cannot be drained by stopping
1317 : * the listener. Morover, some applications, e.g., nginx, might
1318 : * constantly remove and add listeners to their epfds. Removing
1319 : * listeners in such situations causes a lot of churn in vpp as segments
1320 : * and segment managers need to be recreated. */
1321 : /* vls_listener_wrk_stop_listen (vls, vls->vcl_wrk_index); */
1322 0 : break;
1323 0 : case VCL_STATE_LISTEN_NO_MQ:
1324 0 : if (!is_add)
1325 0 : break;
1326 :
1327 : /* Register worker as listener */
1328 0 : vls_listener_wrk_start_listen (vls, vls->vcl_wrk_index);
1329 :
1330 : /* If owner worker did not attempt to accept/xpoll on the session,
1331 : * force a listen stop for it, since it may not be interested in
1332 : * accepting new sessions.
1333 : * This is pretty much a hack done to give app workers the illusion
1334 : * that it is fine to listen and not accept new sessions for a
1335 : * given listener. Without it, we would accumulate unhandled
1336 : * accepts on the passive worker message queue. */
1337 0 : owner_wrk = vls_shared_get_owner (vls);
1338 0 : if (!vls_listener_wrk_is_active (vls, owner_wrk))
1339 0 : vls_listener_wrk_stop_listen (vls, owner_wrk);
1340 0 : break;
1341 0 : default:
1342 0 : break;
1343 : }
1344 : }
1345 :
1346 : vls_handle_t
1347 11 : vls_accept (vls_handle_t listener_vlsh, vppcom_endpt_t * ep, int flags)
1348 : {
1349 : vls_handle_t accepted_vlsh;
1350 : vcl_locked_session_t *vls;
1351 : int sh;
1352 :
1353 11 : vls_mt_detect ();
1354 11 : if (!(vls = vls_get_w_dlock (listener_vlsh)))
1355 0 : return VPPCOM_EBADFD;
1356 11 : if (vcl_n_workers () > 1)
1357 0 : vls_mp_checks (vls, 1 /* is_add */ );
1358 11 : vls_mt_guard (vls, VLS_MT_OP_SPOOL);
1359 11 : sh = vppcom_session_accept (vls_to_sh_tu (vls), ep, flags);
1360 11 : vls_mt_unguard ();
1361 11 : vls_get_and_unlock (listener_vlsh);
1362 11 : if (sh < 0)
1363 0 : return sh;
1364 11 : accepted_vlsh = vls_alloc (sh);
1365 11 : if (PREDICT_FALSE (accepted_vlsh == VLS_INVALID_HANDLE))
1366 0 : vppcom_session_close (sh);
1367 11 : return accepted_vlsh;
1368 : }
1369 :
1370 : vls_handle_t
1371 35 : vls_create (uint8_t proto, uint8_t is_nonblocking)
1372 : {
1373 : vcl_session_handle_t sh;
1374 : vls_handle_t vlsh;
1375 35 : vcl_locked_session_t *vls = NULL;
1376 :
1377 35 : vls_mt_detect ();
1378 35 : vls_mt_guard (vls, VLS_MT_OP_SPOOL);
1379 35 : sh = vppcom_session_create (proto, is_nonblocking);
1380 35 : vls_mt_unguard ();
1381 35 : if (sh == INVALID_SESSION_ID)
1382 0 : return VLS_INVALID_HANDLE;
1383 :
1384 35 : vlsh = vls_alloc (sh);
1385 35 : if (PREDICT_FALSE (vlsh == VLS_INVALID_HANDLE))
1386 0 : vppcom_session_close (sh);
1387 :
1388 35 : return vlsh;
1389 : }
1390 :
1391 : static void
1392 0 : vls_mt_session_cleanup (vcl_locked_session_t * vls)
1393 : {
1394 : u32 session_index, wrk_index, current_vcl_wrk;
1395 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
1396 :
1397 0 : ASSERT (vls_mt_wrk_supported ());
1398 :
1399 0 : current_vcl_wrk = vcl_get_worker_index ();
1400 :
1401 : /* *INDENT-OFF* */
1402 0 : hash_foreach (wrk_index, session_index, vls->vcl_wrk_index_to_session_index,
1403 : ({
1404 : if (current_vcl_wrk != wrk_index)
1405 : vls_send_session_cleanup_rpc (wrk, wrk_index, session_index);
1406 : }));
1407 : /* *INDENT-ON* */
1408 0 : hash_free (vls->vcl_wrk_index_to_session_index);
1409 0 : }
1410 :
1411 : int
1412 35 : vls_close (vls_handle_t vlsh)
1413 : {
1414 : vcl_locked_session_t *vls;
1415 : int rv;
1416 :
1417 35 : vls_mt_detect ();
1418 35 : vls_mt_pool_wlock ();
1419 :
1420 35 : vls = vls_get_and_lock (vlsh);
1421 35 : if (!vls)
1422 : {
1423 0 : vls_mt_pool_wunlock ();
1424 0 : return VPPCOM_EBADFD;
1425 : }
1426 :
1427 35 : vls_mt_guard (vls, VLS_MT_OP_SPOOL);
1428 :
1429 35 : if (vls_is_shared (vls))
1430 0 : rv = vls_unshare_session (vls, vcl_worker_get_current ());
1431 : else
1432 35 : rv = vppcom_session_close (vls_to_sh (vls));
1433 :
1434 35 : if (vls_mt_wrk_supported ())
1435 0 : vls_mt_session_cleanup (vls);
1436 :
1437 35 : vls_free (vls);
1438 35 : vls_mt_unguard ();
1439 :
1440 35 : vls_mt_pool_wunlock ();
1441 :
1442 35 : return rv;
1443 : }
1444 :
1445 : int
1446 0 : vls_shutdown (vls_handle_t vlsh, int how)
1447 : {
1448 : vcl_locked_session_t *vls;
1449 : int rv;
1450 :
1451 0 : vls_mt_detect ();
1452 0 : if (!(vls = vls_get_w_dlock (vlsh)))
1453 0 : return VPPCOM_EBADFD;
1454 :
1455 0 : vls_mt_guard (vls, VLS_MT_OP_SPOOL);
1456 0 : rv = vppcom_session_shutdown (vls_to_sh_tu (vls), how);
1457 0 : vls_mt_unguard ();
1458 0 : vls_get_and_unlock (vlsh);
1459 :
1460 0 : return rv;
1461 : }
1462 :
1463 : vls_handle_t
1464 1 : vls_epoll_create (void)
1465 : {
1466 : vcl_session_handle_t sh;
1467 : vls_handle_t vlsh;
1468 :
1469 1 : vls_mt_detect ();
1470 :
1471 1 : sh = vppcom_epoll_create ();
1472 1 : if (sh == INVALID_SESSION_ID)
1473 0 : return VLS_INVALID_HANDLE;
1474 :
1475 1 : vlsh = vls_alloc (sh);
1476 1 : if (vlsh == VLS_INVALID_HANDLE)
1477 0 : vppcom_session_close (sh);
1478 :
1479 1 : return vlsh;
1480 : }
1481 :
1482 : static void
1483 3 : vls_epoll_ctl_mp_checks (vcl_locked_session_t * vls, int op)
1484 : {
1485 3 : if (vcl_n_workers () <= 1 || op == EPOLL_CTL_MOD)
1486 3 : return;
1487 :
1488 0 : vls_mp_checks (vls, op == EPOLL_CTL_ADD);
1489 : }
1490 :
1491 : int
1492 3 : vls_epoll_ctl (vls_handle_t ep_vlsh, int op, vls_handle_t vlsh,
1493 : struct epoll_event *event)
1494 : {
1495 : vcl_locked_session_t *ep_vls, *vls;
1496 : vcl_session_handle_t ep_sh, sh;
1497 : int rv;
1498 :
1499 3 : vls_mt_detect ();
1500 3 : vls_mt_pool_rlock ();
1501 :
1502 3 : ep_vls = vls_get_and_lock (ep_vlsh);
1503 3 : if (PREDICT_FALSE (!ep_vls))
1504 : {
1505 0 : vls_mt_pool_runlock ();
1506 0 : return VPPCOM_EBADFD;
1507 : }
1508 :
1509 3 : if (vls_mt_session_should_migrate (ep_vls))
1510 : {
1511 0 : ep_vls = vls_mt_session_migrate (ep_vls);
1512 0 : if (PREDICT_FALSE (!ep_vls))
1513 : {
1514 0 : vls_mt_pool_runlock ();
1515 0 : return VPPCOM_EBADFD;
1516 : }
1517 : }
1518 :
1519 3 : vls = vls_get_and_lock (vlsh);
1520 3 : if (PREDICT_FALSE (!vls))
1521 : {
1522 0 : vls_unlock (ep_vls);
1523 0 : vls_mt_pool_runlock ();
1524 0 : return VPPCOM_EBADFD;
1525 : }
1526 :
1527 3 : ep_sh = vls_to_sh (ep_vls);
1528 3 : sh = vls_to_sh (vls);
1529 :
1530 3 : vls_epoll_ctl_mp_checks (vls, op);
1531 3 : vls_mt_pool_runlock ();
1532 3 : rv = vppcom_epoll_ctl (ep_sh, op, sh, event);
1533 :
1534 3 : vls_mt_pool_rlock ();
1535 3 : ep_vls = vls_get (ep_vlsh);
1536 3 : vls = vls_get (vlsh);
1537 3 : vls_unlock (vls);
1538 3 : vls_unlock (ep_vls);
1539 3 : vls_mt_pool_runlock ();
1540 3 : return rv;
1541 : }
1542 :
1543 : int
1544 1374040 : vls_epoll_wait (vls_handle_t ep_vlsh, struct epoll_event *events,
1545 : int maxevents, double wait_for_time)
1546 : {
1547 1374040 : vcl_locked_session_t *vls, *vls_tmp = NULL;
1548 : int rv;
1549 :
1550 1374040 : vls_mt_detect ();
1551 1374040 : if (!(vls = vls_get_w_dlock (ep_vlsh)))
1552 0 : return VPPCOM_EBADFD;
1553 1374040 : vls_mt_guard (vls_tmp, VLS_MT_OP_XPOLL);
1554 1374040 : rv = vppcom_epoll_wait (vls_to_sh_tu (vls), events, maxevents,
1555 : wait_for_time);
1556 1374040 : vls_mt_unguard ();
1557 1374040 : vls_get_and_unlock (ep_vlsh);
1558 1374040 : vls_handle_pending_wrk_cleanup ();
1559 1374040 : return rv;
1560 : }
1561 :
1562 : static void
1563 12 : vls_select_mp_checks (vcl_si_set * read_map)
1564 : {
1565 : vcl_locked_session_t *vls;
1566 : vcl_worker_t *wrk;
1567 : vcl_session_t *s;
1568 : u32 si;
1569 :
1570 12 : if (vcl_n_workers () <= 1)
1571 : {
1572 12 : vlsl->select_mp_check = 1;
1573 12 : return;
1574 : }
1575 :
1576 0 : if (!read_map)
1577 0 : return;
1578 :
1579 0 : vlsl->select_mp_check = 1;
1580 0 : wrk = vcl_worker_get_current ();
1581 :
1582 : /* *INDENT-OFF* */
1583 0 : clib_bitmap_foreach (si, read_map) {
1584 0 : s = vcl_session_get (wrk, si);
1585 0 : if (s->session_state == VCL_STATE_LISTEN)
1586 : {
1587 0 : vls = vls_get (vls_session_index_to_vlsh (si));
1588 0 : vls_mp_checks (vls, 1 /* is_add */);
1589 : }
1590 : }
1591 : /* *INDENT-ON* */
1592 : }
1593 :
1594 : int
1595 26331000 : vls_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
1596 : vcl_si_set * except_map, double wait_for_time)
1597 : {
1598 : int rv;
1599 26331000 : vcl_locked_session_t *vls = NULL;
1600 :
1601 26331000 : vls_mt_detect ();
1602 26331000 : vls_mt_guard (vls, VLS_MT_OP_XPOLL);
1603 26331000 : if (PREDICT_FALSE (!vlsl->select_mp_check))
1604 12 : vls_select_mp_checks (read_map);
1605 26331000 : rv = vppcom_select (n_bits, read_map, write_map, except_map, wait_for_time);
1606 26331000 : vls_mt_unguard ();
1607 26331000 : vls_handle_pending_wrk_cleanup ();
1608 26331000 : return rv;
1609 : }
1610 :
1611 : static void
1612 13 : vls_unshare_vcl_worker_sessions (vcl_worker_t * wrk)
1613 : {
1614 : u32 current_wrk, is_current;
1615 : vcl_locked_session_t *vls;
1616 : vcl_session_t *s;
1617 :
1618 13 : if (pool_elts (vcm->workers) <= 1)
1619 13 : return;
1620 :
1621 0 : current_wrk = vcl_get_worker_index ();
1622 0 : is_current = current_wrk == wrk->wrk_index;
1623 :
1624 : /* *INDENT-OFF* */
1625 0 : pool_foreach (s, wrk->sessions) {
1626 0 : vls = vls_get (vls_si_wi_to_vlsh (s->session_index, wrk->wrk_index));
1627 0 : if (vls && (is_current || vls_is_shared_by_wrk (vls, current_wrk)))
1628 0 : vls_unshare_session (vls, wrk);
1629 : }
1630 : /* *INDENT-ON* */
1631 : }
1632 :
1633 : static void
1634 0 : vls_cleanup_vcl_worker (vcl_worker_t * wrk)
1635 : {
1636 0 : vls_worker_t *vls_wrk = vls_worker_get (wrk->wrk_index);
1637 :
1638 : /* Unshare sessions and also cleanup worker since child may have
1639 : * called _exit () and therefore vcl may not catch the event */
1640 0 : vls_unshare_vcl_worker_sessions (wrk);
1641 :
1642 : /* Since child may have exited and thereforce fd of vpp_app_socket_api
1643 : * may have been closed, so DONOT notify VPP.
1644 : */
1645 0 : vcl_worker_cleanup (wrk, vcm->cfg.vpp_app_socket_api ? 0 : 1);
1646 :
1647 0 : vls_worker_free (vls_wrk);
1648 0 : }
1649 :
1650 : static void
1651 0 : vls_cleanup_forked_child (vcl_worker_t * wrk, vcl_worker_t * child_wrk)
1652 : {
1653 : vcl_worker_t *sub_child;
1654 0 : int tries = 0;
1655 :
1656 0 : if (child_wrk->forked_child != ~0)
1657 : {
1658 0 : sub_child = vcl_worker_get_if_valid (child_wrk->forked_child);
1659 0 : if (sub_child)
1660 : {
1661 : /* Wait a bit, maybe the process is going away */
1662 0 : while (kill (sub_child->current_pid, 0) >= 0 && tries++ < 50)
1663 0 : usleep (1e3);
1664 0 : if (kill (sub_child->current_pid, 0) < 0)
1665 0 : vls_cleanup_forked_child (child_wrk, sub_child);
1666 : }
1667 : }
1668 0 : vls_cleanup_vcl_worker (child_wrk);
1669 0 : VDBG (0, "Cleaned up forked child wrk %u", child_wrk->wrk_index);
1670 0 : wrk->forked_child = ~0;
1671 0 : }
1672 :
1673 : static void
1674 27705100 : vls_handle_pending_wrk_cleanup (void)
1675 : {
1676 : u32 *wip;
1677 : vcl_worker_t *child_wrk, *wrk;
1678 27705100 : vls_worker_t *vls_wrk = vls_worker_get_current ();
1679 :
1680 27705100 : if (PREDICT_TRUE (vec_len (vls_wrk->pending_vcl_wrk_cleanup) == 0))
1681 27705100 : return;
1682 :
1683 0 : wrk = vcl_worker_get_current ();
1684 0 : vec_foreach (wip, vls_wrk->pending_vcl_wrk_cleanup)
1685 : {
1686 0 : child_wrk = vcl_worker_get_if_valid (*wip);
1687 0 : if (!child_wrk)
1688 0 : continue;
1689 0 : vls_cleanup_forked_child (wrk, child_wrk);
1690 : }
1691 0 : vec_reset_length (vls_wrk->pending_vcl_wrk_cleanup);
1692 : }
1693 :
1694 : static struct sigaction old_sa;
1695 :
1696 : static void
1697 0 : vls_intercept_sigchld_handler (int signum, siginfo_t * si, void *uc)
1698 : {
1699 : vcl_worker_t *wrk, *child_wrk;
1700 : vls_worker_t *vls_wrk;
1701 :
1702 0 : if (vcl_get_worker_index () == ~0)
1703 0 : return;
1704 :
1705 0 : if (sigaction (SIGCHLD, &old_sa, 0))
1706 : {
1707 0 : VERR ("couldn't restore sigchld");
1708 0 : exit (-1);
1709 : }
1710 :
1711 0 : wrk = vcl_worker_get_current ();
1712 0 : if (wrk->forked_child == ~0)
1713 0 : return;
1714 :
1715 0 : child_wrk = vcl_worker_get_if_valid (wrk->forked_child);
1716 0 : if (!child_wrk)
1717 0 : goto done;
1718 :
1719 0 : if (si && si->si_pid != child_wrk->current_pid)
1720 : {
1721 0 : VDBG (0, "unexpected child pid %u", si->si_pid);
1722 0 : goto done;
1723 : }
1724 :
1725 : /* Parent process may enter sighandler with a lock, such as lock in localtime
1726 : * or in mspace_free, and child wrk cleanup may try to get such locks and
1727 : * cause deadlock.
1728 : * So move child wrk cleanup from sighandler to vls_epoll_wait/vls_select.
1729 : */
1730 0 : vls_wrk = vls_worker_get_current ();
1731 0 : vec_add1 (vls_wrk->pending_vcl_wrk_cleanup, child_wrk->wrk_index);
1732 :
1733 0 : done:
1734 0 : if (old_sa.sa_flags & SA_SIGINFO)
1735 : {
1736 0 : void (*fn) (int, siginfo_t *, void *) = old_sa.sa_sigaction;
1737 0 : fn (signum, si, uc);
1738 : }
1739 : else
1740 : {
1741 0 : void (*fn) (int) = old_sa.sa_handler;
1742 0 : if (fn)
1743 0 : fn (signum);
1744 : }
1745 : }
1746 :
1747 : static void
1748 0 : vls_incercept_sigchld ()
1749 : {
1750 : struct sigaction sa;
1751 0 : if (old_sa.sa_sigaction)
1752 : {
1753 0 : VDBG (0, "have intercepted sigchld");
1754 0 : return;
1755 : }
1756 0 : clib_memset (&sa, 0, sizeof (sa));
1757 0 : sa.sa_sigaction = vls_intercept_sigchld_handler;
1758 0 : sa.sa_flags = SA_SIGINFO;
1759 0 : if (sigaction (SIGCHLD, &sa, &old_sa))
1760 : {
1761 0 : VERR ("couldn't intercept sigchld");
1762 0 : exit (-1);
1763 : }
1764 : }
1765 :
1766 : static void
1767 0 : vls_app_pre_fork (void)
1768 : {
1769 0 : vls_incercept_sigchld ();
1770 0 : vcl_flush_mq_events ();
1771 0 : }
1772 :
1773 : static void
1774 0 : vls_app_fork_child_handler (void)
1775 : {
1776 : vcl_worker_t *parent_wrk;
1777 : int parent_wrk_index;
1778 :
1779 0 : parent_wrk_index = vcl_get_worker_index ();
1780 0 : VDBG (0, "initializing forked child %u with parent wrk %u", getpid (),
1781 : parent_wrk_index);
1782 :
1783 : /*
1784 : * Clear old state
1785 : */
1786 0 : vcl_set_worker_index (~0);
1787 :
1788 : /*
1789 : * Allocate and register vcl worker with vpp
1790 : */
1791 0 : if (vppcom_worker_register ())
1792 : {
1793 0 : VERR ("couldn't register new worker!");
1794 0 : return;
1795 : }
1796 :
1797 : /*
1798 : * Allocate/initialize vls worker and share sessions
1799 : */
1800 0 : vls_worker_alloc ();
1801 :
1802 : /* Reset number of threads and set wrk index */
1803 0 : vlsl->vls_mt_n_threads = 0;
1804 0 : vlsl->vls_wrk_index = vcl_get_worker_index ();
1805 0 : vlsl->select_mp_check = 0;
1806 0 : clib_rwlock_init (&vlsl->vls_pool_lock);
1807 0 : vls_mt_locks_init ();
1808 :
1809 0 : parent_wrk = vcl_worker_get (parent_wrk_index);
1810 0 : vls_worker_copy_on_fork (parent_wrk);
1811 0 : parent_wrk->forked_child = vcl_get_worker_index ();
1812 :
1813 0 : VDBG (0, "forked child main worker initialized");
1814 0 : vcm->forking = 0;
1815 : }
1816 :
1817 : static void
1818 0 : vls_app_fork_parent_handler (void)
1819 : {
1820 0 : vcm->forking = 1;
1821 0 : while (vcm->forking)
1822 : ;
1823 0 : }
1824 :
1825 : void
1826 13 : vls_app_exit (void)
1827 : {
1828 13 : vls_worker_t *wrk = vls_worker_get_current ();
1829 :
1830 : /* Handle pending wrk cleanup */
1831 13 : vls_handle_pending_wrk_cleanup ();
1832 :
1833 : /* Unshare the sessions. VCL will clean up the worker */
1834 13 : vls_unshare_vcl_worker_sessions (vcl_worker_get_current ());
1835 13 : vls_worker_free (wrk);
1836 13 : }
1837 :
1838 : static void
1839 0 : vls_clone_and_share_rpc_handler (void *args)
1840 : {
1841 0 : vls_clone_and_share_msg_t *msg = (vls_clone_and_share_msg_t *) args;
1842 0 : vls_worker_t *wrk = vls_worker_get_current (), *dst_wrk;
1843 : vcl_locked_session_t *vls, *dst_vls;
1844 0 : vcl_worker_t *vcl_wrk = vcl_worker_get_current (), *dst_vcl_wrk;
1845 : vcl_session_t *s, *dst_s;
1846 :
1847 0 : VDBG (1, "process session clone of worker (session): %u (%u) -> %u (%u)",
1848 : vcl_wrk->wrk_index, msg->session_index, msg->origin_vcl_wrk,
1849 : msg->origin_session_index);
1850 :
1851 : /* VCL locked session can't been protected, so DONT touch it.
1852 : * VCL session may been free, check it.
1853 : */
1854 0 : dst_vcl_wrk = vcl_worker_get (msg->origin_vcl_wrk);
1855 0 : s = vcl_session_get (vcl_wrk, msg->session_index);
1856 0 : if (PREDICT_FALSE (!s))
1857 : {
1858 0 : dst_vcl_wrk->rpc_done = VLS_RPC_STATE_SESSION_NOT_EXIST;
1859 0 : return;
1860 : }
1861 :
1862 0 : if (!vls_mt_wrk_supported ())
1863 : {
1864 0 : vls = vls_session_get (wrk, msg->vls_index);
1865 0 : vls_init_share_session (wrk, vls);
1866 0 : dst_wrk = vls_worker_get (msg->origin_vls_wrk);
1867 0 : dst_vls = vls_session_get (dst_wrk, msg->origin_vls_index);
1868 0 : dst_vls->shared_data_index = vls->shared_data_index;
1869 : }
1870 0 : dst_s = vcl_session_get (dst_vcl_wrk, msg->origin_session_index);
1871 0 : clib_memcpy (dst_s, s, sizeof (*s));
1872 :
1873 0 : dst_vcl_wrk->rpc_done = VLS_RPC_STATE_SUCCESS;
1874 : }
1875 :
1876 : static void
1877 0 : vls_session_cleanup_rpc_handler (void *args)
1878 : {
1879 0 : vls_sess_cleanup_msg_t *msg = (vls_sess_cleanup_msg_t *) args;
1880 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
1881 0 : vls_worker_t *vls_wrk = vls_worker_get_current ();
1882 0 : vcl_session_handle_t sh = vcl_session_handle_from_index (msg->session_index);
1883 :
1884 0 : VDBG (1, "process session cleanup of worker (session): %u (%u) from %u ()",
1885 : wrk->wrk_index, msg->session_index, msg->origin_vcl_wrk);
1886 :
1887 0 : vppcom_session_close (sh);
1888 0 : vls_sh_to_vlsh_table_del (vls_wrk, sh);
1889 0 : }
1890 :
1891 : static void
1892 0 : vls_rpc_handler (void *args)
1893 : {
1894 0 : vls_rpc_msg_t *msg = (vls_rpc_msg_t *) args;
1895 0 : switch (msg->type)
1896 : {
1897 0 : case VLS_RPC_CLONE_AND_SHARE:
1898 0 : vls_clone_and_share_rpc_handler (msg->data);
1899 0 : break;
1900 0 : case VLS_RPC_SESS_CLEANUP:
1901 0 : vls_session_cleanup_rpc_handler (msg->data);
1902 0 : break;
1903 0 : default:
1904 0 : break;
1905 : }
1906 0 : }
1907 :
1908 : void
1909 0 : vls_send_clone_and_share_rpc (vcl_worker_t *wrk, u32 origin_vls_index,
1910 : u32 session_index, u32 vls_wrk_index,
1911 : u32 dst_wrk_index, u32 dst_vls_index,
1912 : u32 dst_session_index)
1913 : {
1914 : u8 data[sizeof (u8) + sizeof (vls_clone_and_share_msg_t)];
1915 : vls_clone_and_share_msg_t *msg;
1916 : vls_rpc_msg_t *rpc;
1917 : int ret;
1918 0 : f64 timeout = clib_time_now (&wrk->clib_time) + VLS_WORKER_RPC_TIMEOUT;
1919 :
1920 0 : rpc = (vls_rpc_msg_t *) & data;
1921 0 : rpc->type = VLS_RPC_CLONE_AND_SHARE;
1922 0 : msg = (vls_clone_and_share_msg_t *) & rpc->data;
1923 0 : msg->origin_vls_wrk = vls_wrk_index;
1924 0 : msg->origin_vls_index = origin_vls_index;
1925 0 : msg->origin_vcl_wrk = wrk->wrk_index;
1926 0 : msg->origin_session_index = session_index;
1927 0 : msg->vls_index = dst_vls_index;
1928 0 : msg->session_index = dst_session_index;
1929 :
1930 : /* Try lock and handle rpcs if two threads send each other
1931 : * clone requests at the same time.
1932 : */
1933 0 : wrk->rpc_done = VLS_RPC_STATE_INIT;
1934 0 : while (!clib_spinlock_trylock (&vlsm->worker_rpc_lock))
1935 0 : vcl_flush_mq_events ();
1936 0 : ret = vcl_send_worker_rpc (dst_wrk_index, rpc, sizeof (data));
1937 :
1938 0 : VDBG (1, "send session clone to wrk (session): %u (%u) -> %u (%u), ret=%d",
1939 : dst_wrk_index, msg->session_index, msg->origin_vcl_wrk,
1940 : msg->origin_session_index, ret);
1941 0 : while (!ret && wrk->rpc_done == VLS_RPC_STATE_INIT &&
1942 0 : clib_time_now (&wrk->clib_time) < timeout)
1943 : ;
1944 0 : clib_spinlock_unlock (&vlsm->worker_rpc_lock);
1945 0 : }
1946 :
1947 : void
1948 0 : vls_send_session_cleanup_rpc (vcl_worker_t * wrk,
1949 : u32 dst_wrk_index, u32 dst_session_index)
1950 : {
1951 : u8 data[sizeof (u8) + sizeof (vls_sess_cleanup_msg_t)];
1952 : vls_sess_cleanup_msg_t *msg;
1953 : vls_rpc_msg_t *rpc;
1954 : int ret;
1955 :
1956 0 : rpc = (vls_rpc_msg_t *) & data;
1957 0 : rpc->type = VLS_RPC_SESS_CLEANUP;
1958 0 : msg = (vls_sess_cleanup_msg_t *) & rpc->data;
1959 0 : msg->origin_vcl_wrk = wrk->wrk_index;
1960 0 : msg->session_index = dst_session_index;
1961 :
1962 0 : ret = vcl_send_worker_rpc (dst_wrk_index, rpc, sizeof (data));
1963 :
1964 0 : VDBG (1, "send session cleanup to wrk (session): %u (%u) from %u, ret=%d",
1965 : dst_wrk_index, msg->session_index, msg->origin_vcl_wrk, ret);
1966 0 : }
1967 :
1968 : int
1969 13 : vls_app_create (char *app_name)
1970 : {
1971 : int rv;
1972 :
1973 13 : if ((rv = vppcom_app_create (app_name)))
1974 0 : return rv;
1975 :
1976 13 : vlsm = clib_mem_alloc (sizeof (vls_main_t));
1977 13 : clib_memset (vlsm, 0, sizeof (*vlsm));
1978 13 : clib_rwlock_init (&vlsm->shared_data_lock);
1979 13 : clib_spinlock_init (&vlsm->worker_rpc_lock);
1980 13 : pool_alloc (vlsm->workers, vcm->cfg.max_workers);
1981 :
1982 13 : pthread_atfork (vls_app_pre_fork, vls_app_fork_parent_handler,
1983 : vls_app_fork_child_handler);
1984 13 : atexit (vls_app_exit);
1985 13 : vls_worker_alloc ();
1986 13 : vlsl->vls_wrk_index = vcl_get_worker_index ();
1987 13 : clib_rwlock_init (&vlsl->vls_pool_lock);
1988 13 : vls_mt_locks_init ();
1989 13 : vcm->wrk_rpc_fn = vls_rpc_handler;
1990 13 : return VPPCOM_OK;
1991 : }
1992 :
1993 : unsigned char
1994 248 : vls_use_eventfd (void)
1995 : {
1996 248 : return vcm->cfg.use_mq_eventfd;
1997 : }
1998 :
1999 : unsigned char
2000 30487000 : vls_mt_wrk_supported (void)
2001 : {
2002 30487000 : return vcm->cfg.mt_wrk_supported;
2003 : }
2004 :
2005 : int
2006 1 : vls_use_real_epoll (void)
2007 : {
2008 1 : if (vcl_get_worker_index () == ~0)
2009 0 : return 0;
2010 :
2011 1 : return vcl_worker_get_current ()->vcl_needs_real_epoll;
2012 : }
2013 :
2014 : void
2015 0 : vls_register_vcl_worker (void)
2016 : {
2017 0 : vls_mt_add ();
2018 0 : }
2019 :
2020 : /*
2021 : * fd.io coding-style-patch-verification: ON
2022 : *
2023 : * Local Variables:
2024 : * eval: (c-set-style "gnu")
2025 : * End:
2026 : */
|
