Line data Source code
1 : /*
2 : *------------------------------------------------------------------
3 : * Copyright (c) 2018 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : *------------------------------------------------------------------
16 : */
17 : #include <signal.h>
18 :
19 : #include <vlib/vlib.h>
20 : #include <vlibapi/api.h>
21 : #include <vlibmemory/api.h>
22 : #include <vlibmemory/memory_api.h>
23 :
24 : #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
25 :
26 : #define vl_typedefs /* define message structures */
27 : #include <vlibmemory/vl_memory_api_h.h>
28 : #undef vl_typedefs
29 :
30 : /* instantiate all the print functions we know about */
31 : #define vl_printfun
32 : #include <vlibmemory/vl_memory_api_h.h>
33 : #undef vl_printfun
34 :
35 : /* instantiate all the endian swap functions we know about */
36 : #define vl_endianfun
37 : #include <vlibmemory/vl_memory_api_h.h>
38 : #undef vl_endianfun
39 :
40 : volatile int **vl_api_queue_cursizes;
41 :
42 : static void
43 638307000 : memclnt_queue_callback (vlib_main_t * vm)
44 : {
45 : int i;
46 638307000 : api_main_t *am = vlibapi_get_main ();
47 : int have_pending_rpcs;
48 :
49 638307000 : if (PREDICT_FALSE (vec_len (vl_api_queue_cursizes) !=
50 : 1 + vec_len (am->vlib_private_rps)))
51 : {
52 648 : vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr;
53 : svm_queue_t *q;
54 :
55 648 : if (shmem_hdr == 0)
56 0 : return;
57 :
58 648 : q = shmem_hdr->vl_input_queue;
59 648 : if (q == 0)
60 0 : return;
61 :
62 648 : vec_add1 (vl_api_queue_cursizes, &q->cursize);
63 :
64 785 : for (i = 0; i < vec_len (am->vlib_private_rps); i++)
65 : {
66 137 : svm_region_t *vlib_rp = am->vlib_private_rps[i];
67 :
68 137 : shmem_hdr = (void *) vlib_rp->user_ctx;
69 137 : q = shmem_hdr->vl_input_queue;
70 137 : vec_add1 (vl_api_queue_cursizes, &q->cursize);
71 : }
72 : }
73 :
74 1352610000 : for (i = 0; i < vec_len (vl_api_queue_cursizes); i++)
75 : {
76 714310000 : if (*vl_api_queue_cursizes[i])
77 : {
78 11055 : vm->queue_signal_pending = 1;
79 11055 : vm->api_queue_nonempty = 1;
80 11055 : vlib_process_signal_event (vm, vl_api_clnt_node.index,
81 : /* event_type */ QUEUE_SIGNAL_EVENT,
82 : /* event_data */ 0);
83 11055 : break;
84 : }
85 : }
86 :
87 638307000 : clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
88 638307000 : have_pending_rpcs = vec_len (vm->pending_rpc_requests) > 0;
89 638307000 : clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
90 :
91 638307000 : if (have_pending_rpcs)
92 : {
93 204 : vm->queue_signal_pending = 1;
94 204 : vm->api_queue_nonempty = 1;
95 204 : vlib_process_signal_event (vm, vl_api_clnt_node.index,
96 : /* event_type */ QUEUE_SIGNAL_EVENT,
97 : /* event_data */ 0);
98 : }
99 : }
100 :
101 : /*
102 : * vl_api_memclnt_create_internal
103 : */
104 : u32
105 1 : vl_api_memclnt_create_internal (char *name, svm_queue_t * q)
106 : {
107 : vl_api_registration_t **regpp;
108 : vl_api_registration_t *regp;
109 : void *oldheap;
110 1 : api_main_t *am = vlibapi_get_main ();
111 :
112 1 : ASSERT (vlib_get_thread_index () == 0);
113 1 : pool_get (am->vl_clients, regpp);
114 :
115 :
116 1 : oldheap = vl_msg_push_heap ();
117 1 : *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
118 :
119 1 : regp = *regpp;
120 1 : clib_memset (regp, 0, sizeof (*regp));
121 1 : regp->registration_type = REGISTRATION_TYPE_SHMEM;
122 1 : regp->vl_api_registration_pool_index = regpp - am->vl_clients;
123 1 : regp->vlib_rp = am->vlib_rp;
124 1 : regp->shmem_hdr = am->shmem_hdr;
125 :
126 1 : regp->vl_input_queue = q;
127 1 : regp->name = format (0, "%s%c", name, 0);
128 :
129 1 : vl_msg_pop_heap (oldheap);
130 2 : return vl_msg_api_handle_from_index_and_epoch
131 : (regp->vl_api_registration_pool_index,
132 1 : am->shmem_hdr->application_restarts);
133 : }
134 :
135 : /*
136 : * vl_api_memclnt_create_t_handler
137 : */
138 : void
139 43 : vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
140 : {
141 : vl_api_registration_t **regpp;
142 : vl_api_registration_t *regp;
143 : vl_api_memclnt_create_reply_t *rp;
144 : svm_queue_t *q;
145 43 : int rv = 0;
146 : void *oldheap;
147 43 : api_main_t *am = vlibapi_get_main ();
148 : u8 *msg_table;
149 :
150 : /*
151 : * This is tortured. Maintain a vlib-address-space private
152 : * pool of client registrations. We use the shared-memory virtual
153 : * address of client structure as a handle, to allow direct
154 : * manipulation of context quota vbls from the client library.
155 : *
156 : * This scheme causes trouble w/ API message trace replay, since
157 : * some random VA from clib_mem_alloc() certainly won't
158 : * occur in the Linux sim. The (very) few places
159 : * that care need to use the pool index.
160 : *
161 : * Putting the registration object(s) into a pool in shared memory and
162 : * using the pool index as a handle seems like a great idea.
163 : * Unfortunately, each and every reference to that pool would need
164 : * to be protected by a mutex:
165 : *
166 : * Client VLIB
167 : * ------ ----
168 : * convert pool index to
169 : * pointer.
170 : * <deschedule>
171 : * expand pool
172 : * <deschedule>
173 : * kaboom!
174 : */
175 :
176 43 : pool_get (am->vl_clients, regpp);
177 :
178 43 : oldheap = vl_msg_push_heap ();
179 43 : *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
180 :
181 43 : regp = *regpp;
182 43 : clib_memset (regp, 0, sizeof (*regp));
183 43 : regp->registration_type = REGISTRATION_TYPE_SHMEM;
184 43 : regp->vl_api_registration_pool_index = regpp - am->vl_clients;
185 43 : regp->vlib_rp = am->vlib_rp;
186 43 : regp->shmem_hdr = am->shmem_hdr;
187 43 : regp->clib_file_index = am->shmem_hdr->clib_file_index;
188 :
189 43 : q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
190 43 : VL_MSG_API_SVM_QUEUE_UNPOISON (q);
191 :
192 43 : regp->name = format (0, "%s", mp->name);
193 43 : vec_add1 (regp->name, 0);
194 43 : regp->keepalive = true;
195 :
196 43 : if (am->serialized_message_table_in_shmem == 0)
197 13 : am->serialized_message_table_in_shmem =
198 13 : vl_api_serialize_message_table (am, 0);
199 :
200 43 : if (am->vlib_rp != am->vlib_primary_rp)
201 43 : msg_table = vl_api_serialize_message_table (am, 0);
202 : else
203 0 : msg_table = am->serialized_message_table_in_shmem;
204 :
205 43 : vl_msg_pop_heap (oldheap);
206 :
207 43 : rp = vl_msg_api_alloc (sizeof (*rp));
208 43 : rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
209 43 : rp->handle = (uword) regp;
210 86 : rp->index = vl_msg_api_handle_from_index_and_epoch
211 : (regp->vl_api_registration_pool_index,
212 43 : am->shmem_hdr->application_restarts);
213 43 : rp->context = mp->context;
214 43 : rp->response = ntohl (rv);
215 43 : rp->message_table = pointer_to_uword (msg_table);
216 :
217 43 : vl_msg_api_send_shmem (q, (u8 *) & rp);
218 43 : }
219 :
220 : void
221 18 : vl_api_memclnt_create_v2_t_handler (vl_api_memclnt_create_v2_t *mp)
222 : {
223 : vl_api_registration_t **regpp;
224 : vl_api_registration_t *regp;
225 : vl_api_memclnt_create_v2_reply_t *rp;
226 : svm_queue_t *q;
227 18 : int rv = 0;
228 : void *oldheap;
229 18 : api_main_t *am = vlibapi_get_main ();
230 : u8 *msg_table;
231 :
232 : /*
233 : * This is tortured. Maintain a vlib-address-space private
234 : * pool of client registrations. We use the shared-memory virtual
235 : * address of client structure as a handle, to allow direct
236 : * manipulation of context quota vbls from the client library.
237 : *
238 : * This scheme causes trouble w/ API message trace replay, since
239 : * some random VA from clib_mem_alloc() certainly won't
240 : * occur in the Linux sim. The (very) few places
241 : * that care need to use the pool index.
242 : *
243 : * Putting the registration object(s) into a pool in shared memory and
244 : * using the pool index as a handle seems like a great idea.
245 : * Unfortunately, each and every reference to that pool would need
246 : * to be protected by a mutex:
247 : *
248 : * Client VLIB
249 : * ------ ----
250 : * convert pool index to
251 : * pointer.
252 : * <deschedule>
253 : * expand pool
254 : * <deschedule>
255 : * kaboom!
256 : */
257 :
258 18 : pool_get (am->vl_clients, regpp);
259 :
260 18 : oldheap = vl_msg_push_heap ();
261 18 : *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
262 :
263 18 : regp = *regpp;
264 18 : clib_memset (regp, 0, sizeof (*regp));
265 18 : regp->registration_type = REGISTRATION_TYPE_SHMEM;
266 18 : regp->vl_api_registration_pool_index = regpp - am->vl_clients;
267 18 : regp->vlib_rp = am->vlib_rp;
268 18 : regp->shmem_hdr = am->shmem_hdr;
269 18 : regp->clib_file_index = am->shmem_hdr->clib_file_index;
270 :
271 18 : q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
272 18 : VL_MSG_API_SVM_QUEUE_UNPOISON (q);
273 :
274 18 : regp->name = format (0, "%s", mp->name);
275 18 : vec_add1 (regp->name, 0);
276 18 : regp->keepalive = mp->keepalive;
277 :
278 18 : if (am->serialized_message_table_in_shmem == 0)
279 2 : am->serialized_message_table_in_shmem =
280 2 : vl_api_serialize_message_table (am, 0);
281 :
282 18 : if (am->vlib_rp != am->vlib_primary_rp)
283 0 : msg_table = vl_api_serialize_message_table (am, 0);
284 : else
285 18 : msg_table = am->serialized_message_table_in_shmem;
286 :
287 18 : vl_msg_pop_heap (oldheap);
288 :
289 18 : rp = vl_msg_api_alloc (sizeof (*rp));
290 18 : rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_V2_REPLY);
291 18 : rp->handle = (uword) regp;
292 36 : rp->index = vl_msg_api_handle_from_index_and_epoch (
293 18 : regp->vl_api_registration_pool_index, am->shmem_hdr->application_restarts);
294 18 : rp->context = mp->context;
295 18 : rp->response = ntohl (rv);
296 18 : rp->message_table = pointer_to_uword (msg_table);
297 :
298 18 : vl_msg_api_send_shmem (q, (u8 *) &rp);
299 18 : }
300 :
301 : void
302 1241 : vl_api_call_reaper_functions (u32 client_index)
303 : {
304 1241 : clib_error_t *error = 0;
305 : _vl_msg_api_function_list_elt_t *i;
306 :
307 1241 : i = vlibapi_get_main ()->reaper_function_registrations;
308 14892 : while (i)
309 : {
310 13651 : error = i->f (client_index);
311 13651 : if (error)
312 0 : clib_error_report (error);
313 13651 : i = i->next_init_function;
314 : }
315 1241 : }
316 :
317 : /*
318 : * vl_api_memclnt_delete_t_handler
319 : */
320 : void
321 47 : vl_api_memclnt_delete_t_handler (vl_api_memclnt_delete_t * mp)
322 : {
323 : vl_api_registration_t **regpp;
324 : vl_api_registration_t *regp;
325 : vl_api_memclnt_delete_reply_t *rp;
326 : void *oldheap;
327 47 : api_main_t *am = vlibapi_get_main ();
328 : u32 handle, client_index, epoch;
329 :
330 47 : handle = mp->index;
331 :
332 47 : vl_api_call_reaper_functions (handle);
333 :
334 47 : epoch = vl_msg_api_handle_get_epoch (handle);
335 47 : client_index = vl_msg_api_handle_get_index (handle);
336 :
337 47 : if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK))
338 : {
339 0 : clib_warning
340 : ("Stale clnt delete index %d old epoch %d cur epoch %d",
341 : client_index, epoch,
342 : (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK));
343 0 : return;
344 : }
345 :
346 47 : regpp = pool_elt_at_index (am->vl_clients, client_index);
347 :
348 47 : if (!pool_is_free (am->vl_clients, regpp))
349 : {
350 : int i;
351 47 : regp = *regpp;
352 47 : int private_registration = 0;
353 :
354 : /* Send reply unless client asked us to do the cleanup */
355 47 : if (!mp->do_cleanup)
356 : {
357 : /*
358 : * Note: the API message handling path will set am->vlib_rp
359 : * as appropriate for pairwise / private memory segments
360 : */
361 47 : rp = vl_msg_api_alloc (sizeof (*rp));
362 47 : rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY);
363 47 : rp->handle = mp->handle;
364 47 : rp->response = 1;
365 :
366 47 : vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp);
367 47 : if (client_index != regp->vl_api_registration_pool_index)
368 : {
369 0 : clib_warning ("mismatch client_index %d pool_index %d",
370 : client_index,
371 : regp->vl_api_registration_pool_index);
372 0 : vl_msg_api_free (rp);
373 0 : return;
374 : }
375 : }
376 :
377 : /* No dangling references, please */
378 47 : *regpp = 0;
379 :
380 : /* For horizontal scaling, add a hash table... */
381 73 : for (i = 0; i < vec_len (am->vlib_private_rps); i++)
382 : {
383 : /* Is this a pairwise / private API segment? */
384 55 : if (am->vlib_private_rps[i] == am->vlib_rp)
385 : {
386 : /* Note: account for the memfd header page */
387 29 : uword virtual_base = am->vlib_rp->virtual_base - MMAP_PAGESIZE;
388 29 : uword virtual_size = am->vlib_rp->virtual_size + MMAP_PAGESIZE;
389 :
390 : /*
391 : * Kill the registration pool element before we make
392 : * the index vanish forever
393 : */
394 29 : pool_put_index (am->vl_clients,
395 : regp->vl_api_registration_pool_index);
396 :
397 29 : vec_delete (am->vlib_private_rps, 1, i);
398 : /* Kill it, accounting for the memfd header page */
399 29 : if (munmap ((void *) virtual_base, virtual_size) < 0)
400 0 : clib_unix_warning ("munmap");
401 : /* Reset the queue-length-address cache */
402 29 : vec_reset_length (vl_api_queue_cursizes);
403 29 : private_registration = 1;
404 29 : break;
405 : }
406 : }
407 :
408 47 : if (private_registration == 0)
409 : {
410 18 : pool_put_index (am->vl_clients,
411 : regp->vl_api_registration_pool_index);
412 18 : oldheap = vl_msg_push_heap ();
413 18 : if (mp->do_cleanup)
414 0 : svm_queue_free (regp->vl_input_queue);
415 18 : vec_free (regp->name);
416 : /* Poison the old registration */
417 18 : clib_memset (regp, 0xF1, sizeof (*regp));
418 18 : clib_mem_free (regp);
419 18 : vl_msg_pop_heap (oldheap);
420 : /*
421 : * These messages must be freed manually, since they're set up
422 : * as "bounce" messages. In the private_registration == 1 case,
423 : * we kill the shared-memory segment which contains the message
424 : * with munmap.
425 : */
426 18 : vl_msg_api_free (mp);
427 : }
428 : }
429 : else
430 : {
431 0 : clib_warning ("unknown client ID %d", mp->index);
432 : }
433 : }
434 :
435 : /**
436 : * client answered a ping, stave off the grim reaper...
437 : */
438 : void
439 0 : vl_api_memclnt_keepalive_reply_t_handler
440 : (vl_api_memclnt_keepalive_reply_t * mp)
441 : {
442 : vl_api_registration_t *regp;
443 0 : vlib_main_t *vm = vlib_get_main ();
444 :
445 0 : regp = vl_api_client_index_to_registration (mp->context);
446 0 : if (regp)
447 : {
448 0 : regp->last_heard = vlib_time_now (vm);
449 0 : regp->unanswered_pings = 0;
450 : }
451 : else
452 0 : clib_warning ("BUG: anonymous memclnt_keepalive_reply");
453 0 : }
454 :
455 : /**
456 : * We can send ourselves these messages if someone uses the
457 : * builtin binary api test tool...
458 : */
459 : static void
460 0 : vl_api_memclnt_keepalive_t_handler (vl_api_memclnt_keepalive_t * mp)
461 : {
462 : vl_api_memclnt_keepalive_reply_t *rmp;
463 : api_main_t *am;
464 : vl_shmem_hdr_t *shmem_hdr;
465 :
466 0 : am = vlibapi_get_main ();
467 0 : shmem_hdr = am->shmem_hdr;
468 :
469 0 : rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp));
470 0 : clib_memset (rmp, 0, sizeof (*rmp));
471 0 : rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY);
472 0 : rmp->context = mp->context;
473 0 : vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp);
474 0 : }
475 :
476 : /*
477 : * To avoid filling the API trace buffer with boring messages,
478 : * don't trace memclnt_keepalive[_reply] msgs
479 : */
480 :
481 : #define foreach_vlib_api_msg \
482 : _ (MEMCLNT_CREATE, memclnt_create, 0) \
483 : _ (MEMCLNT_CREATE_V2, memclnt_create_v2, 0) \
484 : _ (MEMCLNT_DELETE, memclnt_delete, 0) \
485 : _ (MEMCLNT_KEEPALIVE, memclnt_keepalive, 0) \
486 : _ (MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply, 0)
487 :
488 : /*
489 : * memory_api_init
490 : */
491 : int
492 575 : vl_mem_api_init (const char *region_name)
493 : {
494 : int rv;
495 575 : api_main_t *am = vlibapi_get_main ();
496 : vl_msg_api_msg_config_t cfg;
497 575 : vl_msg_api_msg_config_t *c = &cfg;
498 : vl_shmem_hdr_t *shm;
499 575 : vlib_main_t *vm = vlib_get_main ();
500 :
501 575 : clib_memset (c, 0, sizeof (*c));
502 :
503 575 : if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0)
504 0 : return rv;
505 :
506 : #define _(N, n, t) \
507 : do \
508 : { \
509 : c->id = VL_API_##N; \
510 : c->name = #n; \
511 : c->handler = vl_api_##n##_t_handler; \
512 : c->endian = vl_api_##n##_t_endian; \
513 : c->format_fn = vl_api_##n##_t_format; \
514 : c->size = sizeof (vl_api_##n##_t); \
515 : c->traced = t; /* trace, so these msgs print */ \
516 : c->replay = 0; /* don't replay client create/delete msgs */ \
517 : c->message_bounce = 0; /* don't bounce this message */ \
518 : vl_msg_api_config (c); \
519 : } \
520 : while (0);
521 :
522 575 : foreach_vlib_api_msg;
523 : #undef _
524 :
525 : #define vl_msg_name_crc_list
526 : #include <vlibmemory/memclnt.api.h>
527 : #undef vl_msg_name_crc_list
528 :
529 : #define _(id, n, crc) vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id);
530 575 : foreach_vl_msg_name_crc_memclnt;
531 : #undef _
532 :
533 : /*
534 : * special-case freeing of memclnt_delete messages, so we can
535 : * simply munmap pairwise / private API segments...
536 : */
537 575 : am->msg_data[VL_API_MEMCLNT_DELETE].bounce = 1;
538 575 : vl_api_set_msg_thread_safe (am, VL_API_MEMCLNT_KEEPALIVE_REPLY, 1);
539 575 : vl_api_set_msg_thread_safe (am, VL_API_MEMCLNT_KEEPALIVE, 1);
540 :
541 575 : vlib_set_queue_signal_callback (vm, memclnt_queue_callback);
542 :
543 575 : shm = am->shmem_hdr;
544 575 : ASSERT (shm && shm->vl_input_queue);
545 :
546 : /* Make a note so we can always find the primary region easily */
547 575 : am->vlib_primary_rp = am->vlib_rp;
548 :
549 575 : return 0;
550 : }
551 :
552 : clib_error_t *
553 575 : map_api_segment_init (vlib_main_t * vm)
554 : {
555 575 : api_main_t *am = vlibapi_get_main ();
556 : int rv;
557 :
558 575 : if ((rv = vl_mem_api_init (am->region_name)) < 0)
559 : {
560 0 : return clib_error_return (0, "vl_mem_api_init (%s) failed",
561 : am->region_name);
562 : }
563 575 : return 0;
564 : }
565 :
566 : static void
567 3 : send_memclnt_keepalive (vl_api_registration_t * regp, f64 now)
568 : {
569 : vl_api_memclnt_keepalive_t *mp;
570 : svm_queue_t *q;
571 3 : api_main_t *am = vlibapi_get_main ();
572 :
573 3 : q = regp->vl_input_queue;
574 :
575 : /*
576 : * If the queue head is moving, assume that the client is processing
577 : * messages and skip the ping. This heuristic may fail if the queue
578 : * is in the same position as last time, net of wrapping; in which
579 : * case, the client will receive a keepalive.
580 : */
581 3 : if (regp->last_queue_head != q->head)
582 : {
583 1 : regp->last_heard = now;
584 1 : regp->unanswered_pings = 0;
585 1 : regp->last_queue_head = q->head;
586 1 : return;
587 : }
588 :
589 : /*
590 : * push/pop shared memory segment, so this routine
591 : * will work with "normal" as well as "private segment"
592 : * memory clients..
593 : */
594 :
595 2 : mp = vl_mem_api_alloc_as_if_client_w_reg (regp, sizeof (*mp));
596 2 : clib_memset (mp, 0, sizeof (*mp));
597 2 : mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE);
598 4 : mp->context = mp->client_index =
599 2 : vl_msg_api_handle_from_index_and_epoch
600 : (regp->vl_api_registration_pool_index,
601 2 : am->shmem_hdr->application_restarts);
602 :
603 2 : regp->unanswered_pings++;
604 :
605 : /* Failure-to-send due to a stuffed queue is absolutely expected */
606 2 : if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ ))
607 0 : vl_msg_api_free_w_region (regp->vlib_rp, mp);
608 : }
609 :
610 : static void
611 4 : vl_mem_send_client_keepalive_w_reg (api_main_t * am, f64 now,
612 : vl_api_registration_t ** regpp,
613 : u32 ** dead_indices,
614 : u32 ** confused_indices)
615 : {
616 4 : vl_api_registration_t *regp = *regpp;
617 4 : if (regp)
618 : {
619 : /* If we haven't heard from this client recently... */
620 4 : if (regp->last_heard < (now - 10.0))
621 : {
622 4 : if (regp->unanswered_pings == 2)
623 : {
624 : svm_queue_t *q;
625 1 : q = regp->vl_input_queue;
626 1 : if (kill (q->consumer_pid, 0) >= 0)
627 : {
628 0 : clib_warning ("REAPER: lazy binary API client '%s'",
629 : regp->name);
630 0 : regp->unanswered_pings = 0;
631 0 : regp->last_heard = now;
632 : }
633 : else
634 : {
635 1 : clib_warning ("REAPER: binary API client '%s' died",
636 : regp->name);
637 1 : vec_add1 (*dead_indices, regpp - am->vl_clients);
638 : }
639 : }
640 : else
641 3 : send_memclnt_keepalive (regp, now);
642 : }
643 : else
644 0 : regp->unanswered_pings = 0;
645 : }
646 : else
647 : {
648 0 : clib_warning ("NULL client registration index %d",
649 : regpp - am->vl_clients);
650 0 : vec_add1 (*confused_indices, regpp - am->vl_clients);
651 : }
652 4 : }
653 :
654 : void
655 498 : vl_mem_api_dead_client_scan (api_main_t * am, vl_shmem_hdr_t * shm, f64 now)
656 : {
657 : vl_api_registration_t **regpp;
658 : static u32 *dead_indices;
659 : static u32 *confused_indices;
660 :
661 498 : vec_reset_length (dead_indices);
662 498 : vec_reset_length (confused_indices);
663 :
664 : /* *INDENT-OFF* */
665 503 : pool_foreach (regpp, am->vl_clients) {
666 5 : if (!(*regpp)->keepalive)
667 1 : continue;
668 4 : vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices,
669 : &confused_indices);
670 : }
671 : /* *INDENT-ON* */
672 :
673 : /* This should "never happen," but if it does, fix it... */
674 498 : if (PREDICT_FALSE (vec_len (confused_indices) > 0))
675 : {
676 : int i;
677 0 : for (i = 0; i < vec_len (confused_indices); i++)
678 : {
679 0 : pool_put_index (am->vl_clients, confused_indices[i]);
680 : }
681 : }
682 :
683 498 : if (PREDICT_FALSE (vec_len (dead_indices) > 0))
684 : {
685 : int i;
686 : void *oldheap;
687 :
688 : /* Allow the application to clean up its registrations */
689 2 : for (i = 0; i < vec_len (dead_indices); i++)
690 : {
691 1 : regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
692 1 : if (regpp)
693 : {
694 : u32 handle;
695 :
696 1 : handle = vl_msg_api_handle_from_index_and_epoch
697 1 : (dead_indices[i], shm->application_restarts);
698 1 : vl_api_call_reaper_functions (handle);
699 : }
700 : }
701 :
702 1 : oldheap = vl_msg_push_heap ();
703 :
704 2 : for (i = 0; i < vec_len (dead_indices); i++)
705 : {
706 1 : regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
707 1 : if (regpp)
708 : {
709 : /* Is this a pairwise SVM segment? */
710 1 : if ((*regpp)->vlib_rp != am->vlib_rp)
711 : {
712 : int i;
713 1 : svm_region_t *dead_rp = (*regpp)->vlib_rp;
714 : /* Note: account for the memfd header page */
715 1 : uword virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE;
716 1 : uword virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE;
717 :
718 : /* For horizontal scaling, add a hash table... */
719 1 : for (i = 0; i < vec_len (am->vlib_private_rps); i++)
720 1 : if (am->vlib_private_rps[i] == dead_rp)
721 : {
722 1 : vec_delete (am->vlib_private_rps, 1, i);
723 1 : goto found;
724 : }
725 0 : svm_pop_heap (oldheap);
726 0 : clib_warning ("private rp %llx AWOL", dead_rp);
727 0 : oldheap = svm_push_data_heap (am->vlib_rp);
728 :
729 1 : found:
730 : /* Kill it, accounting for the memfd header page */
731 1 : svm_pop_heap (oldheap);
732 1 : if (munmap ((void *) virtual_base, virtual_size) < 0)
733 0 : clib_unix_warning ("munmap");
734 : /* Reset the queue-length-address cache */
735 1 : vec_reset_length (vl_api_queue_cursizes);
736 1 : oldheap = svm_push_data_heap (am->vlib_rp);
737 : }
738 : else
739 : {
740 : /* Poison the old registration */
741 0 : clib_memset (*regpp, 0xF3, sizeof (**regpp));
742 0 : clib_mem_free (*regpp);
743 : }
744 : /* no dangling references, please */
745 1 : *regpp = 0;
746 : }
747 : else
748 : {
749 0 : svm_pop_heap (oldheap);
750 0 : clib_warning ("Duplicate free, client index %d",
751 : regpp - am->vl_clients);
752 0 : oldheap = svm_push_data_heap (am->vlib_rp);
753 : }
754 : }
755 :
756 1 : svm_client_scan_this_region_nolock (am->vlib_rp);
757 :
758 1 : vl_msg_pop_heap (oldheap);
759 2 : for (i = 0; i < vec_len (dead_indices); i++)
760 1 : pool_put_index (am->vl_clients, dead_indices[i]);
761 : }
762 498 : }
763 :
764 : void (*vl_mem_api_fuzz_hook) (u16, void *);
765 :
766 : /* This is only to be called from a vlib/vnet app */
767 : static void
768 63108 : vl_mem_api_handler_with_vm_node (api_main_t *am, svm_region_t *vlib_rp,
769 : void *the_msg, vlib_main_t *vm,
770 : vlib_node_runtime_t *node, u8 is_private)
771 : {
772 63108 : u16 id = clib_net_to_host_u16 (*((u16 *) the_msg));
773 63108 : vl_api_msg_data_t *m = vl_api_get_msg_data (am, id);
774 : u8 *(*handler) (void *, void *, void *);
775 : svm_region_t *old_vlib_rp;
776 : void *save_shmem_hdr;
777 63108 : int is_mp_safe = 1;
778 :
779 63108 : if (PREDICT_FALSE (am->elog_trace_api_messages))
780 : {
781 : ELOG_TYPE_DECLARE (e) = {
782 : .format = "api-msg: %s",
783 : .format_args = "T4",
784 : };
785 : struct
786 : {
787 : u32 c;
788 : } * ed;
789 63106 : ed = ELOG_DATA (am->elog_main, e);
790 63106 : if (m && m->name)
791 63104 : ed->c = elog_string (am->elog_main, (char *) m->name);
792 : else
793 2 : ed->c = elog_string (am->elog_main, "BOGUS");
794 : }
795 :
796 63108 : if (m && m->handler)
797 : {
798 63106 : handler = (void *) m->handler;
799 :
800 63106 : if (PREDICT_FALSE (am->rx_trace && am->rx_trace->enabled))
801 63106 : vl_msg_api_trace (am, am->rx_trace, the_msg);
802 :
803 63106 : if (PREDICT_FALSE (am->msg_print_flag))
804 : {
805 0 : fformat (stdout, "[%d]: %s\n", id, m->name);
806 0 : fformat (stdout, "%U", format_vl_api_msg_text, am, id, the_msg);
807 : }
808 63106 : is_mp_safe = am->msg_data[id].is_mp_safe;
809 :
810 63106 : if (!is_mp_safe)
811 : {
812 : vl_msg_api_barrier_trace_context (am->msg_data[id].name);
813 545 : vl_msg_api_barrier_sync ();
814 : }
815 63106 : if (is_private)
816 : {
817 211 : old_vlib_rp = am->vlib_rp;
818 211 : save_shmem_hdr = am->shmem_hdr;
819 211 : am->vlib_rp = vlib_rp;
820 211 : am->shmem_hdr = (void *) vlib_rp->user_ctx;
821 : }
822 :
823 63106 : if (PREDICT_FALSE (vl_mem_api_fuzz_hook != 0))
824 0 : (*vl_mem_api_fuzz_hook) (id, the_msg);
825 :
826 63106 : if (m->is_autoendian)
827 : {
828 : void (*endian_fp) (void *);
829 0 : endian_fp = am->msg_data[id].endian_handler;
830 0 : (*endian_fp) (the_msg);
831 : }
832 63106 : if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0))
833 0 : clib_call_callbacks (am->perf_counter_cbs, am, id, 0 /* before */);
834 :
835 63106 : (*handler) (the_msg, vm, node);
836 :
837 63106 : if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0))
838 0 : clib_call_callbacks (am->perf_counter_cbs, am, id, 1 /* after */);
839 63106 : if (is_private)
840 : {
841 211 : am->vlib_rp = old_vlib_rp;
842 211 : am->shmem_hdr = save_shmem_hdr;
843 : }
844 63106 : if (!is_mp_safe)
845 545 : vl_msg_api_barrier_release ();
846 : }
847 : else
848 : {
849 2 : clib_warning ("no handler for msg id %d", id);
850 : }
851 :
852 : /*
853 : * Special-case, so we can e.g. bounce messages off the vnet
854 : * main thread without copying them...
855 : */
856 63108 : if (!m || !m->bounce)
857 : {
858 63061 : if (is_private)
859 : {
860 182 : old_vlib_rp = am->vlib_rp;
861 182 : save_shmem_hdr = am->shmem_hdr;
862 182 : am->vlib_rp = vlib_rp;
863 182 : am->shmem_hdr = (void *) vlib_rp->user_ctx;
864 : }
865 63061 : vl_msg_api_free (the_msg);
866 63061 : if (is_private)
867 : {
868 182 : am->vlib_rp = old_vlib_rp;
869 182 : am->shmem_hdr = save_shmem_hdr;
870 : }
871 : }
872 :
873 63108 : if (PREDICT_FALSE (am->elog_trace_api_messages))
874 : {
875 : ELOG_TYPE_DECLARE (e) = { .format = "api-msg-done(%s): %s",
876 : .format_args = "t4T4",
877 : .n_enum_strings = 2,
878 : .enum_strings = {
879 : "barrier",
880 : "mp-safe",
881 : } };
882 :
883 : struct
884 : {
885 : u32 barrier;
886 : u32 c;
887 : } * ed;
888 63106 : ed = ELOG_DATA (am->elog_main, e);
889 63106 : if (m && m->name)
890 63104 : ed->c = elog_string (am->elog_main, (char *) m->name);
891 : else
892 2 : ed->c = elog_string (am->elog_main, "BOGUS");
893 63106 : ed->barrier = is_mp_safe;
894 : }
895 63108 : }
896 :
897 : static inline int
898 495089 : void_mem_api_handle_msg_i (api_main_t * am, svm_region_t * vlib_rp,
899 : vlib_main_t * vm, vlib_node_runtime_t * node,
900 : u8 is_private)
901 : {
902 : svm_queue_t *q;
903 : uword mp;
904 :
905 495089 : q = ((vl_shmem_hdr_t *) (void *) vlib_rp->user_ctx)->vl_input_queue;
906 :
907 495089 : if (!svm_queue_sub2 (q, (u8 *) & mp))
908 : {
909 62885 : VL_MSG_API_UNPOISON ((void *) mp);
910 62885 : vl_mem_api_handler_with_vm_node (am, vlib_rp, (void *) mp, vm, node,
911 : is_private);
912 62885 : return 0;
913 : }
914 432204 : return -1;
915 : }
916 :
917 : int
918 494350 : vl_mem_api_handle_msg_main (vlib_main_t * vm, vlib_node_runtime_t * node)
919 : {
920 494350 : api_main_t *am = vlibapi_get_main ();
921 494350 : return void_mem_api_handle_msg_i (am, am->vlib_rp, vm, node,
922 : 0 /* is_private */ );
923 : }
924 :
925 : int
926 494350 : vl_mem_api_handle_rpc (vlib_main_t * vm, vlib_node_runtime_t * node)
927 : {
928 494350 : api_main_t *am = vlibapi_get_main ();
929 : int i;
930 : uword *tmp, mp;
931 :
932 : /*
933 : * Swap pending and processing vectors, then process the RPCs
934 : * Avoid deadlock conditions by construction.
935 : */
936 494350 : clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
937 494350 : tmp = vm->processing_rpc_requests;
938 494350 : vec_reset_length (tmp);
939 494350 : vm->processing_rpc_requests = vm->pending_rpc_requests;
940 494350 : vm->pending_rpc_requests = tmp;
941 494350 : clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
942 :
943 : /*
944 : * RPCs are used to reflect function calls to thread 0
945 : * when the underlying code is not thread-safe.
946 : *
947 : * Grabbing the thread barrier across a set of RPCs
948 : * greatly increases efficiency, and avoids
949 : * running afoul of the barrier sync holddown timer.
950 : * The barrier sync code supports recursive locking.
951 : *
952 : * We really need to rewrite RPC-based code...
953 : */
954 494350 : if (PREDICT_TRUE (vec_len (vm->processing_rpc_requests)))
955 : {
956 213 : vl_msg_api_barrier_sync ();
957 436 : for (i = 0; i < vec_len (vm->processing_rpc_requests); i++)
958 : {
959 223 : mp = vm->processing_rpc_requests[i];
960 223 : vl_mem_api_handler_with_vm_node (am, am->vlib_rp, (void *) mp, vm,
961 : node, 0 /* is_private */);
962 : }
963 213 : vl_msg_api_barrier_release ();
964 : }
965 :
966 494350 : return 0;
967 : }
968 :
969 : int
970 739 : vl_mem_api_handle_msg_private (vlib_main_t * vm, vlib_node_runtime_t * node,
971 : u32 reg_index)
972 : {
973 739 : api_main_t *am = vlibapi_get_main ();
974 739 : return void_mem_api_handle_msg_i (am, am->vlib_private_rps[reg_index], vm,
975 : node, 1 /* is_private */ );
976 : }
977 :
978 : vl_api_registration_t *
979 805 : vl_mem_api_client_index_to_registration (u32 handle)
980 : {
981 : vl_api_registration_t **regpp;
982 : vl_api_registration_t *regp;
983 805 : api_main_t *am = vlibapi_get_main ();
984 : vl_shmem_hdr_t *shmem_hdr;
985 : u32 index;
986 :
987 805 : index = vl_msg_api_handle_get_index (handle);
988 805 : regpp = am->vl_clients + index;
989 :
990 805 : if (pool_is_free (am->vl_clients, regpp))
991 : {
992 1 : vl_msg_api_increment_missing_client_counter ();
993 1 : return 0;
994 : }
995 804 : regp = *regpp;
996 :
997 804 : shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr;
998 804 : if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts))
999 : {
1000 0 : vl_msg_api_increment_missing_client_counter ();
1001 0 : return 0;
1002 : }
1003 :
1004 804 : return (regp);
1005 : }
1006 :
1007 : svm_queue_t *
1008 0 : vl_api_client_index_to_input_queue (u32 index)
1009 : {
1010 : vl_api_registration_t *regp;
1011 0 : api_main_t *am = vlibapi_get_main ();
1012 :
1013 : /* Special case: vlib trying to send itself a message */
1014 0 : if (index == (u32) ~ 0)
1015 0 : return (am->shmem_hdr->vl_input_queue);
1016 :
1017 0 : regp = vl_mem_api_client_index_to_registration (index);
1018 0 : if (!regp)
1019 0 : return 0;
1020 0 : return (regp->vl_input_queue);
1021 : }
1022 :
1023 : static clib_error_t *
1024 575 : setup_memclnt_exit (vlib_main_t * vm)
1025 : {
1026 575 : atexit (vl_unmap_shmem);
1027 575 : return 0;
1028 : }
1029 :
1030 100223 : VLIB_INIT_FUNCTION (setup_memclnt_exit);
1031 :
1032 : u8 *
1033 0 : format_api_message_rings (u8 * s, va_list * args)
1034 : {
1035 0 : api_main_t *am = va_arg (*args, api_main_t *);
1036 0 : vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *);
1037 0 : int main_segment = va_arg (*args, int);
1038 : ring_alloc_t *ap;
1039 : int i;
1040 :
1041 0 : if (shmem_hdr == 0)
1042 0 : return format (s, "%8s %8s %8s %8s %8s\n",
1043 : "Owner", "Size", "Nitems", "Hits", "Misses");
1044 :
1045 0 : ap = shmem_hdr->vl_rings;
1046 :
1047 0 : for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++)
1048 : {
1049 0 : s = format (s, "%8s %8d %8d %8d %8d\n",
1050 0 : "vlib", ap->size, ap->nitems, ap->hits, ap->misses);
1051 0 : ap++;
1052 : }
1053 :
1054 0 : ap = shmem_hdr->client_rings;
1055 :
1056 0 : for (i = 0; i < vec_len (shmem_hdr->client_rings); i++)
1057 : {
1058 0 : s = format (s, "%8s %8d %8d %8d %8d\n",
1059 0 : "clnt", ap->size, ap->nitems, ap->hits, ap->misses);
1060 0 : ap++;
1061 : }
1062 :
1063 0 : if (main_segment)
1064 : {
1065 0 : s = format (s, "%d ring miss fallback allocations\n", am->ring_misses);
1066 0 : s = format
1067 : (s,
1068 : "%d application restarts, %d reclaimed msgs, %d garbage collects\n",
1069 : shmem_hdr->application_restarts, shmem_hdr->restart_reclaims,
1070 : shmem_hdr->garbage_collects);
1071 : }
1072 0 : return s;
1073 : }
1074 :
1075 : static clib_error_t *
1076 0 : vl_api_ring_command (vlib_main_t * vm,
1077 : unformat_input_t * input, vlib_cli_command_t * cli_cmd)
1078 : {
1079 : int i;
1080 : vl_shmem_hdr_t *shmem_hdr;
1081 0 : api_main_t *am = vlibapi_get_main ();
1082 :
1083 : /* First, dump the primary region rings.. */
1084 :
1085 0 : if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0)
1086 : {
1087 0 : vlib_cli_output (vm, "Shared memory segment not initialized...\n");
1088 0 : return 0;
1089 : }
1090 :
1091 0 : shmem_hdr = (void *) am->vlib_primary_rp->user_ctx;
1092 :
1093 0 : vlib_cli_output (vm, "Main API segment rings:");
1094 :
1095 0 : vlib_cli_output (vm, "%U", format_api_message_rings, am,
1096 : 0 /* print header */ , 0 /* notused */ );
1097 :
1098 0 : vlib_cli_output (vm, "%U", format_api_message_rings, am,
1099 : shmem_hdr, 1 /* main segment */ );
1100 :
1101 0 : for (i = 0; i < vec_len (am->vlib_private_rps); i++)
1102 : {
1103 0 : svm_region_t *vlib_rp = am->vlib_private_rps[i];
1104 0 : shmem_hdr = (void *) vlib_rp->user_ctx;
1105 : vl_api_registration_t **regpp;
1106 0 : vl_api_registration_t *regp = 0;
1107 :
1108 : /* For horizontal scaling, add a hash table... */
1109 : /* *INDENT-OFF* */
1110 0 : pool_foreach (regpp, am->vl_clients)
1111 : {
1112 0 : regp = *regpp;
1113 0 : if (regp && regp->vlib_rp == vlib_rp)
1114 : {
1115 0 : vlib_cli_output (vm, "%s segment rings:", regp->name);
1116 0 : goto found;
1117 : }
1118 : }
1119 0 : vlib_cli_output (vm, "regp %llx not found?", regp);
1120 0 : continue;
1121 : /* *INDENT-ON* */
1122 0 : found:
1123 0 : vlib_cli_output (vm, "%U", format_api_message_rings, am,
1124 : 0 /* print header */ , 0 /* notused */ );
1125 0 : vlib_cli_output (vm, "%U", format_api_message_rings, am,
1126 : shmem_hdr, 0 /* main segment */ );
1127 : }
1128 :
1129 0 : return 0;
1130 : }
1131 :
1132 : /*?
1133 : * Display binary api message allocation ring statistics
1134 : ?*/
1135 : /* *INDENT-OFF* */
1136 285289 : VLIB_CLI_COMMAND (cli_show_api_ring_command, static) =
1137 : {
1138 : .path = "show api ring-stats",
1139 : .short_help = "Message ring statistics",
1140 : .function = vl_api_ring_command,
1141 : };
1142 : /* *INDENT-ON* */
1143 :
1144 : clib_error_t *
1145 575 : vlibmemory_init (vlib_main_t * vm)
1146 : {
1147 575 : api_main_t *am = vlibapi_get_main ();
1148 575 : svm_map_region_args_t _a, *a = &_a;
1149 : u8 *remove_path1, *remove_path2;
1150 : void vlibsocket_reference (void);
1151 :
1152 575 : vlibsocket_reference ();
1153 :
1154 : /*
1155 : * By popular request / to avoid support fires, remove any old api segment
1156 : * files Right Here.
1157 : */
1158 575 : if (am->root_path == 0)
1159 : {
1160 0 : remove_path1 = format (0, "/dev/shm/global_vm%c", 0);
1161 0 : remove_path2 = format (0, "/dev/shm/vpe-api%c", 0);
1162 : }
1163 : else
1164 : {
1165 575 : remove_path1 = format (0, "/dev/shm/%s-global_vm%c", am->root_path, 0);
1166 575 : remove_path2 = format (0, "/dev/shm/%s-vpe-api%c", am->root_path, 0);
1167 : }
1168 :
1169 575 : (void) unlink ((char *) remove_path1);
1170 575 : (void) unlink ((char *) remove_path2);
1171 :
1172 575 : vec_free (remove_path1);
1173 575 : vec_free (remove_path2);
1174 :
1175 575 : clib_memset (a, 0, sizeof (*a));
1176 575 : a->root_path = am->root_path;
1177 575 : a->name = SVM_GLOBAL_REGION_NAME;
1178 1150 : a->baseva = (am->global_baseva != 0) ?
1179 575 : am->global_baseva : +svm_get_global_region_base_va ();
1180 575 : a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
1181 575 : a->flags = SVM_FLAGS_NODATA;
1182 575 : a->uid = am->api_uid;
1183 575 : a->gid = am->api_gid;
1184 575 : a->pvt_heap_size =
1185 575 : (am->global_pvt_heap_size !=
1186 575 : 0) ? am->global_pvt_heap_size : SVM_PVT_MHEAP_SIZE;
1187 :
1188 575 : svm_region_init_args (a);
1189 :
1190 575 : return 0;
1191 : }
1192 :
1193 : void
1194 575 : vl_set_memory_region_name (const char *name)
1195 : {
1196 575 : api_main_t *am = vlibapi_get_main ();
1197 575 : am->region_name = name;
1198 575 : }
1199 :
1200 : /*
1201 : * fd.io coding-style-patch-verification: ON
1202 : *
1203 : * Local Variables:
1204 : * eval: (c-set-style "gnu")
1205 : * End:
1206 : */
|
