Line data Source code
1 : /*
2 : * Copyright (c) 2018-2019 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef SRC_VCL_VCL_PRIVATE_H_
17 : #define SRC_VCL_VCL_PRIVATE_H_
18 :
19 : #include <vnet/session/application_interface.h>
20 : #include <vcl/vppcom.h>
21 : #include <vcl/vcl_debug.h>
22 :
23 : #if (CLIB_DEBUG > 0)
24 : /* Set VPPCOM_DEBUG_INIT 2 for connection debug,
25 : * 3 for read/write debug output
26 : * or
27 : * export VCL_DEBUG=<#> to set dynamically.
28 : */
29 : #define VPPCOM_DEBUG_INIT 1
30 : #else
31 : #define VPPCOM_DEBUG_INIT 0
32 : #endif
33 :
34 : #define VPPCOM_DEBUG vcm->debug
35 :
36 : extern __thread uword __vcl_worker_index;
37 :
38 : static inline void
39 103 : vcl_set_worker_index (uword wrk_index)
40 : {
41 103 : __vcl_worker_index = wrk_index;
42 103 : }
43 :
44 : static inline uword
45 184582493 : vcl_get_worker_index (void)
46 : {
47 184582493 : return __vcl_worker_index;
48 : }
49 :
50 : /*
51 : * VPPCOM Private definitions and functions.
52 : */
53 : typedef enum
54 : {
55 : STATE_APP_START,
56 : STATE_APP_CONN_VPP,
57 : STATE_APP_ENABLED,
58 : STATE_APP_ATTACHED,
59 : STATE_APP_ADDING_WORKER,
60 : STATE_APP_ADDING_TLS_DATA,
61 : STATE_APP_FAILED,
62 : STATE_APP_READY
63 : } vcl_bapi_app_state_t;
64 :
65 : typedef enum vcl_session_state_
66 : {
67 : VCL_STATE_CLOSED,
68 : VCL_STATE_LISTEN,
69 : VCL_STATE_READY,
70 : VCL_STATE_VPP_CLOSING,
71 : VCL_STATE_DISCONNECT,
72 : VCL_STATE_DETACHED,
73 : VCL_STATE_UPDATED,
74 : VCL_STATE_LISTEN_NO_MQ,
75 : } vcl_session_state_t;
76 :
77 : typedef struct epoll_event vppcom_epoll_event_t;
78 :
79 : typedef struct
80 : {
81 : u32 next_sh;
82 : u32 prev_sh;
83 : u32 vep_sh;
84 : vppcom_epoll_event_t ev;
85 : #define VEP_DEFAULT_ET_MASK (EPOLLIN|EPOLLOUT)
86 : #define VEP_UNSUPPORTED_EVENTS (EPOLLONESHOT|EPOLLEXCLUSIVE)
87 : u32 et_mask;
88 : u32 lt_next;
89 : u32 lt_prev;
90 : } vppcom_epoll_t;
91 :
92 : /* Select uses the vcl_si_set as if a clib_bitmap. Make sure they are the
93 : * same size */
94 : STATIC_ASSERT (sizeof (clib_bitmap_t) == sizeof (vcl_si_set),
95 : "vppcom bitmap size mismatch");
96 :
97 : typedef struct
98 : {
99 : u8 is_ip4;
100 : ip46_address_t ip46;
101 : } vppcom_ip46_t;
102 :
103 : #define VCL_ACCEPTED_F_CLOSED (1 << 0)
104 : #define VCL_ACCEPTED_F_RESET (1 << 1)
105 :
106 : typedef struct vcl_session_msg
107 : {
108 : union
109 : {
110 : session_accepted_msg_t accepted_msg;
111 : };
112 : u32 flags;
113 : } vcl_session_msg_t;
114 :
115 : typedef enum
116 : {
117 : VCL_SESS_ATTR_SERVER,
118 : VCL_SESS_ATTR_CUT_THRU,
119 : VCL_SESS_ATTR_VEP,
120 : VCL_SESS_ATTR_VEP_SESSION,
121 : VCL_SESS_ATTR_LISTEN, // SOL_SOCKET,SO_ACCEPTCONN
122 : VCL_SESS_ATTR_NONBLOCK, // fcntl,O_NONBLOCK
123 : VCL_SESS_ATTR_REUSEADDR, // SOL_SOCKET,SO_REUSEADDR
124 : VCL_SESS_ATTR_REUSEPORT, // SOL_SOCKET,SO_REUSEPORT
125 : VCL_SESS_ATTR_BROADCAST, // SOL_SOCKET,SO_BROADCAST
126 : VCL_SESS_ATTR_V6ONLY, // SOL_TCP,IPV6_V6ONLY
127 : VCL_SESS_ATTR_KEEPALIVE, // SOL_SOCKET,SO_KEEPALIVE
128 : VCL_SESS_ATTR_TCP_NODELAY, // SOL_TCP,TCP_NODELAY
129 : VCL_SESS_ATTR_TCP_KEEPIDLE, // SOL_TCP,TCP_KEEPIDLE
130 : VCL_SESS_ATTR_TCP_KEEPINTVL, // SOL_TCP,TCP_KEEPINTVL
131 : VCL_SESS_ATTR_IP_PKTINFO, /* IPPROTO_IP, IP_PKTINFO */
132 : VCL_SESS_ATTR_MAX
133 : } vppcom_session_attr_t;
134 :
135 : typedef enum vcl_session_flags_
136 : {
137 : VCL_SESSION_F_CONNECTED = 1 << 0,
138 : VCL_SESSION_F_IS_VEP = 1 << 1,
139 : VCL_SESSION_F_IS_VEP_SESSION = 1 << 2,
140 : VCL_SESSION_F_HAS_RX_EVT = 1 << 3,
141 : VCL_SESSION_F_RD_SHUTDOWN = 1 << 4,
142 : VCL_SESSION_F_WR_SHUTDOWN = 1 << 5,
143 : VCL_SESSION_F_PENDING_DISCONNECT = 1 << 6,
144 : VCL_SESSION_F_PENDING_FREE = 1 << 7,
145 : VCL_SESSION_F_PENDING_LISTEN = 1 << 8,
146 : } __clib_packed vcl_session_flags_t;
147 :
148 : typedef struct vcl_session_
149 : {
150 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
151 :
152 : #define _(type, name) type name;
153 : foreach_app_session_field
154 : #undef _
155 : vcl_session_flags_t flags; /**< see @ref vcl_session_flags_t */
156 : u32 rx_bytes_pending; /**< bytes rx-ed as segs but not yet freed */
157 :
158 : svm_fifo_t *ct_rx_fifo;
159 : svm_fifo_t *ct_tx_fifo;
160 : vcl_session_msg_t *accept_evts_fifo;
161 :
162 : u64 vpp_handle;
163 : u64 parent_handle;
164 : u32 listener_index; /**< index of parent listener (if any) */
165 : int n_accepted_sessions; /**< sessions accepted by this listener */
166 : vppcom_epoll_t vep;
167 : u32 attributes; /**< see @ref vppcom_session_attr_t */
168 : int libc_epfd;
169 : u32 vrf;
170 : u16 gso_size;
171 :
172 : u32 sndbuf_size; // VPP-TBD: Hack until support setsockopt(SO_SNDBUF)
173 : u32 rcvbuf_size; // VPP-TBD: Hack until support setsockopt(SO_RCVBUF)
174 :
175 : transport_endpt_ext_cfg_t *ext_config;
176 : u8 dscp;
177 :
178 : i32 vpp_error;
179 :
180 : #if VCL_ELOG
181 : elog_track_t elog_track;
182 : #endif
183 : } vcl_session_t;
184 :
185 : typedef struct vppcom_cfg_t_
186 : {
187 : uword heapsize;
188 : u32 max_workers;
189 : uword segment_baseva;
190 : uword segment_size;
191 : uword add_segment_size;
192 : u32 preallocated_fifo_pairs;
193 : u32 rx_fifo_size;
194 : u32 tx_fifo_size;
195 : u32 event_queue_size;
196 : u8 app_proxy_transport_tcp;
197 : u8 app_proxy_transport_udp;
198 : u8 app_scope_local;
199 : u8 app_scope_global;
200 : u8 *namespace_id;
201 : u64 namespace_secret;
202 : u8 use_mq_eventfd;
203 : f64 app_timeout;
204 : f64 session_timeout;
205 : char *event_log_path;
206 : u8 *vpp_app_socket_api; /**< app socket api socket file name */
207 : u8 *vpp_bapi_socket_name; /**< bapi socket transport socket name */
208 : u32 tls_engine;
209 : u8 mt_wrk_supported;
210 : u8 huge_page;
211 : } vppcom_cfg_t;
212 :
213 : void vppcom_cfg (vppcom_cfg_t * vcl_cfg);
214 :
215 : typedef struct vcl_cut_through_registration_
216 : {
217 : svm_msg_q_t *mq;
218 : svm_msg_q_t *peer_mq;
219 : u32 sid;
220 : u32 epoll_evt_conn_index; /*< mq evt connection index part of
221 : the mqs evtfd epoll (if used) */
222 : } vcl_cut_through_registration_t;
223 :
224 : typedef struct vcl_mq_evt_conn_
225 : {
226 : svm_msg_q_t *mq;
227 : int mq_fd;
228 : } vcl_mq_evt_conn_t;
229 :
230 : typedef struct vcl_worker_
231 : {
232 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
233 :
234 : /* Session pool */
235 : vcl_session_t *sessions;
236 :
237 : /** Worker/thread index in current process */
238 : u32 wrk_index;
239 :
240 : /** Worker index in vpp*/
241 : u32 vpp_wrk_index;
242 :
243 : /**
244 : * Generic api client handle. When binary api is in used, it stores
245 : * the "client_index" and when socket api is use, it stores the sapi
246 : * client handle */
247 : u32 api_client_handle;
248 :
249 : /** VPP binary api input queue */
250 : svm_queue_t *vl_input_queue;
251 :
252 : /** VPP mq to be used for exchanging control messages */
253 : svm_msg_q_t *ctrl_mq;
254 :
255 : /** Message queues epoll fd. Initialized only if using mqs with eventfds */
256 : int mqs_epfd;
257 :
258 : /** Pool of event message queue event connections */
259 : vcl_mq_evt_conn_t *mq_evt_conns;
260 :
261 : /** Per worker buffer for receiving mq epoll events */
262 : struct epoll_event *mq_events;
263 :
264 : /** Next session to be lt polled */
265 : u32 ep_lt_current;
266 :
267 : /** Hash table for disconnect processing */
268 : uword *session_index_by_vpp_handles;
269 :
270 : /** Select bitmaps */
271 : clib_bitmap_t *rd_bitmap;
272 : clib_bitmap_t *wr_bitmap;
273 : clib_bitmap_t *ex_bitmap;
274 :
275 : /** Our event message queue */
276 : svm_msg_q_t *app_event_queue;
277 :
278 : /** For deadman timers */
279 : clib_time_t clib_time;
280 :
281 : /** Vector acting as buffer for mq messages */
282 : svm_msg_q_msg_t *mq_msg_vector;
283 :
284 : /** Vector of unhandled events */
285 : session_event_t *unhandled_evts_vector;
286 :
287 : u32 *pending_session_wrk_updates;
288 :
289 : /** Used also as a thread stop key buffer */
290 : pthread_t thread_id;
291 :
292 : /** Current pid, may be different from main_pid if forked child */
293 : pid_t current_pid;
294 :
295 : u32 forked_child;
296 :
297 : clib_socket_t app_api_sock;
298 : socket_client_main_t bapi_sock_ctx;
299 : api_main_t bapi_api_ctx;
300 : memory_client_main_t bapi_mem_ctx;
301 :
302 : /* State of the connection, shared between msg RX thread and main thread */
303 : volatile vcl_bapi_app_state_t bapi_app_state;
304 : volatile uword bapi_return;
305 :
306 : u8 session_attr_op;
307 : int session_attr_op_rv;
308 : transport_endpt_attr_t session_attr_rv;
309 :
310 : /** vcl needs next epoll_create to go to libc_epoll */
311 : u8 vcl_needs_real_epoll;
312 : volatile int rpc_done;
313 : } vcl_worker_t;
314 :
315 : STATIC_ASSERT (sizeof (session_disconnected_msg_t) <= 16,
316 : "disconnected must fit in session_event_t");
317 : STATIC_ASSERT (sizeof (session_reset_msg_t) <= 16,
318 : "disconnected must fit in session_event_t");
319 :
320 : typedef void (vcl_rpc_fn_t) (void *args);
321 :
322 : typedef struct vppcom_main_t_
323 : {
324 : u8 is_init;
325 : u32 debug;
326 : pthread_t main_cpu;
327 :
328 : /** Main process pid */
329 : pid_t main_pid;
330 :
331 : /** App's index in vpp. It's used by vpp to identify the app */
332 : u32 app_index;
333 :
334 : u8 *app_name;
335 :
336 : /** VCL configuration */
337 : vppcom_cfg_t cfg;
338 :
339 : volatile u32 forking;
340 :
341 : /** Workers */
342 : vcl_worker_t *workers;
343 :
344 : /** Lock to protect worker registrations */
345 : clib_spinlock_t workers_lock;
346 :
347 : /** Counter to determine order of execution of `vcl_api_retry_attach`
348 : * function by multiple workers */
349 : int reattach_count;
350 :
351 : /** Lock to protect segment hash table */
352 : clib_rwlock_t segment_table_lock;
353 :
354 : /** Mapped segments table */
355 : uword *segment_table;
356 :
357 : /** Control mq obtained from attach */
358 : svm_msg_q_t *ctrl_mq;
359 :
360 : fifo_segment_main_t segment_main;
361 :
362 : vcl_rpc_fn_t *wrk_rpc_fn;
363 :
364 : /*
365 : * Binary api context
366 : */
367 :
368 : /* VNET_API_ERROR_FOO -> "Foo" hash table */
369 : uword *error_string_by_error_number;
370 :
371 : #ifdef VCL_ELOG
372 : /* VPP Event-logger */
373 : elog_main_t elog_main;
374 : elog_track_t elog_track;
375 : #endif
376 :
377 : } vppcom_main_t;
378 :
379 : extern vppcom_main_t *vcm;
380 : extern vppcom_main_t _vppcom_main;
381 :
382 : #define VCL_INVALID_SESSION_INDEX ((u32)~0)
383 : #define VCL_INVALID_SESSION_HANDLE ((u64)~0)
384 : #define VCL_INVALID_SEGMENT_INDEX ((u32)~0)
385 : #define VCL_INVALID_SEGMENT_HANDLE ((u64)~0)
386 :
387 : void vcl_session_detach_fifos (vcl_session_t *s);
388 :
389 : static inline vcl_session_t *
390 135 : vcl_session_alloc (vcl_worker_t * wrk)
391 : {
392 : vcl_session_t *s;
393 135 : pool_get (wrk->sessions, s);
394 135 : memset (s, 0, sizeof (*s));
395 135 : s->session_index = s - wrk->sessions;
396 135 : s->listener_index = VCL_INVALID_SESSION_INDEX;
397 135 : return s;
398 : }
399 :
400 : static inline void
401 24 : vcl_session_free (vcl_worker_t * wrk, vcl_session_t * s)
402 : {
403 : /* Debug level set to 1 to avoid debug messages while ldp is cleaning up */
404 24 : VDBG (1, "session %u [0x%llx] removed", s->session_index, s->vpp_handle);
405 24 : vcl_session_detach_fifos (s);
406 24 : if (s->ext_config)
407 0 : clib_mem_free (s->ext_config);
408 24 : pool_put (wrk->sessions, s);
409 24 : }
410 :
411 : static inline vcl_session_t *
412 181831000 : vcl_session_get (vcl_worker_t * wrk, u32 session_index)
413 : {
414 181831000 : if (pool_is_free_index (wrk->sessions, session_index))
415 1 : return 0;
416 181831000 : return pool_elt_at_index (wrk->sessions, session_index);
417 : }
418 :
419 : static inline vcl_session_handle_t
420 442744 : vcl_session_handle_from_wrk_session_index (u32 session_index, u32 wrk_index)
421 : {
422 442744 : ASSERT (session_index < 2 << 24);
423 442744 : return (wrk_index << 24 | session_index);
424 : }
425 :
426 : static inline vcl_session_handle_t
427 3971875 : vcl_session_handle_from_index (u32 session_index)
428 : {
429 3971875 : ASSERT (session_index < 2 << 24);
430 3971875 : return (vcl_get_worker_index () << 24 | session_index);
431 : }
432 :
433 : static inline vcl_session_handle_t
434 135 : vcl_session_handle (vcl_session_t * s)
435 : {
436 135 : return vcl_session_handle_from_index (s->session_index);
437 : }
438 :
439 : static inline void
440 123736000 : vcl_session_handle_parse (u32 handle, u32 * wrk_index, u32 * session_index)
441 : {
442 123736000 : *wrk_index = handle >> 24;
443 123736000 : *session_index = handle & 0xFFFFFF;
444 123736000 : }
445 :
446 : static inline vcl_session_t *
447 123736000 : vcl_session_get_w_handle (vcl_worker_t * wrk, u32 session_handle)
448 : {
449 : u32 session_index, wrk_index;
450 123736000 : vcl_session_handle_parse (session_handle, &wrk_index, &session_index);
451 123736000 : ASSERT (wrk_index == wrk->wrk_index);
452 123736000 : return vcl_session_get (wrk, session_index);
453 : }
454 :
455 : static inline vcl_session_t *
456 80 : vcl_session_get_w_vpp_handle (vcl_worker_t * wrk, u64 vpp_handle)
457 : {
458 : uword *p;
459 80 : if ((p = hash_get (wrk->session_index_by_vpp_handles, vpp_handle)))
460 43 : return vcl_session_get (wrk, (u32) p[0]);
461 37 : return 0;
462 : }
463 :
464 : static inline u32
465 5 : vcl_session_index_from_vpp_handle (vcl_worker_t * wrk, u64 vpp_handle)
466 : {
467 : uword *p;
468 5 : if ((p = hash_get (wrk->session_index_by_vpp_handles, vpp_handle)))
469 5 : return p[0];
470 0 : return VCL_INVALID_SESSION_INDEX;
471 : }
472 :
473 : static inline void
474 121 : vcl_session_table_add_vpp_handle (vcl_worker_t * wrk, u64 handle, u32 value)
475 : {
476 121 : hash_set (wrk->session_index_by_vpp_handles, handle, value);
477 121 : }
478 :
479 : static inline void
480 24 : vcl_session_table_del_vpp_handle (vcl_worker_t * wrk, u64 vpp_handle)
481 : {
482 24 : hash_unset (wrk->session_index_by_vpp_handles, vpp_handle);
483 24 : }
484 :
485 : static inline uword *
486 : vcl_session_table_lookup_vpp_handle (vcl_worker_t * wrk, u64 handle)
487 : {
488 : return hash_get (wrk->session_index_by_vpp_handles, handle);
489 : }
490 :
491 : static inline void
492 40 : vcl_session_table_add_listener (vcl_worker_t * wrk, u64 listener_handle,
493 : u32 value)
494 : {
495 40 : hash_set (wrk->session_index_by_vpp_handles, listener_handle, value);
496 40 : }
497 :
498 : static inline void
499 : vcl_session_table_del_listener (vcl_worker_t * wrk, u64 listener_handle)
500 : {
501 : hash_unset (wrk->session_index_by_vpp_handles, listener_handle);
502 : }
503 :
504 : static inline int
505 4729 : vcl_session_is_connectable_listener (vcl_worker_t * wrk,
506 : vcl_session_t * session)
507 : {
508 : /* Tell if we session_handle is a QUIC session.
509 : * We can be in the following cases :
510 : * Listen session <- QUIC session <- Stream session
511 : * QUIC session <- Stream session
512 : */
513 : vcl_session_t *ls;
514 4729 : if (session->session_type != VPPCOM_PROTO_QUIC)
515 4729 : return 0;
516 0 : if (session->listener_index == VCL_INVALID_SESSION_INDEX)
517 0 : return !(session->session_state == VCL_STATE_LISTEN);
518 0 : ls = vcl_session_get_w_handle (wrk, session->listener_index);
519 0 : if (!ls)
520 0 : return VPPCOM_EBADFD;
521 0 : return ls->session_state == VCL_STATE_LISTEN;
522 : }
523 :
524 : static inline vcl_session_t *
525 37 : vcl_session_table_lookup_listener (vcl_worker_t * wrk, u64 handle)
526 : {
527 : uword *p;
528 : vcl_session_t *s;
529 :
530 37 : p = hash_get (wrk->session_index_by_vpp_handles, handle);
531 37 : if (!p)
532 : {
533 0 : VDBG (0, "could not find listen session: unknown vpp listener handle"
534 : " %llx", handle);
535 0 : return 0;
536 : }
537 37 : s = vcl_session_get (wrk, p[0]);
538 37 : if (!s)
539 : {
540 0 : VDBG (1, "invalid listen session index (%u)", p[0]);
541 0 : return 0;
542 : }
543 :
544 37 : if (s->session_state == VCL_STATE_DISCONNECT)
545 : {
546 0 : VDBG (0, "listen session [0x%llx] is closing", s->vpp_handle);
547 0 : return 0;
548 : }
549 :
550 37 : ASSERT (s->session_state == VCL_STATE_LISTEN
551 : || s->session_state == VCL_STATE_LISTEN_NO_MQ
552 : || vcl_session_is_connectable_listener (wrk, s));
553 37 : return s;
554 : }
555 :
556 : static inline u8
557 52900640 : vcl_session_is_ct (vcl_session_t * s)
558 : {
559 52900640 : return (s->ct_tx_fifo != 0);
560 : }
561 :
562 : static inline u8
563 14213357 : vcl_session_is_cl (vcl_session_t * s)
564 : {
565 14213357 : if (s->session_type == VPPCOM_PROTO_UDP)
566 892 : return !(s->flags & VCL_SESSION_F_CONNECTED);
567 14212450 : return 0;
568 : }
569 :
570 : static inline u8
571 6 : vcl_session_has_crypto (vcl_session_t *s)
572 : {
573 8 : return (s->session_type == VPPCOM_PROTO_TLS ||
574 8 : s->session_type == VPPCOM_PROTO_QUIC ||
575 2 : s->session_type == VPPCOM_PROTO_DTLS);
576 : }
577 :
578 : static inline u8
579 59660240 : vcl_session_is_ready (vcl_session_t * s)
580 : {
581 59660240 : return (s->session_state == VCL_STATE_READY
582 59660240 : || s->session_state == VCL_STATE_VPP_CLOSING);
583 : }
584 :
585 : static inline u8
586 59660130 : vcl_session_is_open (vcl_session_t * s)
587 : {
588 59660130 : return ((vcl_session_is_ready (s))
589 59660130 : || (s->session_state == VCL_STATE_LISTEN && vcl_session_is_cl (s)));
590 : }
591 :
592 : static inline u8
593 261327 : vcl_session_is_closing (vcl_session_t * s)
594 : {
595 261327 : return (s->session_state == VCL_STATE_VPP_CLOSING
596 261327 : || s->session_state == VCL_STATE_DISCONNECT);
597 : }
598 :
599 : static inline u8
600 5783 : vcl_session_is_closed (vcl_session_t * s)
601 : {
602 5783 : return (!s || (s->session_state == VCL_STATE_CLOSED));
603 : }
604 :
605 : static inline int
606 0 : vcl_session_closing_error (vcl_session_t * s)
607 : {
608 : /* Return 0 on closing sockets */
609 0 : return s->session_state == VCL_STATE_DISCONNECT ? VPPCOM_ECONNRESET : 0;
610 : }
611 :
612 : static inline int
613 0 : vcl_session_closed_error (vcl_session_t * s)
614 : {
615 0 : return s->session_state == VCL_STATE_DISCONNECT
616 0 : ? VPPCOM_ECONNRESET : VPPCOM_ENOTCONN;
617 : }
618 :
619 : static inline void
620 47 : vcl_ip_copy_from_ep (ip46_address_t * ip, vppcom_endpt_t * ep)
621 : {
622 47 : if (ep->is_ip4)
623 43 : clib_memcpy_fast (&ip->ip4, ep->ip, sizeof (ip4_address_t));
624 : else
625 4 : clib_memcpy_fast (&ip->ip6, ep->ip, sizeof (ip6_address_t));
626 47 : }
627 :
628 : static inline void
629 : vcl_ip_copy_to_ep (ip46_address_t * ip, vppcom_endpt_t * ep, u8 is_ip4)
630 : {
631 : ep->is_ip4 = is_ip4;
632 : if (is_ip4)
633 : clib_memcpy_fast (ep->ip, &ip->ip4, sizeof (ip4_address_t));
634 : else
635 : clib_memcpy_fast (ep->ip, &ip->ip6, sizeof (ip6_address_t));
636 : }
637 :
638 : static inline int
639 122 : vcl_proto_is_dgram (uint8_t proto)
640 : {
641 122 : return proto == VPPCOM_PROTO_UDP || proto == VPPCOM_PROTO_DTLS ||
642 : proto == VPPCOM_PROTO_SRTP;
643 : }
644 :
645 : static inline u8
646 1566220 : vcl_session_has_attr (vcl_session_t * s, u8 attr)
647 : {
648 1566220 : return (s->attributes & (1 << attr)) ? 1 : 0;
649 : }
650 :
651 : static inline void
652 62 : vcl_session_set_attr (vcl_session_t * s, u8 attr)
653 : {
654 62 : s->attributes |= 1 << attr;
655 62 : }
656 :
657 : static inline void
658 4 : vcl_session_clear_attr (vcl_session_t * s, u8 attr)
659 : {
660 4 : s->attributes &= ~(1 << attr);
661 4 : }
662 :
663 : static inline session_evt_type_t
664 1380 : vcl_session_dgram_tx_evt (vcl_session_t *s, session_evt_type_t et)
665 : {
666 1380 : return (s->flags & VCL_SESSION_F_CONNECTED) ? et : SESSION_IO_EVT_TX_MAIN;
667 : }
668 :
669 : static inline void
670 7513460 : vcl_session_add_want_deq_ntf (vcl_session_t *s, svm_fifo_deq_ntf_t evt)
671 : {
672 7513460 : svm_fifo_t *txf = vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo;
673 7513460 : if (txf)
674 7513460 : svm_fifo_add_want_deq_ntf (txf, evt);
675 7513460 : }
676 :
677 : static inline void
678 27 : vcl_session_del_want_deq_ntf (vcl_session_t *s, svm_fifo_deq_ntf_t evt)
679 : {
680 27 : svm_fifo_t *txf = vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo;
681 27 : if (txf)
682 27 : svm_fifo_del_want_deq_ntf (txf, evt);
683 27 : }
684 :
685 : /*
686 : * Helpers
687 : */
688 : vcl_mq_evt_conn_t *vcl_mq_evt_conn_alloc (vcl_worker_t * wrk);
689 : u32 vcl_mq_evt_conn_index (vcl_worker_t * wrk, vcl_mq_evt_conn_t * mqc);
690 : vcl_mq_evt_conn_t *vcl_mq_evt_conn_get (vcl_worker_t * wrk, u32 mq_conn_idx);
691 : int vcl_mq_epoll_add_evfd (vcl_worker_t * wrk, svm_msg_q_t * mq);
692 : int vcl_mq_epoll_del_evfd (vcl_worker_t * wrk, u32 mqc_index);
693 :
694 : vcl_worker_t *vcl_worker_alloc_and_init (void);
695 : void vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp);
696 : int vcl_worker_register_with_vpp (void);
697 : svm_msg_q_t *vcl_worker_ctrl_mq (vcl_worker_t * wrk);
698 :
699 : void vcl_flush_mq_events (void);
700 : int vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * session,
701 : vcl_session_handle_t sh, u8 do_disconnect);
702 :
703 : void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index);
704 : u32 vcl_segment_table_lookup (u64 segment_handle);
705 : void vcl_segment_table_del (u64 segment_handle);
706 :
707 : int vcl_session_read_ready (vcl_session_t * session);
708 : int vcl_session_write_ready (vcl_session_t * session);
709 : int vcl_session_alloc_ext_cfg (vcl_session_t *s,
710 : transport_endpt_ext_cfg_type_t type, u32 len);
711 :
712 : static inline vcl_worker_t *
713 150587343 : vcl_worker_get (u32 wrk_index)
714 : {
715 150587343 : return pool_elt_at_index (vcm->workers, wrk_index);
716 : }
717 :
718 : static inline vcl_worker_t *
719 0 : vcl_worker_get_if_valid (u32 wrk_index)
720 : {
721 0 : if (pool_is_free_index (vcm->workers, wrk_index))
722 0 : return 0;
723 0 : return pool_elt_at_index (vcm->workers, wrk_index);
724 : }
725 :
726 : static inline vcl_worker_t *
727 150587285 : vcl_worker_get_current (void)
728 : {
729 150587285 : return vcl_worker_get (vcl_get_worker_index ());
730 : }
731 :
732 : static inline u8
733 26 : vcl_n_workers (void)
734 : {
735 26 : return pool_elts (vcm->workers);
736 : }
737 :
738 : static inline u64
739 204 : vcl_vpp_worker_segment_handle (u32 wrk_index)
740 : {
741 204 : return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1);
742 : }
743 :
744 : void vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
745 : u32 wrk_index);
746 : int vcl_send_worker_rpc (u32 dst_wrk_index, void *data, u32 data_len);
747 :
748 : int vcl_segment_attach (u64 segment_handle, char *name,
749 : ssvm_segment_type_t type, int fd);
750 : void vcl_segment_detach (u64 segment_handle);
751 : void vcl_segment_detach_segments (u32 *seg_indices);
752 : void vcl_send_session_listen (vcl_worker_t *wrk, vcl_session_t *s);
753 : void vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s);
754 :
755 : int vcl_segment_attach_session (uword segment_handle, uword rxf_offset,
756 : uword txf_offset, uword mq_offset,
757 : u32 mq_index, u8 is_ct, vcl_session_t *s);
758 : int vcl_segment_attach_mq (uword segment_handle, uword mq_offset, u32 mq_index,
759 : svm_msg_q_t **mq);
760 : int vcl_segment_discover_mqs (uword segment_handle, int *fds, u32 n_fds);
761 : svm_fifo_chunk_t *vcl_segment_alloc_chunk (uword segment_handle,
762 : u32 slice_index, u32 size,
763 : uword *offset);
764 : int vcl_session_share_fifos (vcl_session_t *s, svm_fifo_t *rxf,
765 : svm_fifo_t *txf);
766 : void vcl_worker_detach_sessions (vcl_worker_t *wrk);
767 :
768 : /*
769 : * VCL Binary API
770 : */
771 : int vcl_bapi_attach (void);
772 : int vcl_bapi_app_worker_add (void);
773 : void vcl_bapi_app_worker_del (vcl_worker_t * wrk);
774 : void vcl_bapi_disconnect_from_vpp (void);
775 : int vcl_bapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds);
776 : int vcl_bapi_add_cert_key_pair (vppcom_cert_key_pair_t *ckpair);
777 : int vcl_bapi_del_cert_key_pair (u32 ckpair_index);
778 : u32 vcl_bapi_max_nsid_len (void);
779 : int vcl_bapi_worker_set (void);
780 :
781 : /*
782 : * VCL Socket API
783 : */
784 : int vcl_sapi_attach (void);
785 : int vcl_sapi_app_worker_add (void);
786 : void vcl_sapi_app_worker_del (vcl_worker_t * wrk);
787 : void vcl_sapi_detach (vcl_worker_t * wrk);
788 : int vcl_sapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds);
789 : int vcl_sapi_add_cert_key_pair (vppcom_cert_key_pair_t *ckpair);
790 : int vcl_sapi_del_cert_key_pair (u32 ckpair_index);
791 :
792 : /*
793 : * Utility functions
794 : */
795 : const char *vcl_session_state_str (vcl_session_state_t state);
796 : u8 *vcl_format_ip4_address (u8 *s, va_list *args);
797 : u8 *vcl_format_ip6_address (u8 *s, va_list *args);
798 : u8 *vcl_format_ip46_address (u8 *s, va_list *args);
799 :
800 : #endif /* SRC_VCL_VCL_PRIVATE_H_ */
801 :
802 : /*
803 : * fd.io coding-style-patch-verification: ON
804 : *
805 : * Local Variables:
806 : * eval: (c-set-style "gnu")
807 : * End:
808 : */
|
