Line data Source code
1 : /*
2 : * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : #ifndef __included_session_h__
16 : #define __included_session_h__
17 :
18 : #include <vppinfra/llist.h>
19 : #include <vnet/session/session_types.h>
20 : #include <vnet/session/session_lookup.h>
21 : #include <vnet/session/session_debug.h>
22 : #include <svm/message_queue.h>
23 : #include <svm/fifo_segment.h>
24 : #include <vlib/dma/dma.h>
25 :
26 : typedef struct session_wrk_stats_
27 : {
28 : u32 errors[SESSION_N_ERRORS];
29 : } session_wrk_stats_t;
30 :
31 : typedef struct session_tx_context_
32 : {
33 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
34 : session_t *s;
35 : transport_proto_vft_t *transport_vft;
36 : transport_connection_t *tc;
37 : transport_send_params_t sp;
38 : u32 max_dequeue;
39 : u32 left_to_snd;
40 : u32 max_len_to_snd;
41 : u16 deq_per_first_buf;
42 : u16 deq_per_buf;
43 : u16 n_segs_per_evt;
44 : u16 n_bufs_needed;
45 : u8 n_bufs_per_seg;
46 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
47 : session_dgram_hdr_t hdr;
48 :
49 : /** Vector of tx buffer free lists */
50 : u32 *tx_buffers;
51 : vlib_buffer_t **transport_pending_bufs;
52 : } session_tx_context_t;
53 :
54 : typedef struct session_evt_elt
55 : {
56 : clib_llist_anchor_t evt_list;
57 : session_event_t evt;
58 : } session_evt_elt_t;
59 :
60 : typedef struct session_ctrl_evt_data_
61 : {
62 : u8 data[SESSION_CTRL_MSG_MAX_SIZE];
63 : } session_evt_ctrl_data_t;
64 :
65 : typedef enum session_wrk_state_
66 : {
67 : SESSION_WRK_POLLING,
68 : SESSION_WRK_INTERRUPT,
69 : SESSION_WRK_IDLE,
70 : } __clib_packed session_wrk_state_t;
71 :
72 : typedef enum session_wrk_flags_
73 : {
74 : SESSION_WRK_F_ADAPTIVE = 1 << 0,
75 : } __clib_packed session_wrk_flag_t;
76 :
77 : #define DMA_TRANS_SIZE 1024
78 : typedef struct
79 : {
80 : u32 *pending_tx_buffers;
81 : u16 *pending_tx_nexts;
82 : } session_dma_transfer;
83 :
84 : typedef struct session_worker_
85 : {
86 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
87 :
88 : /** Worker session pool */
89 : session_t *sessions;
90 :
91 : /** vpp event message queue for worker */
92 : svm_msg_q_t *vpp_event_queue;
93 :
94 : /** vlib_time_now last time around the track */
95 : clib_time_type_t last_vlib_time;
96 :
97 : /** vlib_time_now rounded to us precision and as u64 */
98 : clib_us_time_t last_vlib_us_time;
99 :
100 : /** Convenience pointer to this worker's vlib_main */
101 : vlib_main_t *vm;
102 :
103 : /** Per-proto vector of session handles to enqueue */
104 : session_handle_t **session_to_enqueue;
105 :
106 : /** Timerfd used to periodically signal wrk session queue node */
107 : int timerfd;
108 :
109 : /** Worker flags */
110 : session_wrk_flag_t flags;
111 :
112 : /** Worker state */
113 : session_wrk_state_t state;
114 :
115 : /** Context for session tx */
116 : session_tx_context_t ctx;
117 :
118 : /** Pool of session event list elements */
119 : session_evt_elt_t *event_elts;
120 :
121 : /** Pool of ctrl events data buffers */
122 : session_evt_ctrl_data_t *ctrl_evts_data;
123 :
124 : /** Head of control events list */
125 : clib_llist_index_t ctrl_head;
126 :
127 : /** Head of list of elements */
128 : clib_llist_index_t new_head;
129 :
130 : /** Head of list of pending events */
131 : clib_llist_index_t old_head;
132 :
133 : /** Vector of buffers to be sent */
134 : u32 *pending_tx_buffers;
135 :
136 : /** Vector of nexts for the pending tx buffers */
137 : u16 *pending_tx_nexts;
138 :
139 : /** Clib file for timerfd. Used only if adaptive mode is on */
140 : uword timerfd_file;
141 :
142 : /** List of pending connects for first worker */
143 : clib_llist_index_t pending_connects;
144 :
145 : /** Flag that is set if main thread signaled to handle connects */
146 : u32 n_pending_connects;
147 :
148 : /** List head for first worker evts pending handling on main */
149 : clib_llist_index_t evts_pending_main;
150 :
151 : /** Per-app-worker bitmap of pending notifications */
152 : uword *app_wrks_pending_ntf;
153 :
154 : int config_index;
155 : u8 dma_enabled;
156 : session_dma_transfer *dma_trans;
157 : u16 trans_head;
158 : u16 trans_tail;
159 : u16 trans_size;
160 : u16 batch_num;
161 : vlib_dma_batch_t *batch;
162 :
163 : session_wrk_stats_t stats;
164 :
165 : #if SESSION_DEBUG
166 : /** last event poll time by thread */
167 : clib_time_type_t last_event_poll;
168 : #endif
169 : } session_worker_t;
170 :
171 : typedef int (session_fifo_rx_fn) (session_worker_t * wrk,
172 : vlib_node_runtime_t * node,
173 : session_evt_elt_t * e, int *n_tx_packets);
174 :
175 : extern session_fifo_rx_fn session_tx_fifo_peek_and_snd;
176 : extern session_fifo_rx_fn session_tx_fifo_dequeue_and_snd;
177 : extern session_fifo_rx_fn session_tx_fifo_dequeue_internal;
178 :
179 : u8 session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e);
180 :
181 : typedef void (*session_update_time_fn) (f64 time_now, u8 thread_index);
182 : typedef void (*nat44_original_dst_lookup_fn) (
183 : ip4_address_t *i2o_src, u16 i2o_src_port, ip4_address_t *i2o_dst,
184 : u16 i2o_dst_port, ip_protocol_t proto, u32 *original_dst,
185 : u16 *original_dst_port);
186 :
187 : typedef struct session_main_
188 : {
189 : /** Worker contexts */
190 : session_worker_t *wrk;
191 :
192 : /** Vector of transport update time functions */
193 : session_update_time_fn *update_time_fns;
194 :
195 : /** Event queues memfd segment */
196 : fifo_segment_t wrk_mqs_segment;
197 :
198 : /** Unique segment name counter */
199 : u32 unique_segment_name_counter;
200 :
201 : /** Per transport rx function that can either dequeue or peek */
202 : session_fifo_rx_fn **session_tx_fns;
203 :
204 : /** Per session type output nodes. Could optimize to group nodes by
205 : * fib but lookup would then require session type parsing in session node.
206 : * Trade memory for speed, for now */
207 : u32 *session_type_to_next;
208 :
209 : /** Thread used for allocating active open connections, i.e., half-opens
210 : * for transports like tcp, and sessions that will be migrated for cl
211 : * transports like udp. If vpp has workers, this will be first worker. */
212 : u32 transport_cl_thread;
213 :
214 : transport_proto_t last_transport_proto_type;
215 :
216 : /** Number of workers at pool realloc barrier */
217 : volatile u32 pool_realloc_at_barrier;
218 :
219 : /** Number of workers doing reallocs */
220 : volatile u32 pool_realloc_doing_work;
221 :
222 : /** Lock to synchronize parallel forced reallocs */
223 : clib_spinlock_t pool_realloc_lock;
224 :
225 : /*
226 : * Config parameters
227 : */
228 :
229 : /** Session manager is enabled */
230 : u8 is_enabled;
231 :
232 : /** Session manager initialized (not necessarily enabled) */
233 : u8 is_initialized;
234 :
235 : /** Enable session manager at startup */
236 : u8 session_enable_asap;
237 :
238 : /** Poll session node in main thread */
239 : u8 poll_main;
240 :
241 : /** Allocate private rx mqs for external apps */
242 : u8 use_private_rx_mqs;
243 :
244 : /** Do not enable session queue node adaptive mode */
245 : u8 no_adaptive;
246 :
247 : /** vpp fifo event queue configured length */
248 : u32 configured_wrk_mq_length;
249 :
250 : /** Session ssvm segment configs*/
251 : uword wrk_mqs_segment_size;
252 :
253 : /** Session enable dma*/
254 : u8 dma_enabled;
255 :
256 : /** Session table size parameters */
257 : u32 configured_v4_session_table_buckets;
258 : u32 configured_v4_session_table_memory;
259 : u32 configured_v4_halfopen_table_buckets;
260 : u32 configured_v4_halfopen_table_memory;
261 : u32 configured_v6_session_table_buckets;
262 : u32 configured_v6_session_table_memory;
263 : u32 configured_v6_halfopen_table_buckets;
264 : u32 configured_v6_halfopen_table_memory;
265 :
266 : /** Transport table (preallocation) size parameters */
267 : u32 local_endpoints_table_memory;
268 : u32 local_endpoints_table_buckets;
269 :
270 : /** Transport source port allocation range */
271 : u16 port_allocator_min_src_port;
272 : u16 port_allocator_max_src_port;
273 :
274 : /** Preallocate session config parameter */
275 : u32 preallocated_sessions;
276 :
277 : u16 msg_id_base;
278 :
279 : /** Query nat44-ed session to get original dst ip4 & dst port. */
280 : nat44_original_dst_lookup_fn original_dst_lookup;
281 : } session_main_t;
282 :
283 : extern session_main_t session_main;
284 : extern vlib_node_registration_t session_queue_node;
285 : extern vlib_node_registration_t session_input_node;
286 : extern vlib_node_registration_t session_queue_process_node;
287 : extern vlib_node_registration_t session_queue_pre_input_node;
288 :
289 : typedef enum session_q_process_evt_
290 : {
291 : SESSION_Q_PROCESS_RUN_ON_MAIN = 1,
292 : SESSION_Q_PROCESS_STOP
293 : } session_q_process_evt_t;
294 :
295 : #define TRANSPORT_PROTO_INVALID (session_main.last_transport_proto_type + 1)
296 : #define TRANSPORT_N_PROTOS (session_main.last_transport_proto_type + 1)
297 :
298 : static inline void
299 1306200 : session_evt_add_old (session_worker_t * wrk, session_evt_elt_t * elt)
300 : {
301 1306200 : clib_llist_add_tail (wrk->event_elts, evt_list, elt,
302 : clib_llist_elt (wrk->event_elts, wrk->old_head));
303 1306200 : }
304 :
305 : static inline void
306 67490 : session_evt_add_head_old (session_worker_t * wrk, session_evt_elt_t * elt)
307 : {
308 67490 : clib_llist_add (wrk->event_elts, evt_list, elt,
309 : clib_llist_elt (wrk->event_elts, wrk->old_head));
310 67490 : }
311 :
312 :
313 : static inline u32
314 275 : session_evt_ctrl_data_alloc (session_worker_t * wrk)
315 : {
316 : session_evt_ctrl_data_t *data;
317 275 : pool_get (wrk->ctrl_evts_data, data);
318 275 : return (data - wrk->ctrl_evts_data);
319 : }
320 :
321 : static inline session_evt_elt_t *
322 928 : session_evt_alloc_ctrl (session_worker_t * wrk)
323 : {
324 : session_evt_elt_t *elt;
325 928 : clib_llist_get (wrk->event_elts, elt);
326 928 : clib_llist_add_tail (wrk->event_elts, evt_list, elt,
327 : clib_llist_elt (wrk->event_elts, wrk->ctrl_head));
328 928 : return elt;
329 : }
330 :
331 : static inline void *
332 550 : session_evt_ctrl_data (session_worker_t * wrk, session_evt_elt_t * elt)
333 : {
334 550 : return (void *) (pool_elt_at_index (wrk->ctrl_evts_data,
335 : elt->evt.ctrl_data_index));
336 : }
337 :
338 : static inline void
339 275 : session_evt_ctrl_data_free (session_worker_t * wrk, session_evt_elt_t * elt)
340 : {
341 275 : ASSERT (elt->evt.event_type >= SESSION_CTRL_EVT_RPC);
342 275 : pool_put_index (wrk->ctrl_evts_data, elt->evt.ctrl_data_index);
343 275 : }
344 :
345 : static inline session_evt_elt_t *
346 209018 : session_evt_alloc_new (session_worker_t * wrk)
347 : {
348 : session_evt_elt_t *elt;
349 209018 : clib_llist_get (wrk->event_elts, elt);
350 209018 : clib_llist_add_tail (wrk->event_elts, evt_list, elt,
351 : clib_llist_elt (wrk->event_elts, wrk->new_head));
352 209018 : return elt;
353 : }
354 :
355 : static inline session_evt_elt_t *
356 0 : session_evt_alloc_old (session_worker_t * wrk)
357 : {
358 : session_evt_elt_t *elt;
359 0 : clib_llist_get (wrk->event_elts, elt);
360 0 : clib_llist_add_tail (wrk->event_elts, evt_list, elt,
361 : clib_llist_elt (wrk->event_elts, wrk->old_head));
362 0 : return elt;
363 : }
364 :
365 : int session_wrk_handle_mq (session_worker_t *wrk, svm_msg_q_t *mq);
366 :
367 : session_t *session_alloc (u32 thread_index);
368 : void session_free (session_t * s);
369 : void session_cleanup (session_t *s);
370 : void session_program_cleanup (session_t *s);
371 : void session_cleanup_half_open (session_handle_t ho_handle);
372 : u8 session_is_valid (u32 si, u8 thread_index);
373 :
374 : always_inline session_t *
375 5070767 : session_get (u32 si, u32 thread_index)
376 : {
377 5070767 : ASSERT (session_is_valid (si, thread_index));
378 5070767 : return pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
379 : }
380 :
381 : always_inline session_t *
382 15215 : session_get_if_valid (u64 si, u32 thread_index)
383 : {
384 15215 : if (thread_index >= vec_len (session_main.wrk))
385 18 : return 0;
386 :
387 15197 : if (pool_is_free_index (session_main.wrk[thread_index].sessions, si))
388 16 : return 0;
389 :
390 15181 : ASSERT (session_is_valid (si, thread_index));
391 15181 : return pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
392 : }
393 :
394 : always_inline session_t *
395 1398195 : session_get_from_handle (session_handle_t handle)
396 : {
397 1398195 : session_main_t *smm = &session_main;
398 : u32 session_index, thread_index;
399 1398195 : session_parse_handle (handle, &session_index, &thread_index);
400 1398195 : return pool_elt_at_index (smm->wrk[thread_index].sessions, session_index);
401 : }
402 :
403 : always_inline session_t *
404 14422 : session_get_from_handle_if_valid (session_handle_t handle)
405 : {
406 : u32 session_index, thread_index;
407 14422 : session_parse_handle (handle, &session_index, &thread_index);
408 14422 : return session_get_if_valid (session_index, thread_index);
409 : }
410 :
411 : u64 session_segment_handle (session_t * s);
412 :
413 : /**
414 : * Get session from handle and avoid pool validation if no same thread
415 : *
416 : * Peekers are fine because pool grows with barrier (see @ref session_alloc)
417 : */
418 : always_inline session_t *
419 59042 : session_get_from_handle_safe (u64 handle)
420 : {
421 59042 : u32 thread_index = session_thread_from_handle (handle);
422 59042 : session_worker_t *wrk = &session_main.wrk[thread_index];
423 :
424 59042 : if (thread_index == vlib_get_thread_index ())
425 : {
426 59042 : return pool_elt_at_index (wrk->sessions,
427 : session_index_from_handle (handle));
428 : }
429 : else
430 : {
431 : /* Don't use pool_elt_at index to avoid pool bitmap reallocs */
432 0 : return wrk->sessions + session_index_from_handle (handle);
433 : }
434 : }
435 :
436 : always_inline session_t *
437 0 : session_clone_safe (u32 session_index, u32 thread_index)
438 : {
439 0 : u32 current_thread_index = vlib_get_thread_index (), new_index;
440 : session_t *old_s, *new_s;
441 :
442 0 : new_s = session_alloc (current_thread_index);
443 0 : new_index = new_s->session_index;
444 : /* Session pools are reallocated with barrier (see @ref session_alloc) */
445 0 : old_s = session_main.wrk[thread_index].sessions + session_index;
446 0 : clib_memcpy_fast (new_s, old_s, sizeof (*new_s));
447 0 : new_s->thread_index = current_thread_index;
448 0 : new_s->session_index = new_index;
449 0 : return new_s;
450 : }
451 :
452 : int session_open (session_endpoint_cfg_t *sep, session_handle_t *rsh);
453 : int session_listen (session_t * s, session_endpoint_cfg_t * sep);
454 : int session_stop_listen (session_t * s);
455 : void session_half_close (session_t *s);
456 : void session_close (session_t * s);
457 : void session_reset (session_t * s);
458 : void session_transport_half_close (session_t *s);
459 : void session_transport_close (session_t * s);
460 : void session_transport_reset (session_t * s);
461 : void session_transport_cleanup (session_t * s);
462 : int session_send_io_evt_to_thread (svm_fifo_t * f,
463 : session_evt_type_t evt_type);
464 : int session_enqueue_notify (session_t *s);
465 : int session_dequeue_notify (session_t * s);
466 : int session_enqueue_notify_cl (session_t *s);
467 : int session_send_io_evt_to_thread_custom (void *data, u32 thread_index,
468 : session_evt_type_t evt_type);
469 : void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
470 : void *rpc_args);
471 : void session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp,
472 : void *rpc_args);
473 : void session_add_self_custom_tx_evt (transport_connection_t * tc,
474 : u8 has_prio);
475 : void sesssion_reschedule_tx (transport_connection_t * tc);
476 : transport_connection_t *session_get_transport (session_t * s);
477 : void session_get_endpoint (session_t * s, transport_endpoint_t * tep,
478 : u8 is_lcl);
479 : int session_transport_attribute (session_t *s, u8 is_get,
480 : transport_endpt_attr_t *attr);
481 :
482 : u8 *format_session (u8 * s, va_list * args);
483 : uword unformat_session (unformat_input_t * input, va_list * args);
484 : uword unformat_transport_connection (unformat_input_t * input,
485 : va_list * args);
486 :
487 : /*
488 : * Interface to transport protos
489 : */
490 :
491 : int session_enqueue_stream_connection (transport_connection_t * tc,
492 : vlib_buffer_t * b, u32 offset,
493 : u8 queue_event, u8 is_in_order);
494 : int session_enqueue_dgram_connection (session_t * s,
495 : session_dgram_hdr_t * hdr,
496 : vlib_buffer_t * b, u8 proto,
497 : u8 queue_event);
498 : int session_enqueue_dgram_connection_cl (session_t *s,
499 : session_dgram_hdr_t *hdr,
500 : vlib_buffer_t *b, u8 proto,
501 : u8 queue_event);
502 : int session_stream_connect_notify (transport_connection_t * tc,
503 : session_error_t err);
504 : int session_dgram_connect_notify (transport_connection_t * tc,
505 : u32 old_thread_index,
506 : session_t ** new_session);
507 : int session_stream_accept_notify (transport_connection_t * tc);
508 : void session_transport_closing_notify (transport_connection_t * tc);
509 : void session_transport_delete_notify (transport_connection_t * tc);
510 : void session_half_open_delete_notify (transport_connection_t *tc);
511 : void session_half_open_migrate_notify (transport_connection_t *tc);
512 : int session_half_open_migrated_notify (transport_connection_t *tc);
513 : void session_transport_closed_notify (transport_connection_t * tc);
514 : void session_transport_reset_notify (transport_connection_t * tc);
515 : int session_stream_accept (transport_connection_t * tc, u32 listener_index,
516 : u32 thread_index, u8 notify);
517 : int session_dgram_accept (transport_connection_t * tc, u32 listener_index,
518 : u32 thread_index);
519 :
520 : /**
521 : * Initialize session layer for given transport proto and ip version
522 : *
523 : * Allocates per session type (transport proto + ip version) data structures
524 : * and adds arc from session queue node to session type output node.
525 : *
526 : * @param transport_proto transport proto to be registered
527 : * @param vft virtual function table for transport
528 : * @param is_ip4 flag that indicates if transports uses ipv4
529 : * as underlying network layer
530 : * @param output_node output node for transport
531 : */
532 : void session_register_transport (transport_proto_t transport_proto,
533 : const transport_proto_vft_t * vft, u8 is_ip4,
534 : u32 output_node);
535 : transport_proto_t session_add_transport_proto (void);
536 : void session_register_update_time_fn (session_update_time_fn fn, u8 is_add);
537 : int session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer,
538 : u32 offset, u32 max_bytes);
539 : u32 session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes);
540 :
541 : always_inline void
542 2802 : session_set_state (session_t *s, session_state_t session_state)
543 : {
544 2802 : s->session_state = session_state;
545 : SESSION_EVT (SESSION_EVT_STATE_CHANGE, s);
546 2802 : }
547 :
548 : always_inline u32
549 116667 : transport_max_rx_enqueue (transport_connection_t * tc)
550 : {
551 116667 : session_t *s = session_get (tc->s_index, tc->thread_index);
552 116667 : return svm_fifo_max_enqueue_prod (s->rx_fifo);
553 : }
554 :
555 : always_inline u32
556 2232756 : transport_max_tx_dequeue (transport_connection_t * tc)
557 : {
558 2232756 : session_t *s = session_get (tc->s_index, tc->thread_index);
559 2232756 : return svm_fifo_max_dequeue_cons (s->tx_fifo);
560 : }
561 :
562 : always_inline u32
563 133 : transport_max_rx_dequeue (transport_connection_t * tc)
564 : {
565 133 : session_t *s = session_get (tc->s_index, tc->thread_index);
566 133 : return svm_fifo_max_dequeue (s->rx_fifo);
567 : }
568 :
569 : always_inline u32
570 0 : transport_rx_fifo_size (transport_connection_t * tc)
571 : {
572 0 : session_t *s = session_get (tc->s_index, tc->thread_index);
573 0 : return svm_fifo_size (s->rx_fifo);
574 : }
575 :
576 : always_inline u32
577 267 : transport_tx_fifo_size (transport_connection_t * tc)
578 : {
579 267 : session_t *s = session_get (tc->s_index, tc->thread_index);
580 267 : return svm_fifo_size (s->tx_fifo);
581 : }
582 :
583 : always_inline u8
584 : transport_rx_fifo_has_ooo_data (transport_connection_t * tc)
585 : {
586 : session_t *s = session_get (tc->c_index, tc->thread_index);
587 : return svm_fifo_has_ooo_data (s->rx_fifo);
588 : }
589 :
590 : always_inline u32
591 45 : transport_tx_fifo_has_dgram (transport_connection_t *tc)
592 : {
593 45 : session_t *s = session_get (tc->s_index, tc->thread_index);
594 45 : u32 max_deq = svm_fifo_max_dequeue_cons (s->tx_fifo);
595 : session_dgram_pre_hdr_t phdr;
596 :
597 45 : if (max_deq <= sizeof (session_dgram_hdr_t))
598 43 : return 0;
599 2 : svm_fifo_peek (s->tx_fifo, 0, sizeof (phdr), (u8 *) &phdr);
600 2 : return max_deq >= phdr.data_length + sizeof (session_dgram_hdr_t);
601 : }
602 :
603 : always_inline void
604 0 : transport_rx_fifo_req_deq_ntf (transport_connection_t *tc)
605 : {
606 0 : session_t *s = session_get (tc->s_index, tc->thread_index);
607 0 : svm_fifo_add_want_deq_ntf (s->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
608 0 : }
609 :
610 : always_inline clib_time_type_t
611 : transport_time_now (u32 thread_index)
612 : {
613 : return session_main.wrk[thread_index].last_vlib_time;
614 : }
615 :
616 : always_inline clib_us_time_t
617 140868 : transport_us_time_now (u32 thread_index)
618 : {
619 140868 : return session_main.wrk[thread_index].last_vlib_us_time;
620 : }
621 :
622 : always_inline clib_time_type_t
623 41971 : transport_seconds_per_loop (u32 thread_index)
624 : {
625 41971 : return session_main.wrk[thread_index].vm->seconds_per_loop;
626 : }
627 :
628 : always_inline void
629 : transport_add_tx_event (transport_connection_t * tc)
630 : {
631 : session_t *s = session_get (tc->s_index, tc->thread_index);
632 : if (svm_fifo_has_event (s->tx_fifo))
633 : return;
634 : session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX);
635 : }
636 :
637 : always_inline u32
638 2838 : transport_cl_thread (void)
639 : {
640 2838 : return session_main.transport_cl_thread;
641 : }
642 :
643 : always_inline u32
644 708 : session_vlib_thread_is_cl_thread (void)
645 : {
646 708 : return (vlib_get_thread_index () == transport_cl_thread () ||
647 0 : vlib_thread_is_main_w_barrier ());
648 : }
649 :
650 : /*
651 : * Listen sessions
652 : */
653 :
654 : always_inline u64
655 431 : listen_session_get_handle (session_t * s)
656 : {
657 431 : ASSERT (s->session_state == SESSION_STATE_LISTENING ||
658 : session_get_transport_proto (s) == TRANSPORT_PROTO_QUIC);
659 431 : return session_handle (s);
660 : }
661 :
662 : always_inline session_t *
663 364 : listen_session_get_from_handle (session_handle_t handle)
664 : {
665 364 : return session_get_from_handle (handle);
666 : }
667 :
668 : always_inline void
669 : listen_session_parse_handle (session_handle_t handle, u32 * index,
670 : u32 * thread_index)
671 : {
672 : session_parse_handle (handle, index, thread_index);
673 : }
674 :
675 : always_inline session_t *
676 78 : listen_session_alloc (u8 thread_index, session_type_t type)
677 : {
678 : session_t *s;
679 78 : s = session_alloc (thread_index);
680 78 : s->session_type = type;
681 78 : s->session_state = SESSION_STATE_LISTENING;
682 78 : return s;
683 : }
684 :
685 : always_inline session_t *
686 549 : listen_session_get (u32 ls_index)
687 : {
688 549 : return session_get (ls_index, 0);
689 : }
690 :
691 : always_inline void
692 62 : listen_session_free (session_t * s)
693 : {
694 62 : ASSERT (!s->rx_fifo);
695 62 : session_free (s);
696 62 : }
697 :
698 : always_inline session_t *
699 148 : ho_session_alloc (void)
700 : {
701 : session_t *s;
702 148 : ASSERT (session_vlib_thread_is_cl_thread ());
703 148 : s = session_alloc (transport_cl_thread ());
704 148 : s->session_state = SESSION_STATE_CONNECTING;
705 148 : s->flags |= SESSION_F_HALF_OPEN;
706 148 : return s;
707 : }
708 :
709 : always_inline session_t *
710 452 : ho_session_get (u32 ho_index)
711 : {
712 452 : return session_get (ho_index, transport_cl_thread ());
713 : }
714 :
715 : always_inline void
716 : ho_session_free (session_t *s)
717 : {
718 : ASSERT (!s->rx_fifo && s->thread_index == 0);
719 : session_free (s);
720 : }
721 :
722 : transport_connection_t *listen_session_get_transport (session_t * s);
723 :
724 : /*
725 : * Session layer functions
726 : */
727 :
728 : always_inline session_main_t *
729 73705849 : vnet_get_session_main ()
730 : {
731 73705849 : return &session_main;
732 : }
733 :
734 : always_inline session_worker_t *
735 677725 : session_main_get_worker (u32 thread_index)
736 : {
737 677725 : return vec_elt_at_index (session_main.wrk, thread_index);
738 : }
739 :
740 : static inline session_worker_t *
741 0 : session_main_get_worker_if_valid (u32 thread_index)
742 : {
743 0 : if (thread_index > vec_len (session_main.wrk))
744 0 : return 0;
745 0 : return session_main_get_worker (thread_index);
746 : }
747 :
748 : always_inline svm_msg_q_t *
749 2190 : session_main_get_vpp_event_queue (u32 thread_index)
750 : {
751 2190 : return session_main_get_worker (thread_index)->vpp_event_queue;
752 : }
753 :
754 : always_inline u8
755 97 : session_main_is_enabled ()
756 : {
757 97 : return session_main.is_enabled == 1;
758 : }
759 :
760 : always_inline void
761 0 : session_worker_stat_error_inc (session_worker_t *wrk, int error, int value)
762 : {
763 0 : if ((-(error) >= 0 && -(error) < SESSION_N_ERRORS))
764 0 : wrk->stats.errors[-error] += value;
765 : else
766 : SESSION_DBG ("unknown session counter");
767 0 : }
768 :
769 : always_inline void
770 0 : session_stat_error_inc (int error, int value)
771 : {
772 : session_worker_t *wrk;
773 0 : wrk = session_main_get_worker (vlib_get_thread_index ());
774 0 : session_worker_stat_error_inc (wrk, error, value);
775 0 : }
776 :
777 : #define session_cli_return_if_not_enabled() \
778 : do { \
779 : if (!session_main.is_enabled) \
780 : return clib_error_return (0, "session layer is not enabled"); \
781 : } while (0)
782 :
783 : void session_main_flush_enqueue_events (transport_proto_t transport_proto,
784 : u32 thread_index);
785 : void session_queue_run_on_main_thread (vlib_main_t * vm);
786 :
787 : /**
788 : * Add session node pending buffer with custom node
789 : *
790 : * @param thread_index worker thread expected to send the buffer
791 : * @param bi buffer index
792 : * @param next_node next node edge index for buffer. Edge to next node
793 : * must exist
794 : */
795 : always_inline void
796 33349 : session_add_pending_tx_buffer (u32 thread_index, u32 bi, u32 next_node)
797 : {
798 33349 : session_worker_t *wrk = session_main_get_worker (thread_index);
799 33349 : vec_add1 (wrk->pending_tx_buffers, bi);
800 33349 : vec_add1 (wrk->pending_tx_nexts, next_node);
801 33349 : if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
802 28 : vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
803 33349 : }
804 :
805 : always_inline void
806 73801609 : session_wrk_update_time (session_worker_t *wrk, f64 now)
807 : {
808 73801609 : wrk->last_vlib_time = now;
809 73801609 : wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
810 73801609 : }
811 :
812 : void session_wrk_enable_adaptive_mode (session_worker_t *wrk);
813 : fifo_segment_t *session_main_get_wrk_mqs_segment (void);
814 : void session_node_enable_disable (u8 is_en);
815 : clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en);
816 : void session_wrk_handle_evts_main_rpc (void *);
817 : void session_wrk_program_app_wrk_evts (session_worker_t *wrk,
818 : u32 app_wrk_index);
819 :
820 : session_t *session_alloc_for_connection (transport_connection_t * tc);
821 : session_t *session_alloc_for_half_open (transport_connection_t *tc);
822 : void session_get_original_dst (transport_endpoint_t *i2o_src,
823 : transport_endpoint_t *i2o_dst,
824 : transport_proto_t transport_proto,
825 : u32 *original_dst, u16 *original_dst_port);
826 :
827 : typedef void (pool_safe_realloc_rpc_fn) (void *rpc_args);
828 :
829 : typedef struct
830 : {
831 : u8 ph[STRUCT_OFFSET_OF (pool_header_t, max_elts) + 4];
832 : u32 flag;
833 : } pool_safe_realloc_header_t;
834 :
835 : STATIC_ASSERT_SIZEOF (pool_safe_realloc_header_t, sizeof (pool_header_t));
836 :
837 : #define POOL_REALLOC_SAFE_ELT_THRESH 32
838 :
839 : #define pool_realloc_flag(PH) \
840 : ((pool_safe_realloc_header_t *) pool_header (PH))->flag
841 :
842 : typedef struct pool_realloc_rpc_args_
843 : {
844 : void **pool;
845 : uword elt_size;
846 : uword align;
847 : } pool_realloc_rpc_args_t;
848 :
849 : always_inline void
850 79 : pool_program_safe_realloc_rpc (void *args)
851 : {
852 79 : vlib_main_t *vm = vlib_get_main ();
853 : u32 free_elts, max_elts, n_alloc;
854 : pool_realloc_rpc_args_t *pra;
855 :
856 79 : ASSERT (vlib_get_thread_index () == 0);
857 79 : pra = (pool_realloc_rpc_args_t *) args;
858 :
859 79 : vlib_worker_thread_barrier_sync (vm);
860 :
861 79 : free_elts = _pool_free_elts (*pra->pool, pra->elt_size);
862 79 : if (free_elts < POOL_REALLOC_SAFE_ELT_THRESH)
863 : {
864 79 : max_elts = _vec_max_len (*pra->pool, pra->elt_size);
865 79 : n_alloc = clib_max (2 * max_elts, POOL_REALLOC_SAFE_ELT_THRESH);
866 79 : _pool_alloc (pra->pool, n_alloc, pra->align, 0, pra->elt_size);
867 : }
868 79 : pool_realloc_flag (*pra->pool) = 0;
869 79 : clib_mem_free (args);
870 :
871 79 : vlib_worker_thread_barrier_release (vm);
872 79 : }
873 :
874 : always_inline void
875 79 : pool_program_safe_realloc (void **p, u32 elt_size, u32 align)
876 : {
877 : pool_realloc_rpc_args_t *pra;
878 :
879 : /* Reuse pad as a realloc flag */
880 79 : if (pool_realloc_flag (*p))
881 0 : return;
882 :
883 79 : pra = clib_mem_alloc (sizeof (*pra));
884 79 : pra->pool = p;
885 79 : pra->elt_size = elt_size;
886 79 : pra->align = align;
887 79 : pool_realloc_flag (*p) = 1;
888 :
889 79 : session_send_rpc_evt_to_thread (0 /* thread index */,
890 : pool_program_safe_realloc_rpc, pra);
891 : }
892 :
893 : #define pool_needs_realloc(P) \
894 : ((!P) || \
895 : (vec_len (pool_header (P)->free_indices) < POOL_REALLOC_SAFE_ELT_THRESH && \
896 : pool_free_elts (P) < POOL_REALLOC_SAFE_ELT_THRESH))
897 :
898 : #define pool_get_aligned_safe(P, E, align) \
899 : do \
900 : { \
901 : if (PREDICT_FALSE (pool_needs_realloc (P))) \
902 : { \
903 : if (PREDICT_FALSE (!(P))) \
904 : { \
905 : pool_alloc_aligned (P, POOL_REALLOC_SAFE_ELT_THRESH, align); \
906 : } \
907 : else if (PREDICT_FALSE (!pool_free_elts (P))) \
908 : { \
909 : vlib_workers_sync (); \
910 : pool_alloc_aligned (P, pool_max_len (P), align); \
911 : vlib_workers_continue (); \
912 : ALWAYS_ASSERT (pool_free_elts (P) > 0); \
913 : } \
914 : else \
915 : { \
916 : pool_program_safe_realloc ((void **) &(P), sizeof ((P)[0]), \
917 : _vec_align (P, align)); \
918 : } \
919 : } \
920 : pool_get_aligned (P, E, align); \
921 : } \
922 : while (0)
923 :
924 : #endif /* __included_session_h__ */
925 :
926 : /*
927 : * fd.io coding-style-patch-verification: ON
928 : *
929 : * Local Variables:
930 : * eval: (c-set-style "gnu")
931 : * End:
932 : */
|
