Line data Source code
1 : /*
2 : * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #include <stdio.h>
17 : #include <stdlib.h>
18 : #include <vcl/vppcom.h>
19 : #include <vcl/vcl_debug.h>
20 : #include <vcl/vcl_private.h>
21 : #include <svm/fifo_segment.h>
22 :
23 : __thread uword __vcl_worker_index = ~0;
24 :
25 : static inline int
26 89062 : vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq, u32 n_max_msg)
27 : {
28 89062 : u32 n_msgs = 0, sz, len;
29 :
30 177304 : while ((sz = svm_msg_q_size (mq)))
31 : {
32 88242 : len = vec_len (wrk->mq_msg_vector);
33 88242 : vec_validate (wrk->mq_msg_vector, len + sz - 1);
34 88242 : svm_msg_q_sub_raw_batch (mq, wrk->mq_msg_vector + len, sz);
35 88242 : n_msgs += sz;
36 : }
37 89062 : return n_msgs;
38 : }
39 :
40 :
41 :
42 : static void
43 6 : vcl_msg_add_ext_config (vcl_session_t *s, uword *offset)
44 : {
45 : svm_fifo_chunk_t *c;
46 :
47 6 : c = vcl_segment_alloc_chunk (vcl_vpp_worker_segment_handle (0),
48 6 : 0 /* one slice only */, s->ext_config->len,
49 : offset);
50 6 : if (c)
51 6 : clib_memcpy_fast (c->data, s->ext_config, s->ext_config->len);
52 6 : }
53 :
54 : void
55 40 : vcl_send_session_listen (vcl_worker_t *wrk, vcl_session_t *s)
56 : {
57 40 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
58 : session_listen_msg_t *mp;
59 : svm_msg_q_t *mq;
60 :
61 40 : mq = vcl_worker_ctrl_mq (wrk);
62 40 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_LISTEN);
63 40 : mp = (session_listen_msg_t *) app_evt->evt->data;
64 40 : memset (mp, 0, sizeof (*mp));
65 40 : mp->client_index = wrk->api_client_handle;
66 40 : mp->context = s->session_index;
67 40 : mp->wrk_index = wrk->vpp_wrk_index;
68 40 : mp->is_ip4 = s->transport.is_ip4;
69 40 : clib_memcpy_fast (&mp->ip, &s->transport.lcl_ip, sizeof (mp->ip));
70 40 : mp->port = s->transport.lcl_port;
71 40 : mp->proto = s->session_type;
72 40 : mp->vrf = s->vrf;
73 40 : if (s->flags & VCL_SESSION_F_CONNECTED)
74 0 : mp->flags = TRANSPORT_CFG_F_CONNECTED;
75 40 : if (s->ext_config)
76 3 : vcl_msg_add_ext_config (s, &mp->ext_config);
77 40 : app_send_ctrl_evt_to_vpp (mq, app_evt);
78 40 : if (s->ext_config)
79 : {
80 3 : clib_mem_free (s->ext_config);
81 3 : s->ext_config = 0;
82 : }
83 40 : s->flags |= VCL_SESSION_F_PENDING_LISTEN;
84 40 : }
85 :
86 : static void
87 47 : vcl_send_session_connect (vcl_worker_t * wrk, vcl_session_t * s)
88 : {
89 47 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
90 : session_connect_msg_t *mp;
91 : svm_msg_q_t *mq;
92 :
93 47 : mq = vcl_worker_ctrl_mq (wrk);
94 47 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_CONNECT);
95 47 : mp = (session_connect_msg_t *) app_evt->evt->data;
96 47 : memset (mp, 0, sizeof (*mp));
97 47 : mp->client_index = wrk->api_client_handle;
98 47 : mp->context = s->session_index;
99 47 : mp->dscp = s->dscp;
100 47 : mp->wrk_index = wrk->vpp_wrk_index;
101 47 : mp->is_ip4 = s->transport.is_ip4;
102 47 : mp->parent_handle = s->parent_handle;
103 47 : clib_memcpy_fast (&mp->ip, &s->transport.rmt_ip, sizeof (mp->ip));
104 47 : clib_memcpy_fast (&mp->lcl_ip, &s->transport.lcl_ip, sizeof (mp->lcl_ip));
105 47 : mp->port = s->transport.rmt_port;
106 47 : mp->lcl_port = s->transport.lcl_port;
107 47 : mp->proto = s->session_type;
108 47 : mp->vrf = s->vrf;
109 47 : if (s->flags & VCL_SESSION_F_CONNECTED)
110 47 : mp->flags |= TRANSPORT_CFG_F_CONNECTED;
111 47 : if (s->ext_config)
112 3 : vcl_msg_add_ext_config (s, &mp->ext_config);
113 47 : app_send_ctrl_evt_to_vpp (mq, app_evt);
114 :
115 47 : if (s->ext_config)
116 : {
117 3 : clib_mem_free (s->ext_config);
118 3 : s->ext_config = 0;
119 : }
120 47 : }
121 :
122 : void
123 23 : vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s)
124 : {
125 23 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
126 : session_unlisten_msg_t *mp;
127 : svm_msg_q_t *mq;
128 :
129 23 : mq = vcl_worker_ctrl_mq (wrk);
130 23 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_UNLISTEN);
131 23 : mp = (session_unlisten_msg_t *) app_evt->evt->data;
132 23 : memset (mp, 0, sizeof (*mp));
133 23 : mp->client_index = wrk->api_client_handle;
134 23 : mp->wrk_index = wrk->vpp_wrk_index;
135 23 : mp->handle = s->vpp_handle;
136 23 : mp->context = wrk->wrk_index;
137 23 : app_send_ctrl_evt_to_vpp (mq, app_evt);
138 23 : }
139 :
140 : static void
141 0 : vcl_send_session_shutdown (vcl_worker_t *wrk, vcl_session_t *s)
142 : {
143 0 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
144 : session_shutdown_msg_t *mp;
145 : svm_msg_q_t *mq;
146 :
147 : /* Send to thread that owns the session */
148 0 : mq = s->vpp_evt_q;
149 0 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_SHUTDOWN);
150 0 : mp = (session_shutdown_msg_t *) app_evt->evt->data;
151 0 : memset (mp, 0, sizeof (*mp));
152 0 : mp->client_index = wrk->api_client_handle;
153 0 : mp->handle = s->vpp_handle;
154 0 : app_send_ctrl_evt_to_vpp (mq, app_evt);
155 0 : }
156 :
157 : static void
158 55 : vcl_send_session_disconnect (vcl_worker_t * wrk, vcl_session_t * s)
159 : {
160 55 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
161 : session_disconnect_msg_t *mp;
162 : svm_msg_q_t *mq;
163 :
164 : /* Send to thread that owns the session */
165 55 : mq = s->vpp_evt_q;
166 55 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_DISCONNECT);
167 55 : mp = (session_disconnect_msg_t *) app_evt->evt->data;
168 55 : memset (mp, 0, sizeof (*mp));
169 55 : mp->client_index = wrk->api_client_handle;
170 55 : mp->handle = s->vpp_handle;
171 55 : app_send_ctrl_evt_to_vpp (mq, app_evt);
172 55 : }
173 :
174 : static void
175 24 : vcl_send_app_detach (vcl_worker_t * wrk)
176 : {
177 24 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
178 : session_app_detach_msg_t *mp;
179 : svm_msg_q_t *mq;
180 :
181 24 : mq = vcl_worker_ctrl_mq (wrk);
182 24 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_DETACH);
183 24 : mp = (session_app_detach_msg_t *) app_evt->evt->data;
184 24 : memset (mp, 0, sizeof (*mp));
185 24 : mp->client_index = wrk->api_client_handle;
186 24 : app_send_ctrl_evt_to_vpp (mq, app_evt);
187 24 : }
188 :
189 : static void
190 37 : vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context,
191 : session_handle_t handle, int retval)
192 : {
193 37 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
194 : session_accepted_reply_msg_t *rmp;
195 37 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_ACCEPTED_REPLY);
196 37 : rmp = (session_accepted_reply_msg_t *) app_evt->evt->data;
197 37 : rmp->handle = handle;
198 37 : rmp->context = context;
199 37 : rmp->retval = retval;
200 37 : app_send_ctrl_evt_to_vpp (mq, app_evt);
201 37 : }
202 :
203 : static void
204 5 : vcl_send_session_disconnected_reply (vcl_worker_t * wrk, vcl_session_t * s,
205 : int retval)
206 : {
207 5 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
208 : session_disconnected_reply_msg_t *rmp;
209 5 : app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
210 : SESSION_CTRL_EVT_DISCONNECTED_REPLY);
211 5 : rmp = (session_disconnected_reply_msg_t *) app_evt->evt->data;
212 5 : rmp->handle = s->vpp_handle;
213 5 : rmp->context = wrk->api_client_handle;
214 5 : rmp->retval = retval;
215 5 : app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
216 5 : }
217 :
218 : static void
219 6 : vcl_send_session_reset_reply (vcl_worker_t * wrk, vcl_session_t * s,
220 : int retval)
221 : {
222 6 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
223 : session_reset_reply_msg_t *rmp;
224 6 : app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
225 : SESSION_CTRL_EVT_RESET_REPLY);
226 6 : rmp = (session_reset_reply_msg_t *) app_evt->evt->data;
227 6 : rmp->handle = s->vpp_handle;
228 6 : rmp->context = wrk->api_client_handle;
229 6 : rmp->retval = retval;
230 6 : app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
231 6 : }
232 :
233 : void
234 0 : vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
235 : u32 wrk_index)
236 : {
237 0 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
238 : session_worker_update_msg_t *mp;
239 :
240 0 : app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
241 : SESSION_CTRL_EVT_WORKER_UPDATE);
242 0 : mp = (session_worker_update_msg_t *) app_evt->evt->data;
243 0 : mp->client_index = wrk->api_client_handle;
244 0 : mp->handle = s->vpp_handle;
245 0 : mp->req_wrk_index = wrk->vpp_wrk_index;
246 0 : mp->wrk_index = wrk_index;
247 0 : app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
248 0 : }
249 :
250 : int
251 0 : vcl_send_worker_rpc (u32 dst_wrk_index, void *data, u32 data_len)
252 : {
253 0 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
254 : session_app_wrk_rpc_msg_t *mp;
255 : vcl_worker_t *dst_wrk, *wrk;
256 : svm_msg_q_t *mq;
257 0 : int ret = -1;
258 :
259 0 : if (data_len > sizeof (mp->data))
260 0 : goto done;
261 :
262 0 : clib_spinlock_lock (&vcm->workers_lock);
263 :
264 0 : dst_wrk = vcl_worker_get_if_valid (dst_wrk_index);
265 0 : if (!dst_wrk)
266 0 : goto done;
267 :
268 0 : wrk = vcl_worker_get_current ();
269 0 : mq = vcl_worker_ctrl_mq (wrk);
270 0 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_WRK_RPC);
271 0 : mp = (session_app_wrk_rpc_msg_t *) app_evt->evt->data;
272 0 : mp->client_index = wrk->api_client_handle;
273 0 : mp->wrk_index = dst_wrk->vpp_wrk_index;
274 0 : clib_memcpy (mp->data, data, data_len);
275 0 : app_send_ctrl_evt_to_vpp (mq, app_evt);
276 0 : ret = 0;
277 :
278 0 : done:
279 0 : clib_spinlock_unlock (&vcm->workers_lock);
280 0 : return ret;
281 : }
282 :
283 : int
284 7 : vcl_session_transport_attr (vcl_worker_t *wrk, vcl_session_t *s, u8 is_get,
285 : transport_endpt_attr_t *attr)
286 : {
287 7 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
288 : session_transport_attr_msg_t *mp;
289 : svm_msg_q_t *mq;
290 : f64 timeout;
291 :
292 7 : ASSERT (!wrk->session_attr_op);
293 7 : mq = s->vpp_evt_q;
294 7 : if (PREDICT_FALSE (!mq))
295 : {
296 : /* FIXME: attribute should be stored and sent once session is
297 : * bound/connected to vpp */
298 2 : return 0;
299 : }
300 :
301 5 : wrk->session_attr_op = 1;
302 5 : wrk->session_attr_op_rv = -1;
303 :
304 5 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_TRANSPORT_ATTR);
305 5 : mp = (session_transport_attr_msg_t *) app_evt->evt->data;
306 5 : memset (mp, 0, sizeof (*mp));
307 5 : mp->client_index = wrk->api_client_handle;
308 5 : mp->handle = s->vpp_handle;
309 5 : mp->is_get = is_get;
310 5 : mp->attr = *attr;
311 5 : app_send_ctrl_evt_to_vpp (mq, app_evt);
312 :
313 5 : timeout = clib_time_now (&wrk->clib_time) + 1;
314 :
315 1162 : while (wrk->session_attr_op && clib_time_now (&wrk->clib_time) < timeout)
316 1157 : vcl_flush_mq_events ();
317 :
318 5 : if (!wrk->session_attr_op_rv && is_get)
319 3 : *attr = wrk->session_attr_rv;
320 :
321 5 : wrk->session_attr_op = 0;
322 :
323 5 : return wrk->session_attr_op_rv;
324 : }
325 :
326 : static u32
327 37 : vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp,
328 : u32 ls_index)
329 : {
330 : vcl_session_t *session, *listen_session;
331 : svm_msg_q_t *evt_q;
332 :
333 37 : session = vcl_session_alloc (wrk);
334 :
335 37 : listen_session = vcl_session_get (wrk, ls_index);
336 37 : if (listen_session->vpp_handle != mp->listener_handle)
337 : {
338 0 : VDBG (0, "ERROR: listener handle %lu does not match session %u",
339 : mp->listener_handle, ls_index);
340 0 : goto error;
341 : }
342 :
343 37 : if (vcl_segment_attach_session (
344 : mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
345 : mp->vpp_event_queue_address, mp->mq_index, 0, session))
346 : {
347 0 : VDBG (0, "session %u [0x%llx]: failed to attach fifos",
348 : session->session_index, mp->handle);
349 0 : goto error;
350 : }
351 :
352 37 : session->vpp_handle = mp->handle;
353 37 : session->session_state = VCL_STATE_READY;
354 37 : if (mp->rmt.is_ip4)
355 : {
356 33 : session->original_dst_ip4 = mp->original_dst_ip4;
357 33 : session->original_dst_port = mp->original_dst_port;
358 : }
359 37 : session->transport.rmt_port = mp->rmt.port;
360 37 : session->transport.is_ip4 = mp->rmt.is_ip4;
361 37 : clib_memcpy_fast (&session->transport.rmt_ip, &mp->rmt.ip,
362 : sizeof (ip46_address_t));
363 :
364 37 : vcl_session_table_add_vpp_handle (wrk, mp->handle, session->session_index);
365 37 : session->transport.lcl_port = mp->lcl.port;
366 37 : session->transport.lcl_ip = mp->lcl.ip;
367 37 : session->session_type = listen_session->session_type;
368 37 : session->is_dgram = vcl_proto_is_dgram (session->session_type);
369 37 : if (session->is_dgram)
370 1 : session->flags |= (listen_session->flags & VCL_SESSION_F_CONNECTED);
371 37 : session->listener_index = listen_session->session_index;
372 37 : listen_session->n_accepted_sessions++;
373 :
374 : vcl_evt (VCL_EVT_ACCEPT, session, listen_session, session_index);
375 :
376 37 : vcl_send_session_accepted_reply (session->vpp_evt_q, mp->context,
377 : session->vpp_handle, 0);
378 :
379 37 : return session->session_index;
380 :
381 0 : error:
382 0 : vcl_segment_attach_mq (vcl_vpp_worker_segment_handle (0),
383 : mp->vpp_event_queue_address, mp->mq_index, &evt_q);
384 0 : vcl_send_session_accepted_reply (evt_q, mp->context, mp->handle,
385 : VNET_API_ERROR_INVALID_ARGUMENT);
386 0 : vcl_session_free (wrk, session);
387 0 : return VCL_INVALID_SESSION_INDEX;
388 : }
389 :
390 : static u32
391 47 : vcl_session_connected_handler (vcl_worker_t * wrk,
392 : session_connected_msg_t * mp)
393 : {
394 47 : vcl_session_t *session = 0;
395 : u32 session_index;
396 :
397 47 : session_index = mp->context;
398 47 : session = vcl_session_get (wrk, session_index);
399 47 : if (PREDICT_FALSE (!session))
400 : {
401 0 : VERR ("vpp handle 0x%llx has no session index (%u)!", mp->handle,
402 : session_index);
403 : /* Should not happen but if it does, force vpp session cleanup */
404 0 : vcl_session_t tmp_session = {
405 0 : .vpp_handle = mp->handle,
406 : .vpp_evt_q = 0,
407 : };
408 0 : vcl_segment_attach_session (
409 : mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
410 : mp->vpp_event_queue_address, mp->mq_index, 0, session);
411 0 : if (tmp_session.vpp_evt_q)
412 0 : vcl_send_session_disconnect (wrk, &tmp_session);
413 0 : return VCL_INVALID_SESSION_INDEX;
414 : }
415 :
416 47 : if (mp->retval)
417 : {
418 0 : VDBG (0, "session %u: connect failed! %U", session_index,
419 : format_session_error, mp->retval);
420 0 : session->session_state = VCL_STATE_DETACHED;
421 0 : session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
422 0 : session->vpp_error = mp->retval;
423 0 : return session_index;
424 : }
425 :
426 47 : session->vpp_handle = mp->handle;
427 :
428 : /* Add to lookup table. Even if something fails, session cannot be
429 : * cleaned up prior to notifying vpp and going through the cleanup
430 : * "procedure" see @ref vcl_session_cleanup_handler */
431 47 : vcl_session_table_add_vpp_handle (wrk, mp->handle, session_index);
432 :
433 47 : if (vcl_segment_attach_session (
434 : mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
435 : mp->vpp_event_queue_address, mp->mq_index, 0, session))
436 : {
437 0 : VDBG (0, "session %u [0x%llx]: failed to attach fifos",
438 : session->session_index, session->vpp_handle);
439 0 : session->session_state = VCL_STATE_UPDATED;
440 0 : vcl_send_session_disconnect (wrk, session);
441 0 : return session_index;
442 : }
443 :
444 47 : if (mp->ct_rx_fifo)
445 : {
446 12 : if (vcl_segment_attach_session (mp->ct_segment_handle, mp->ct_rx_fifo,
447 : mp->ct_tx_fifo, (uword) ~0, ~0, 1,
448 : session))
449 : {
450 0 : VDBG (0, "session %u [0x%llx]: failed to attach ct fifos",
451 : session->session_index, session->vpp_handle);
452 0 : session->session_state = VCL_STATE_UPDATED;
453 0 : vcl_send_session_disconnect (wrk, session);
454 0 : return session_index;
455 : }
456 : }
457 :
458 47 : session->transport.is_ip4 = mp->lcl.is_ip4;
459 47 : clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip,
460 : sizeof (session->transport.lcl_ip));
461 47 : session->transport.lcl_port = mp->lcl.port;
462 :
463 : /* Application closed session before connect reply */
464 47 : if (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK)
465 0 : && session->session_state == VCL_STATE_CLOSED)
466 0 : vcl_send_session_disconnect (wrk, session);
467 : else
468 47 : session->session_state = VCL_STATE_READY;
469 :
470 47 : VDBG (0, "session %u [0x%llx] connected local: %U:%u remote %U:%u",
471 : session->session_index, session->vpp_handle, vcl_format_ip46_address,
472 : &session->transport.lcl_ip,
473 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
474 : clib_net_to_host_u16 (session->transport.lcl_port),
475 : vcl_format_ip46_address, &session->transport.rmt_ip,
476 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
477 : clib_net_to_host_u16 (session->transport.rmt_port));
478 :
479 47 : return session_index;
480 : }
481 :
482 : static int
483 0 : vcl_flag_accepted_session (vcl_session_t * session, u64 handle, u32 flags)
484 : {
485 : vcl_session_msg_t *accepted_msg;
486 : int i;
487 :
488 0 : for (i = 0; i < vec_len (session->accept_evts_fifo); i++)
489 : {
490 0 : accepted_msg = &session->accept_evts_fifo[i];
491 0 : if (accepted_msg->accepted_msg.handle == handle)
492 : {
493 0 : accepted_msg->flags |= flags;
494 0 : return 1;
495 : }
496 : }
497 0 : return 0;
498 : }
499 :
500 : static u32
501 6 : vcl_session_reset_handler (vcl_worker_t * wrk,
502 : session_reset_msg_t * reset_msg)
503 : {
504 : vcl_session_t *session;
505 : u32 sid;
506 :
507 6 : sid = vcl_session_index_from_vpp_handle (wrk, reset_msg->handle);
508 6 : session = vcl_session_get (wrk, sid);
509 6 : if (!session)
510 : {
511 0 : VDBG (0, "request to reset unknown handle 0x%llx", reset_msg->handle);
512 0 : return VCL_INVALID_SESSION_INDEX;
513 : }
514 :
515 : /* Caught a reset before actually accepting the session */
516 6 : if (session->session_state == VCL_STATE_LISTEN ||
517 6 : session->session_state == VCL_STATE_LISTEN_NO_MQ)
518 : {
519 0 : if (!vcl_flag_accepted_session (session, reset_msg->handle,
520 : VCL_ACCEPTED_F_RESET))
521 0 : VDBG (0, "session was not accepted!");
522 0 : return VCL_INVALID_SESSION_INDEX;
523 : }
524 :
525 6 : if (session->session_state != VCL_STATE_CLOSED)
526 6 : session->session_state = VCL_STATE_DISCONNECT;
527 :
528 6 : session->flags |= (VCL_SESSION_F_RD_SHUTDOWN | VCL_SESSION_F_WR_SHUTDOWN);
529 6 : VDBG (0, "session %u [0x%llx]: reset", sid, reset_msg->handle);
530 6 : return sid;
531 : }
532 :
533 : static u32
534 40 : vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
535 : {
536 : vcl_session_t *session;
537 40 : u32 sid = mp->context;
538 :
539 40 : session = vcl_session_get (wrk, sid);
540 40 : if (mp->retval)
541 : {
542 0 : VERR ("session %u [0x%llx]: bind failed: %U", sid, mp->handle,
543 : format_session_error, mp->retval);
544 0 : if (session)
545 : {
546 0 : session->session_state = VCL_STATE_DETACHED;
547 0 : session->vpp_handle = mp->handle;
548 0 : return sid;
549 : }
550 : else
551 : {
552 0 : VDBG (0, "session %u [0x%llx]: Invalid session index!", sid,
553 : mp->handle);
554 0 : return VCL_INVALID_SESSION_INDEX;
555 : }
556 : }
557 :
558 40 : session->vpp_handle = mp->handle;
559 40 : session->transport.is_ip4 = mp->lcl_is_ip4;
560 40 : clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip,
561 : sizeof (ip46_address_t));
562 40 : session->transport.lcl_port = mp->lcl_port;
563 40 : vcl_session_table_add_listener (wrk, mp->handle, sid);
564 40 : session->session_state = VCL_STATE_LISTEN;
565 40 : session->flags &= ~VCL_SESSION_F_PENDING_LISTEN;
566 :
567 40 : if (vcl_session_is_cl (session))
568 : {
569 3 : if (vcl_segment_attach_session (mp->segment_handle, mp->rx_fifo,
570 : mp->tx_fifo, mp->vpp_evt_q, mp->mq_index,
571 : 0, session))
572 : {
573 0 : VDBG (0, "session %u [0x%llx]: failed to attach fifos",
574 : session->session_index, session->vpp_handle);
575 0 : session->session_state = VCL_STATE_DETACHED;
576 0 : return VCL_INVALID_SESSION_INDEX;
577 : }
578 : }
579 :
580 40 : VDBG (0, "session %u [0x%llx]: listen succeeded!", sid, mp->handle);
581 40 : return sid;
582 : }
583 :
584 : static void
585 10 : vcl_session_unlisten_reply_handler (vcl_worker_t * wrk, void *data)
586 : {
587 10 : session_unlisten_reply_msg_t *mp = (session_unlisten_reply_msg_t *) data;
588 : vcl_session_t *s;
589 :
590 10 : s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
591 10 : if (!s)
592 : {
593 0 : VDBG (0, "Unlisten reply with wrong handle %llx", mp->handle);
594 0 : return;
595 : }
596 10 : if (s->session_state != VCL_STATE_DISCONNECT)
597 : {
598 : /* Connected udp listener */
599 2 : if (s->session_type == VPPCOM_PROTO_UDP
600 2 : && s->session_state == VCL_STATE_CLOSED)
601 2 : return;
602 :
603 0 : VDBG (0, "Unlisten session in wrong state %llx", mp->handle);
604 0 : return;
605 : }
606 :
607 8 : if (mp->retval)
608 0 : VDBG (0, "ERROR: session %u [0xllx]: unlisten failed: %U",
609 : s->session_index, mp->handle, format_session_error, mp->retval);
610 :
611 8 : if (mp->context != wrk->wrk_index)
612 0 : VDBG (0, "wrong context");
613 :
614 8 : vcl_session_table_del_vpp_handle (wrk, mp->handle);
615 8 : vcl_session_free (wrk, s);
616 : }
617 :
618 : static void
619 0 : vcl_session_migrated_handler (vcl_worker_t * wrk, void *data)
620 : {
621 0 : session_migrated_msg_t *mp = (session_migrated_msg_t *) data;
622 : vcl_session_t *s;
623 : u32 fs_index;
624 :
625 0 : s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
626 0 : if (!s)
627 : {
628 0 : VDBG (0, "Migrated notification with wrong handle %llx", mp->handle);
629 0 : return;
630 : }
631 :
632 : /* Only validate if a value is provided */
633 0 : if (mp->segment_handle != SESSION_INVALID_HANDLE)
634 : {
635 0 : fs_index = vcl_segment_table_lookup (mp->segment_handle);
636 0 : if (fs_index == VCL_INVALID_SEGMENT_INDEX)
637 : {
638 0 : VDBG (0, "segment %lx for session %u is not mounted!",
639 : mp->segment_handle, s->session_index);
640 0 : s->session_state = VCL_STATE_DETACHED;
641 0 : return;
642 : }
643 : }
644 :
645 0 : s->vpp_handle = mp->new_handle;
646 :
647 0 : vcl_segment_attach_mq (vcl_vpp_worker_segment_handle (0), mp->vpp_evt_q,
648 : mp->vpp_thread_index, &s->vpp_evt_q);
649 :
650 0 : vcl_session_table_del_vpp_handle (wrk, mp->handle);
651 0 : vcl_session_table_add_vpp_handle (wrk, mp->new_handle, s->session_index);
652 :
653 : /* Generate new tx event if we have outstanding data */
654 0 : if (svm_fifo_has_event (s->tx_fifo))
655 0 : app_send_io_evt_to_vpp (s->vpp_evt_q,
656 0 : s->tx_fifo->shr->master_session_index,
657 : SESSION_IO_EVT_TX, SVM_Q_WAIT);
658 :
659 0 : VDBG (0, "Migrated 0x%lx to thread %u 0x%lx", mp->handle,
660 : mp->vpp_thread_index, mp->new_handle);
661 : }
662 :
663 : static vcl_session_t *
664 37 : vcl_session_accepted (vcl_worker_t * wrk, session_accepted_msg_t * msg)
665 : {
666 : vcl_session_msg_t *vcl_msg;
667 : vcl_session_t *session;
668 :
669 37 : session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
670 37 : if (PREDICT_FALSE (session != 0))
671 0 : VWRN ("session overlap handle %lu state %u!", msg->handle,
672 : session->session_state);
673 :
674 37 : session = vcl_session_table_lookup_listener (wrk, msg->listener_handle);
675 37 : if (!session)
676 : {
677 0 : VERR ("couldn't find listen session: listener handle %llx",
678 : msg->listener_handle);
679 0 : return 0;
680 : }
681 :
682 37 : clib_fifo_add2 (session->accept_evts_fifo, vcl_msg);
683 37 : vcl_msg->flags = 0;
684 37 : vcl_msg->accepted_msg = *msg;
685 : /* Session handle points to listener until fully accepted by app */
686 37 : vcl_session_table_add_vpp_handle (wrk, msg->handle, session->session_index);
687 :
688 37 : return session;
689 : }
690 :
691 : static vcl_session_t *
692 10 : vcl_session_disconnected_handler (vcl_worker_t * wrk,
693 : session_disconnected_msg_t * msg)
694 : {
695 : vcl_session_t *session;
696 :
697 10 : session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
698 10 : if (!session)
699 : {
700 0 : VWRN ("request to disconnect unknown handle 0x%llx", msg->handle);
701 0 : return 0;
702 : }
703 :
704 : /* Late disconnect notification on a session that has been closed */
705 10 : if (session->session_state == VCL_STATE_CLOSED)
706 5 : return 0;
707 :
708 : /* Caught a disconnect before actually accepting the session */
709 5 : if (session->session_state == VCL_STATE_LISTEN ||
710 5 : session->session_state == VCL_STATE_LISTEN_NO_MQ)
711 : {
712 0 : if (!vcl_flag_accepted_session (session, msg->handle,
713 : VCL_ACCEPTED_F_CLOSED))
714 0 : VDBG (0, "session was not accepted!");
715 0 : return 0;
716 : }
717 :
718 : /* If not already reset change state */
719 5 : if (session->session_state != VCL_STATE_DISCONNECT)
720 5 : session->session_state = VCL_STATE_VPP_CLOSING;
721 :
722 5 : return session;
723 : }
724 :
725 : int
726 0 : vppcom_session_shutdown (uint32_t session_handle, int how)
727 : {
728 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
729 : vcl_session_t *session;
730 : vcl_session_state_t state;
731 : u64 vpp_handle;
732 :
733 0 : session = vcl_session_get_w_handle (wrk, session_handle);
734 0 : if (PREDICT_FALSE (!session))
735 0 : return VPPCOM_EBADFD;
736 :
737 0 : vpp_handle = session->vpp_handle;
738 0 : state = session->session_state;
739 :
740 0 : VDBG (1, "session %u [0x%llx] state 0x%x (%s)", session->session_index,
741 : vpp_handle, state, vcl_session_state_str (state));
742 :
743 0 : if (PREDICT_FALSE (state == VCL_STATE_LISTEN))
744 : {
745 0 : VDBG (0, "ERROR: Cannot shutdown a listen socket!");
746 0 : return VPPCOM_EBADFD;
747 : }
748 :
749 0 : if (how == SHUT_RD || how == SHUT_RDWR)
750 : {
751 0 : session->flags |= VCL_SESSION_F_RD_SHUTDOWN;
752 0 : if (how == SHUT_RD)
753 0 : return VPPCOM_OK;
754 : }
755 0 : session->flags |= VCL_SESSION_F_WR_SHUTDOWN;
756 :
757 0 : if (PREDICT_TRUE (state == VCL_STATE_READY))
758 : {
759 0 : VDBG (1, "session %u [0x%llx]: sending shutdown...",
760 : session->session_index, vpp_handle);
761 :
762 0 : vcl_send_session_shutdown (wrk, session);
763 : }
764 :
765 0 : return VPPCOM_OK;
766 : }
767 :
768 : static int
769 60 : vppcom_session_disconnect (u32 session_handle)
770 : {
771 60 : vcl_worker_t *wrk = vcl_worker_get_current ();
772 : vcl_session_t *session, *listen_session;
773 : vcl_session_state_t state;
774 : u64 vpp_handle;
775 :
776 60 : session = vcl_session_get_w_handle (wrk, session_handle);
777 60 : if (!session)
778 0 : return VPPCOM_EBADFD;
779 :
780 60 : vpp_handle = session->vpp_handle;
781 60 : state = session->session_state;
782 :
783 60 : VDBG (1, "session %u [0x%llx]: disconnecting state (%s)",
784 : session->session_index, vpp_handle, vcl_session_state_str (state));
785 :
786 60 : if (PREDICT_FALSE (state == VCL_STATE_LISTEN))
787 : {
788 0 : VDBG (0, "ERROR: Cannot disconnect a listen socket!");
789 0 : return VPPCOM_EBADFD;
790 : }
791 :
792 60 : if (state == VCL_STATE_VPP_CLOSING)
793 : {
794 5 : vcl_send_session_disconnected_reply (wrk, session, 0);
795 5 : VDBG (1, "session %u [0x%llx]: sending disconnect REPLY...",
796 : session->session_index, vpp_handle);
797 : }
798 : else
799 : {
800 : /* Session doesn't have an event queue yet. Probably a non-blocking
801 : * connect. Wait for the reply */
802 55 : if (PREDICT_FALSE (!session->vpp_evt_q))
803 0 : return VPPCOM_OK;
804 :
805 55 : VDBG (1, "session %u [0x%llx]: sending disconnect",
806 : session->session_index, vpp_handle);
807 55 : vcl_send_session_disconnect (wrk, session);
808 : }
809 :
810 60 : if (session->listener_index != VCL_INVALID_SESSION_INDEX)
811 : {
812 35 : listen_session = vcl_session_get (wrk, session->listener_index);
813 35 : if (listen_session)
814 34 : listen_session->n_accepted_sessions--;
815 : }
816 :
817 60 : return VPPCOM_OK;
818 : }
819 :
820 : static void
821 32 : vcl_session_cleanup_handler (vcl_worker_t * wrk, void *data)
822 : {
823 : session_cleanup_msg_t *msg;
824 : vcl_session_t *session;
825 :
826 32 : msg = (session_cleanup_msg_t *) data;
827 32 : session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
828 32 : if (!session)
829 : {
830 0 : VWRN ("disconnect confirmed for unknown handle 0x%llx", msg->handle);
831 0 : return;
832 : }
833 :
834 32 : if (msg->type == SESSION_CLEANUP_TRANSPORT)
835 : {
836 : /* Transport was cleaned up before we confirmed close. Probably the
837 : * app is still waiting for some data that cannot be delivered.
838 : * Confirm close to make sure everything is cleaned up.
839 : * Move to undetermined state to ensure that the session is not
840 : * removed before both vpp and the app cleanup.
841 : * - If the app closes first, the session is moved to CLOSED state
842 : * and the session cleanup notification from vpp removes the
843 : * session.
844 : * - If vpp cleans up the session first, the session is moved to
845 : * DETACHED state lower and subsequently the close from the app
846 : * frees the session
847 : */
848 16 : if (session->session_state == VCL_STATE_VPP_CLOSING)
849 : {
850 0 : vppcom_session_disconnect (vcl_session_handle (session));
851 0 : session->session_state = VCL_STATE_UPDATED;
852 : }
853 16 : else if (session->session_state == VCL_STATE_DISCONNECT)
854 : {
855 0 : vcl_send_session_reset_reply (wrk, session, 0);
856 0 : session->session_state = VCL_STATE_UPDATED;
857 : }
858 16 : return;
859 : }
860 :
861 : /* VPP will reuse the handle so clean it up now */
862 16 : vcl_session_table_del_vpp_handle (wrk, msg->handle);
863 :
864 : /* App did not close the connection yet so don't free it. */
865 16 : if (session->session_state != VCL_STATE_CLOSED)
866 : {
867 0 : VDBG (0, "session %u: app did not close", session->session_index);
868 0 : session->session_state = VCL_STATE_DETACHED;
869 0 : session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
870 0 : return;
871 : }
872 :
873 : /* Session probably tracked with epoll, disconnect not yet handled and
874 : * 1) both transport and session cleanup completed 2) app closed. Wait
875 : * until message is drained to free the session.
876 : * See @ref vcl_handle_mq_event */
877 16 : if (session->flags & VCL_SESSION_F_PENDING_DISCONNECT)
878 : {
879 0 : session->flags |= VCL_SESSION_F_PENDING_FREE;
880 0 : return;
881 : }
882 :
883 16 : vcl_session_free (wrk, session);
884 : }
885 :
886 : static void
887 0 : vcl_session_req_worker_update_handler (vcl_worker_t * wrk, void *data)
888 : {
889 : session_req_worker_update_msg_t *msg;
890 : vcl_session_t *s;
891 :
892 0 : msg = (session_req_worker_update_msg_t *) data;
893 0 : s = vcl_session_get_w_vpp_handle (wrk, msg->session_handle);
894 0 : if (!s)
895 0 : return;
896 :
897 0 : vec_add1 (wrk->pending_session_wrk_updates, s->session_index);
898 : }
899 :
900 : static void
901 0 : vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data)
902 : {
903 : session_worker_update_reply_msg_t *msg;
904 : vcl_session_t *s;
905 :
906 0 : msg = (session_worker_update_reply_msg_t *) data;
907 0 : s = vcl_session_get_w_vpp_handle (wrk, msg->handle);
908 0 : if (!s)
909 : {
910 0 : VDBG (0, "unknown handle 0x%llx", msg->handle);
911 0 : return;
912 : }
913 :
914 0 : if (s->rx_fifo)
915 : {
916 0 : if (vcl_segment_attach_session (msg->segment_handle, msg->rx_fifo,
917 : msg->tx_fifo, (uword) ~0, ~0, 0, s))
918 : {
919 0 : VDBG (0, "failed to attach fifos for %u", s->session_index);
920 0 : return;
921 : }
922 : }
923 0 : s->session_state = VCL_STATE_UPDATED;
924 :
925 0 : VDBG (0, "session %u[0x%llx] moved to worker %u", s->session_index,
926 : s->vpp_handle, wrk->wrk_index);
927 : }
928 :
929 : static int
930 37 : vcl_api_recv_fd (vcl_worker_t * wrk, int *fds, int n_fds)
931 : {
932 :
933 37 : if (vcm->cfg.vpp_app_socket_api)
934 7 : return vcl_sapi_recv_fds (wrk, fds, n_fds);
935 :
936 30 : return vcl_bapi_recv_fds (wrk, fds, n_fds);
937 : }
938 :
939 : static void
940 37 : vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data)
941 : {
942 37 : ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM;
943 : session_app_add_segment_msg_t *msg;
944 : u64 segment_handle;
945 37 : int fd = -1;
946 :
947 37 : msg = (session_app_add_segment_msg_t *) data;
948 :
949 37 : if (msg->fd_flags)
950 : {
951 37 : vcl_api_recv_fd (wrk, &fd, 1);
952 37 : seg_type = SSVM_SEGMENT_MEMFD;
953 : }
954 :
955 37 : segment_handle = msg->segment_handle;
956 37 : if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
957 : {
958 0 : clib_warning ("invalid segment handle");
959 0 : return;
960 : }
961 :
962 37 : if (vcl_segment_attach (segment_handle, (char *) msg->segment_name,
963 : seg_type, fd))
964 : {
965 0 : VDBG (0, "vcl_segment_attach ('%s') failed", msg->segment_name);
966 0 : return;
967 : }
968 :
969 37 : VDBG (1, "mapped new segment '%s' size %d", msg->segment_name,
970 : msg->segment_size);
971 : }
972 :
973 : static void
974 4 : vcl_session_app_del_segment_handler (vcl_worker_t * wrk, void *data)
975 : {
976 4 : session_app_del_segment_msg_t *msg = (session_app_del_segment_msg_t *) data;
977 4 : vcl_segment_detach (msg->segment_handle);
978 4 : VDBG (1, "Unmapped segment: %lx", msg->segment_handle);
979 4 : }
980 :
981 : static void
982 0 : vcl_worker_rpc_handler (vcl_worker_t * wrk, void *data)
983 : {
984 0 : if (!vcm->wrk_rpc_fn)
985 0 : return;
986 :
987 0 : (vcm->wrk_rpc_fn) (((session_app_wrk_rpc_msg_t *) data)->data);
988 : }
989 :
990 : static void
991 5 : vcl_session_transport_attr_reply_handler (vcl_worker_t *wrk, void *data)
992 : {
993 : session_transport_attr_reply_msg_t *mp;
994 :
995 5 : if (!wrk->session_attr_op)
996 0 : return;
997 :
998 5 : mp = (session_transport_attr_reply_msg_t *) data;
999 :
1000 5 : wrk->session_attr_op_rv = mp->retval;
1001 5 : wrk->session_attr_op = 0;
1002 5 : wrk->session_attr_rv = mp->attr;
1003 : }
1004 :
1005 : static int
1006 239 : vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
1007 : {
1008 : session_disconnected_msg_t *disconnected_msg;
1009 : session_connected_msg_t *connected_msg;
1010 : session_reset_msg_t *reset_msg;
1011 : session_event_t *ecpy;
1012 : vcl_session_t *s;
1013 : u32 sid;
1014 :
1015 239 : switch (e->event_type)
1016 : {
1017 99 : case SESSION_IO_EVT_RX:
1018 : case SESSION_IO_EVT_TX:
1019 99 : s = vcl_session_get (wrk, e->session_index);
1020 99 : if (!s || !vcl_session_is_open (s))
1021 : break;
1022 88 : vec_add1 (wrk->unhandled_evts_vector, *e);
1023 88 : break;
1024 40 : case SESSION_CTRL_EVT_BOUND:
1025 : /* We can only wait for only one listen so not postponed */
1026 40 : vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
1027 40 : break;
1028 0 : case SESSION_CTRL_EVT_ACCEPTED:
1029 0 : s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
1030 0 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1031 : {
1032 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1033 0 : *ecpy = *e;
1034 0 : ecpy->postponed = 1;
1035 0 : ecpy->session_index = s->session_index;
1036 : }
1037 0 : break;
1038 47 : case SESSION_CTRL_EVT_CONNECTED:
1039 47 : connected_msg = (session_connected_msg_t *) e->data;
1040 47 : sid = vcl_session_connected_handler (wrk, connected_msg);
1041 47 : if (!(s = vcl_session_get (wrk, sid)))
1042 0 : break;
1043 47 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1044 : {
1045 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1046 0 : *ecpy = *e;
1047 0 : ecpy->postponed = 1;
1048 0 : ecpy->session_index = s->session_index;
1049 : }
1050 47 : break;
1051 0 : case SESSION_CTRL_EVT_DISCONNECTED:
1052 0 : disconnected_msg = (session_disconnected_msg_t *) e->data;
1053 0 : if (!(s = vcl_session_get_w_vpp_handle (wrk, disconnected_msg->handle)))
1054 0 : break;
1055 0 : if (s->session_state == VCL_STATE_CLOSED)
1056 0 : break;
1057 : /* We do not postpone for blocking sessions or listen sessions because:
1058 : * 1. Blocking sessions are not part of epoll instead they're used in a
1059 : * synchronous manner, such as read/write and etc.
1060 : * 2. Listen sessions that have not yet been accepted can't change to
1061 : * VPP_CLOSING state instead can been marked as ACCEPTED_F_CLOSED.
1062 : */
1063 0 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK) &&
1064 0 : !(s->session_state == VCL_STATE_LISTEN ||
1065 0 : s->session_state == VCL_STATE_LISTEN_NO_MQ))
1066 : {
1067 0 : s->session_state = VCL_STATE_VPP_CLOSING;
1068 0 : s->flags |= VCL_SESSION_F_PENDING_DISCONNECT;
1069 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1070 0 : *ecpy = *e;
1071 0 : ecpy->postponed = 1;
1072 0 : ecpy->session_index = s->session_index;
1073 0 : break;
1074 : }
1075 0 : if (!(s = vcl_session_disconnected_handler (wrk, disconnected_msg)))
1076 0 : break;
1077 0 : VDBG (0, "disconnected session %u [0x%llx]", s->session_index,
1078 : s->vpp_handle);
1079 0 : break;
1080 4 : case SESSION_CTRL_EVT_RESET:
1081 4 : reset_msg = (session_reset_msg_t *) e->data;
1082 4 : if (!(s = vcl_session_get_w_vpp_handle (wrk, reset_msg->handle)))
1083 0 : break;
1084 4 : if (s->session_state == VCL_STATE_CLOSED)
1085 0 : break;
1086 : /* We do not postpone for blocking sessions or listen sessions because:
1087 : * 1. Blocking sessions are not part of epoll instead they're used in a
1088 : * synchronous manner, such as read/write and etc.
1089 : * 2. Listen sessions that have not yet been accepted can't change to
1090 : * DISCONNECT state instead can been marked as ACCEPTED_F_RESET.
1091 : */
1092 4 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK) &&
1093 0 : !(s->session_state == VCL_STATE_LISTEN ||
1094 0 : s->session_state == VCL_STATE_LISTEN_NO_MQ))
1095 : {
1096 0 : s->flags |= VCL_SESSION_F_PENDING_DISCONNECT;
1097 0 : s->session_state = VCL_STATE_DISCONNECT;
1098 0 : s->flags |= (VCL_SESSION_F_RD_SHUTDOWN | VCL_SESSION_F_WR_SHUTDOWN);
1099 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1100 0 : *ecpy = *e;
1101 0 : ecpy->postponed = 1;
1102 0 : ecpy->session_index = s->session_index;
1103 0 : break;
1104 : }
1105 4 : vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
1106 4 : break;
1107 9 : case SESSION_CTRL_EVT_UNLISTEN_REPLY:
1108 9 : vcl_session_unlisten_reply_handler (wrk, e->data);
1109 9 : break;
1110 0 : case SESSION_CTRL_EVT_MIGRATED:
1111 0 : vcl_session_migrated_handler (wrk, e->data);
1112 0 : break;
1113 22 : case SESSION_CTRL_EVT_CLEANUP:
1114 22 : vcl_session_cleanup_handler (wrk, e->data);
1115 22 : break;
1116 0 : case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
1117 0 : vcl_session_req_worker_update_handler (wrk, e->data);
1118 0 : break;
1119 0 : case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
1120 0 : vcl_session_worker_update_reply_handler (wrk, e->data);
1121 0 : break;
1122 11 : case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
1123 11 : vcl_session_app_add_segment_handler (wrk, e->data);
1124 11 : break;
1125 2 : case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
1126 2 : vcl_session_app_del_segment_handler (wrk, e->data);
1127 2 : break;
1128 0 : case SESSION_CTRL_EVT_APP_WRK_RPC:
1129 0 : vcl_worker_rpc_handler (wrk, e->data);
1130 0 : break;
1131 5 : case SESSION_CTRL_EVT_TRANSPORT_ATTR_REPLY:
1132 5 : vcl_session_transport_attr_reply_handler (wrk, e->data);
1133 5 : break;
1134 0 : default:
1135 0 : clib_warning ("unhandled %u", e->event_type);
1136 : }
1137 239 : return VPPCOM_OK;
1138 : }
1139 :
1140 : static int
1141 87 : vppcom_wait_for_session_state_change (u32 session_index,
1142 : vcl_session_state_t state,
1143 : f64 wait_for_time)
1144 : {
1145 87 : vcl_worker_t *wrk = vcl_worker_get_current ();
1146 87 : f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
1147 : vcl_session_t *volatile session;
1148 : svm_msg_q_msg_t msg;
1149 : session_event_t *e;
1150 :
1151 : do
1152 : {
1153 1359 : session = vcl_session_get (wrk, session_index);
1154 1359 : if (PREDICT_FALSE (!session))
1155 : {
1156 0 : return VPPCOM_EBADFD;
1157 : }
1158 1359 : if (session->session_state == state)
1159 : {
1160 87 : return VPPCOM_OK;
1161 : }
1162 1272 : if (session->session_state == VCL_STATE_DETACHED)
1163 : {
1164 0 : return VPPCOM_ECONNREFUSED;
1165 : }
1166 :
1167 1272 : if (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0))
1168 : {
1169 1150 : usleep (100);
1170 1150 : continue;
1171 : }
1172 122 : e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
1173 122 : vcl_handle_mq_event (wrk, e);
1174 122 : svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1175 : }
1176 1272 : while (clib_time_now (&wrk->clib_time) < timeout);
1177 :
1178 0 : VDBG (0, "timeout waiting for state 0x%x (%s)", state,
1179 : vcl_session_state_str (state));
1180 : vcl_evt (VCL_EVT_SESSION_TIMEOUT, session, session_state);
1181 :
1182 0 : return VPPCOM_ETIMEDOUT;
1183 : }
1184 :
1185 : static void
1186 89062 : vcl_handle_pending_wrk_updates (vcl_worker_t * wrk)
1187 : {
1188 : vcl_session_state_t state;
1189 : vcl_session_t *s;
1190 : u32 *sip;
1191 :
1192 89062 : if (PREDICT_TRUE (vec_len (wrk->pending_session_wrk_updates) == 0))
1193 89062 : return;
1194 :
1195 0 : vec_foreach (sip, wrk->pending_session_wrk_updates)
1196 : {
1197 0 : s = vcl_session_get (wrk, *sip);
1198 0 : vcl_send_session_worker_update (wrk, s, wrk->wrk_index);
1199 0 : state = s->session_state;
1200 0 : vppcom_wait_for_session_state_change (s->session_index, VCL_STATE_UPDATED,
1201 : 5);
1202 0 : s->session_state = state;
1203 : }
1204 0 : vec_reset_length (wrk->pending_session_wrk_updates);
1205 : }
1206 :
1207 : void
1208 1254 : vcl_worker_flush_mq_events (vcl_worker_t *wrk)
1209 : {
1210 : svm_msg_q_msg_t *msg;
1211 : session_event_t *e;
1212 : svm_msg_q_t *mq;
1213 : int i;
1214 :
1215 1254 : mq = wrk->app_event_queue;
1216 1254 : vcl_mq_dequeue_batch (wrk, mq, ~0);
1217 :
1218 1371 : for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
1219 : {
1220 117 : msg = vec_elt_at_index (wrk->mq_msg_vector, i);
1221 117 : e = svm_msg_q_msg_data (mq, msg);
1222 117 : vcl_handle_mq_event (wrk, e);
1223 117 : svm_msg_q_free_msg (mq, msg);
1224 : }
1225 1254 : vec_reset_length (wrk->mq_msg_vector);
1226 1254 : vcl_handle_pending_wrk_updates (wrk);
1227 1254 : }
1228 :
1229 : void
1230 1157 : vcl_flush_mq_events (void)
1231 : {
1232 1157 : vcl_worker_flush_mq_events (vcl_worker_get_current ());
1233 1157 : }
1234 :
1235 : static int
1236 21 : vppcom_session_unbind (u32 session_handle)
1237 : {
1238 21 : vcl_worker_t *wrk = vcl_worker_get_current ();
1239 : session_accepted_msg_t *accepted_msg;
1240 21 : vcl_session_t *session = 0;
1241 : vcl_session_msg_t *evt;
1242 :
1243 21 : session = vcl_session_get_w_handle (wrk, session_handle);
1244 21 : if (!session)
1245 0 : return VPPCOM_EBADFD;
1246 :
1247 : /* Flush pending accept events, if any */
1248 21 : while (clib_fifo_elts (session->accept_evts_fifo))
1249 : {
1250 0 : clib_fifo_sub2 (session->accept_evts_fifo, evt);
1251 0 : accepted_msg = &evt->accepted_msg;
1252 0 : vcl_session_table_del_vpp_handle (wrk, accepted_msg->handle);
1253 0 : vcl_send_session_accepted_reply (session->vpp_evt_q,
1254 : accepted_msg->context,
1255 : accepted_msg->handle, -1);
1256 : }
1257 21 : clib_fifo_free (session->accept_evts_fifo);
1258 :
1259 21 : vcl_send_session_unlisten (wrk, session);
1260 :
1261 21 : VDBG (0, "session %u [0x%llx]: sending unbind!", session->session_index,
1262 : session->vpp_handle);
1263 : vcl_evt (VCL_EVT_UNBIND, session);
1264 :
1265 21 : session->vpp_handle = ~0;
1266 21 : session->session_state = VCL_STATE_DISCONNECT;
1267 :
1268 21 : return VPPCOM_OK;
1269 : }
1270 :
1271 : /**
1272 : * Handle app exit
1273 : *
1274 : * Notify vpp of the disconnect and mark the worker as free. If we're the
1275 : * last worker, do a full cleanup otherwise, since we're probably a forked
1276 : * child, avoid syscalls as much as possible. We might've lost privileges.
1277 : */
1278 : void
1279 37 : vppcom_app_exit (void)
1280 : {
1281 37 : if (!pool_elts (vcm->workers))
1282 24 : return;
1283 13 : vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
1284 13 : vcl_set_worker_index (~0);
1285 : vcl_elog_stop (vcm);
1286 : }
1287 :
1288 : static int
1289 37 : vcl_api_attach (void)
1290 : {
1291 37 : if (vcm->cfg.vpp_app_socket_api)
1292 8 : return vcl_sapi_attach ();
1293 :
1294 29 : return vcl_bapi_attach ();
1295 : }
1296 :
1297 : int
1298 0 : vcl_is_first_reattach_to_execute ()
1299 : {
1300 0 : if (vcm->reattach_count == 0)
1301 0 : return 1;
1302 :
1303 0 : return 0;
1304 : }
1305 :
1306 : void
1307 0 : vcl_set_reattach_counter ()
1308 : {
1309 0 : ++vcm->reattach_count;
1310 :
1311 0 : if (vcm->reattach_count == vec_len (vcm->workers))
1312 0 : vcm->reattach_count = 0;
1313 0 : }
1314 :
1315 : /**
1316 : * Reattach vcl to vpp after it has previously been disconnected.
1317 : *
1318 : * The logic should be:
1319 : * - first worker to hit `vcl_api_retry_attach` should attach to vpp,
1320 : * to reproduce the `vcl_api_attach` in `vppcom_app_create`.
1321 : * - the rest of the workers should `reproduce vcl_worker_register_with_vpp`
1322 : * from `vppcom_worker_register` since they were already allocated.
1323 : */
1324 :
1325 : static void
1326 0 : vcl_api_retry_attach (vcl_worker_t *wrk)
1327 : {
1328 : vcl_session_t *s;
1329 :
1330 0 : clib_spinlock_lock (&vcm->workers_lock);
1331 0 : if (vcl_is_first_reattach_to_execute ())
1332 : {
1333 0 : if (vcl_api_attach ())
1334 : {
1335 0 : clib_spinlock_unlock (&vcm->workers_lock);
1336 0 : return;
1337 : }
1338 0 : vcl_set_reattach_counter ();
1339 0 : clib_spinlock_unlock (&vcm->workers_lock);
1340 : }
1341 : else
1342 : {
1343 0 : vcl_set_reattach_counter ();
1344 0 : clib_spinlock_unlock (&vcm->workers_lock);
1345 0 : vcl_worker_register_with_vpp ();
1346 : }
1347 :
1348 : /* Treat listeners as configuration that needs to be re-added to vpp */
1349 0 : pool_foreach (s, wrk->sessions)
1350 : {
1351 0 : if (s->flags & VCL_SESSION_F_IS_VEP)
1352 0 : continue;
1353 0 : if (s->session_state == VCL_STATE_LISTEN_NO_MQ)
1354 0 : vppcom_session_listen (vcl_session_handle (s), 10);
1355 : else
1356 0 : VDBG (0, "internal error: unexpected state %d", s->session_state);
1357 : }
1358 : }
1359 :
1360 : static void
1361 0 : vcl_api_handle_disconnect (vcl_worker_t *wrk)
1362 : {
1363 0 : wrk->api_client_handle = ~0;
1364 0 : vcl_worker_detach_sessions (wrk);
1365 0 : }
1366 :
1367 : static void
1368 24 : vcl_api_detach (vcl_worker_t * wrk)
1369 : {
1370 24 : if (wrk->api_client_handle == ~0)
1371 0 : return;
1372 :
1373 24 : vcl_send_app_detach (wrk);
1374 :
1375 24 : if (vcm->cfg.vpp_app_socket_api)
1376 6 : return vcl_sapi_detach (wrk);
1377 :
1378 18 : return vcl_bapi_disconnect_from_vpp ();
1379 : }
1380 :
1381 : /*
1382 : * VPPCOM Public API functions
1383 : */
1384 : int
1385 37 : vppcom_app_create (const char *app_name)
1386 : {
1387 37 : vppcom_cfg_t *vcl_cfg = &vcm->cfg;
1388 : int rv;
1389 :
1390 37 : if (vcm->is_init)
1391 : {
1392 0 : VDBG (1, "already initialized");
1393 0 : return VPPCOM_EEXIST;
1394 : }
1395 :
1396 37 : vcm->is_init = 1;
1397 37 : vppcom_cfg (&vcm->cfg);
1398 37 : vcl_cfg = &vcm->cfg;
1399 :
1400 37 : vcm->main_cpu = pthread_self ();
1401 37 : vcm->main_pid = getpid ();
1402 37 : vcm->app_name = format (0, "%s", app_name);
1403 37 : fifo_segment_main_init (&vcm->segment_main, (uword) ~0,
1404 : 20 /* timeout in secs */);
1405 37 : pool_alloc (vcm->workers, vcl_cfg->max_workers);
1406 37 : clib_spinlock_init (&vcm->workers_lock);
1407 37 : clib_rwlock_init (&vcm->segment_table_lock);
1408 37 : atexit (vppcom_app_exit);
1409 : vcl_elog_init (vcm);
1410 :
1411 : /* Allocate default worker */
1412 37 : vcl_worker_alloc_and_init ();
1413 :
1414 37 : if ((rv = vcl_api_attach ()))
1415 : {
1416 0 : vppcom_app_destroy ();
1417 0 : return rv;
1418 : }
1419 :
1420 37 : VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
1421 : vcm->workers[0].api_client_handle, vcm->workers[0].api_client_handle);
1422 :
1423 37 : return VPPCOM_OK;
1424 : }
1425 :
1426 : void
1427 24 : vppcom_app_destroy (void)
1428 : {
1429 : vcl_worker_t *wrk, *current_wrk;
1430 : void *heap;
1431 :
1432 24 : if (!pool_elts (vcm->workers))
1433 0 : return;
1434 :
1435 : vcl_evt (VCL_EVT_DETACH, vcm);
1436 :
1437 24 : current_wrk = vcl_worker_get_current ();
1438 :
1439 : /* *INDENT-OFF* */
1440 48 : pool_foreach (wrk, vcm->workers) {
1441 24 : if (current_wrk != wrk)
1442 0 : vcl_worker_cleanup (wrk, 0 /* notify vpp */ );
1443 : }
1444 : /* *INDENT-ON* */
1445 :
1446 24 : vcl_api_detach (current_wrk);
1447 24 : vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ );
1448 24 : vcl_set_worker_index (~0);
1449 :
1450 : vcl_elog_stop (vcm);
1451 :
1452 : /*
1453 : * Free the heap and fix vcm
1454 : */
1455 24 : heap = clib_mem_get_heap ();
1456 24 : munmap (clib_mem_get_heap_base (heap), clib_mem_get_heap_size (heap));
1457 :
1458 24 : vcm = &_vppcom_main;
1459 24 : vcm->is_init = 0;
1460 : }
1461 :
1462 : int
1463 85 : vppcom_session_create (u8 proto, u8 is_nonblocking)
1464 : {
1465 85 : vcl_worker_t *wrk = vcl_worker_get_current ();
1466 : vcl_session_t *session;
1467 :
1468 85 : session = vcl_session_alloc (wrk);
1469 :
1470 85 : session->session_type = proto;
1471 85 : session->session_state = VCL_STATE_CLOSED;
1472 85 : session->vpp_handle = ~0;
1473 85 : session->is_dgram = vcl_proto_is_dgram (proto);
1474 85 : session->vpp_error = SESSION_E_NONE;
1475 :
1476 85 : if (is_nonblocking)
1477 12 : vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
1478 :
1479 : vcl_evt (VCL_EVT_CREATE, session, session_type, session->session_state,
1480 : is_nonblocking, session_index);
1481 :
1482 85 : VDBG (0, "created session %u", session->session_index);
1483 :
1484 85 : return vcl_session_handle (session);
1485 : }
1486 :
1487 : static void
1488 0 : vcl_epoll_lt_add (vcl_worker_t *wrk, vcl_session_t *s)
1489 : {
1490 : vcl_session_t *cur, *prev;
1491 :
1492 0 : ASSERT (s->vep.lt_next == VCL_INVALID_SESSION_INDEX);
1493 :
1494 0 : if (wrk->ep_lt_current == VCL_INVALID_SESSION_INDEX)
1495 : {
1496 0 : wrk->ep_lt_current = s->session_index;
1497 0 : s->vep.lt_next = s->session_index;
1498 0 : s->vep.lt_prev = s->session_index;
1499 0 : return;
1500 : }
1501 :
1502 0 : cur = vcl_session_get (wrk, wrk->ep_lt_current);
1503 0 : prev = vcl_session_get (wrk, cur->vep.lt_prev);
1504 :
1505 0 : prev->vep.lt_next = s->session_index;
1506 0 : s->vep.lt_prev = prev->session_index;
1507 :
1508 0 : s->vep.lt_next = cur->session_index;
1509 0 : cur->vep.lt_prev = s->session_index;
1510 : }
1511 :
1512 : static void
1513 0 : vcl_epoll_lt_del (vcl_worker_t *wrk, vcl_session_t *s)
1514 : {
1515 : vcl_session_t *prev, *next;
1516 :
1517 0 : ASSERT (s->vep.lt_next != VCL_INVALID_SESSION_INDEX);
1518 :
1519 0 : if (s->vep.lt_next == s->session_index)
1520 : {
1521 0 : wrk->ep_lt_current = VCL_INVALID_SESSION_INDEX;
1522 0 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
1523 0 : s->vep.lt_prev = VCL_INVALID_SESSION_INDEX;
1524 0 : return;
1525 : }
1526 :
1527 0 : prev = vcl_session_get (wrk, s->vep.lt_prev);
1528 0 : next = vcl_session_get (wrk, s->vep.lt_next);
1529 :
1530 0 : prev->vep.lt_next = next->session_index;
1531 0 : next->vep.lt_prev = prev->session_index;
1532 :
1533 0 : if (s->session_index == wrk->ep_lt_current)
1534 0 : wrk->ep_lt_current = s->vep.lt_next;
1535 :
1536 0 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
1537 0 : s->vep.lt_prev = VCL_INVALID_SESSION_INDEX;
1538 : }
1539 :
1540 : int
1541 87 : vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * s,
1542 : vcl_session_handle_t sh, u8 do_disconnect)
1543 : {
1544 87 : int rv = VPPCOM_OK;
1545 :
1546 87 : VDBG (1, "session %u [0x%llx] closing", s->session_index, s->vpp_handle);
1547 :
1548 87 : if (s->flags & VCL_SESSION_F_IS_VEP)
1549 : {
1550 0 : u32 next_sh = s->vep.next_sh;
1551 0 : while (next_sh != ~0)
1552 : {
1553 0 : rv = vppcom_epoll_ctl (sh, EPOLL_CTL_DEL, next_sh, 0);
1554 0 : if (PREDICT_FALSE (rv < 0))
1555 0 : VDBG (0, "vpp handle 0x%llx, sh %u: EPOLL_CTL_DEL vep_idx %u"
1556 : " failed! rv %d (%s)", s->vpp_handle, next_sh,
1557 : s->vep.vep_sh, rv, vppcom_retval_str (rv));
1558 0 : next_sh = s->vep.next_sh;
1559 : }
1560 0 : goto free_session;
1561 : }
1562 :
1563 87 : if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
1564 : {
1565 41 : rv = vppcom_epoll_ctl (s->vep.vep_sh, EPOLL_CTL_DEL, sh, 0);
1566 41 : if (rv < 0)
1567 0 : VDBG (0, "session %u [0x%llx]: EPOLL_CTL_DEL vep_idx %u "
1568 : "failed! rv %d (%s)", s->session_index, s->vpp_handle,
1569 : s->vep.vep_sh, rv, vppcom_retval_str (rv));
1570 : }
1571 :
1572 87 : if (!do_disconnect)
1573 : {
1574 0 : VDBG (1, "session %u [0x%llx] disconnect skipped",
1575 : s->session_index, s->vpp_handle);
1576 0 : goto cleanup;
1577 : }
1578 :
1579 87 : if (s->session_state == VCL_STATE_LISTEN)
1580 : {
1581 21 : rv = vppcom_session_unbind (sh);
1582 21 : if (PREDICT_FALSE (rv < 0))
1583 0 : VDBG (0, "session %u [0x%llx]: listener unbind failed! "
1584 : "rv %d (%s)", s->session_index, s->vpp_handle, rv,
1585 : vppcom_retval_str (rv));
1586 21 : return rv;
1587 : }
1588 66 : else if (vcl_session_is_ready (s)
1589 6 : || (vcl_session_is_connectable_listener (wrk, s)))
1590 : {
1591 60 : rv = vppcom_session_disconnect (sh);
1592 60 : if (PREDICT_FALSE (rv < 0))
1593 0 : VDBG (0, "ERROR: session %u [0x%llx]: disconnect failed!"
1594 : " rv %d (%s)", s->session_index, s->vpp_handle,
1595 : rv, vppcom_retval_str (rv));
1596 : }
1597 6 : else if (s->session_state == VCL_STATE_DISCONNECT)
1598 : {
1599 6 : vcl_send_session_reset_reply (wrk, s, 0);
1600 : }
1601 0 : else if (s->session_state == VCL_STATE_DETACHED)
1602 : {
1603 0 : VDBG (0, "vpp freed session %d before close", s->session_index);
1604 :
1605 0 : if (!(s->flags & VCL_SESSION_F_PENDING_DISCONNECT))
1606 0 : goto free_session;
1607 :
1608 : /* Disconnect/reset messages pending but vpp transport and session
1609 : * cleanups already done. Free only after messages drained. */
1610 0 : s->flags |= VCL_SESSION_F_PENDING_FREE;
1611 : }
1612 :
1613 66 : s->session_state = VCL_STATE_CLOSED;
1614 :
1615 : /* Session is removed only after vpp confirms the disconnect */
1616 66 : return rv;
1617 :
1618 0 : cleanup:
1619 0 : vcl_session_table_del_vpp_handle (wrk, s->vpp_handle);
1620 0 : free_session:
1621 0 : vcl_session_free (wrk, s);
1622 : vcl_evt (VCL_EVT_CLOSE, s, rv);
1623 :
1624 0 : return rv;
1625 : }
1626 :
1627 : int
1628 87 : vppcom_session_close (uint32_t session_handle)
1629 : {
1630 87 : vcl_worker_t *wrk = vcl_worker_get_current ();
1631 : vcl_session_t *session;
1632 :
1633 87 : session = vcl_session_get_w_handle (wrk, session_handle);
1634 87 : if (!session)
1635 0 : return VPPCOM_EBADFD;
1636 87 : return vcl_session_cleanup (wrk, session, session_handle,
1637 : 1 /* do_disconnect */ );
1638 : }
1639 :
1640 : int
1641 40 : vppcom_session_bind (uint32_t session_handle, vppcom_endpt_t * ep)
1642 : {
1643 40 : vcl_worker_t *wrk = vcl_worker_get_current ();
1644 40 : vcl_session_t *session = 0;
1645 :
1646 40 : if (!ep || !ep->ip)
1647 0 : return VPPCOM_EINVAL;
1648 :
1649 40 : session = vcl_session_get_w_handle (wrk, session_handle);
1650 40 : if (!session)
1651 0 : return VPPCOM_EBADFD;
1652 :
1653 40 : if (session->flags & VCL_SESSION_F_IS_VEP)
1654 : {
1655 0 : VDBG (0, "ERROR: cannot bind to epoll session %u!",
1656 : session->session_index);
1657 0 : return VPPCOM_EBADFD;
1658 : }
1659 :
1660 40 : session->transport.is_ip4 = ep->is_ip4;
1661 40 : if (ep->is_ip4)
1662 34 : clib_memcpy_fast (&session->transport.lcl_ip.ip4, ep->ip,
1663 : sizeof (ip4_address_t));
1664 : else
1665 6 : clib_memcpy_fast (&session->transport.lcl_ip.ip6, ep->ip,
1666 : sizeof (ip6_address_t));
1667 40 : session->transport.lcl_port = ep->port;
1668 :
1669 40 : VDBG (0,
1670 : "session %u handle %u: binding to local %s address %U port %u, "
1671 : "proto %s",
1672 : session->session_index, session_handle,
1673 : session->transport.is_ip4 ? "IPv4" : "IPv6", vcl_format_ip46_address,
1674 : &session->transport.lcl_ip,
1675 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1676 : clib_net_to_host_u16 (session->transport.lcl_port),
1677 : vppcom_proto_str (session->session_type));
1678 : vcl_evt (VCL_EVT_BIND, session);
1679 :
1680 40 : if (session->session_type == VPPCOM_PROTO_UDP)
1681 3 : vppcom_session_listen (session_handle, 10);
1682 :
1683 40 : return VPPCOM_OK;
1684 : }
1685 :
1686 : int
1687 40 : vppcom_session_listen (uint32_t listen_sh, uint32_t q_len)
1688 : {
1689 40 : vcl_worker_t *wrk = vcl_worker_get_current ();
1690 40 : vcl_session_t *listen_session = 0;
1691 : u64 listen_vpp_handle;
1692 : int rv;
1693 :
1694 40 : listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1695 40 : if (!listen_session || (listen_session->flags & VCL_SESSION_F_IS_VEP))
1696 0 : return VPPCOM_EBADFD;
1697 :
1698 40 : listen_vpp_handle = listen_session->vpp_handle;
1699 40 : if (listen_session->session_state == VCL_STATE_LISTEN)
1700 : {
1701 0 : VDBG (0, "session %u [0x%llx]: already in listen state!",
1702 : listen_sh, listen_vpp_handle);
1703 0 : return VPPCOM_OK;
1704 : }
1705 :
1706 40 : VDBG (0, "session %u: sending vpp listen request...", listen_sh);
1707 :
1708 : /*
1709 : * Send listen request to vpp and wait for reply
1710 : */
1711 40 : vcl_send_session_listen (wrk, listen_session);
1712 40 : rv = vppcom_wait_for_session_state_change (listen_session->session_index,
1713 : VCL_STATE_LISTEN,
1714 40 : vcm->cfg.session_timeout);
1715 :
1716 40 : if (PREDICT_FALSE (rv))
1717 : {
1718 0 : listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1719 0 : VDBG (0, "session %u [0x%llx]: listen failed! returning %d (%s)",
1720 : listen_sh, listen_session->vpp_handle, rv,
1721 : vppcom_retval_str (rv));
1722 0 : return rv;
1723 : }
1724 :
1725 40 : return VPPCOM_OK;
1726 : }
1727 :
1728 : int
1729 2 : vppcom_unformat_proto (uint8_t * proto, char *proto_str)
1730 : {
1731 2 : if (!strcmp (proto_str, "TCP"))
1732 0 : *proto = VPPCOM_PROTO_TCP;
1733 2 : else if (!strcmp (proto_str, "tcp"))
1734 0 : *proto = VPPCOM_PROTO_TCP;
1735 2 : else if (!strcmp (proto_str, "UDP"))
1736 0 : *proto = VPPCOM_PROTO_UDP;
1737 2 : else if (!strcmp (proto_str, "udp"))
1738 0 : *proto = VPPCOM_PROTO_UDP;
1739 2 : else if (!strcmp (proto_str, "TLS"))
1740 0 : *proto = VPPCOM_PROTO_TLS;
1741 2 : else if (!strcmp (proto_str, "tls"))
1742 0 : *proto = VPPCOM_PROTO_TLS;
1743 2 : else if (!strcmp (proto_str, "QUIC"))
1744 0 : *proto = VPPCOM_PROTO_QUIC;
1745 2 : else if (!strcmp (proto_str, "quic"))
1746 0 : *proto = VPPCOM_PROTO_QUIC;
1747 2 : else if (!strcmp (proto_str, "DTLS"))
1748 0 : *proto = VPPCOM_PROTO_DTLS;
1749 2 : else if (!strcmp (proto_str, "dtls"))
1750 2 : *proto = VPPCOM_PROTO_DTLS;
1751 0 : else if (!strcmp (proto_str, "SRTP"))
1752 0 : *proto = VPPCOM_PROTO_SRTP;
1753 0 : else if (!strcmp (proto_str, "srtp"))
1754 0 : *proto = VPPCOM_PROTO_SRTP;
1755 : else
1756 0 : return 1;
1757 2 : return 0;
1758 : }
1759 :
1760 : int
1761 37 : vppcom_session_accept (uint32_t ls_handle, vppcom_endpt_t *ep, uint32_t flags)
1762 : {
1763 37 : u32 client_session_index = ~0, ls_index, accept_flags = 0;
1764 37 : vcl_worker_t *wrk = vcl_worker_get_current ();
1765 : session_accepted_msg_t accepted_msg;
1766 37 : vcl_session_t *ls, *client_session = 0;
1767 : vcl_session_msg_t *evt;
1768 : u8 is_nonblocking;
1769 :
1770 37 : again:
1771 :
1772 37 : ls = vcl_session_get_w_handle (wrk, ls_handle);
1773 37 : if (!ls)
1774 0 : return VPPCOM_EBADFD;
1775 :
1776 37 : if ((ls->session_state != VCL_STATE_LISTEN) &&
1777 0 : (ls->session_state != VCL_STATE_LISTEN_NO_MQ) &&
1778 0 : (!vcl_session_is_connectable_listener (wrk, ls)))
1779 : {
1780 0 : VDBG (0, "ERROR: session [0x%llx]: not in listen state! state (%s)",
1781 : ls->vpp_handle, vcl_session_state_str (ls->session_state));
1782 0 : return VPPCOM_EBADFD;
1783 : }
1784 :
1785 37 : ls_index = ls->session_index;
1786 :
1787 37 : if (clib_fifo_elts (ls->accept_evts_fifo))
1788 : {
1789 37 : clib_fifo_sub2 (ls->accept_evts_fifo, evt);
1790 37 : accept_flags = evt->flags;
1791 37 : accepted_msg = evt->accepted_msg;
1792 37 : goto handle;
1793 : }
1794 :
1795 0 : is_nonblocking = vcl_session_has_attr (ls, VCL_SESS_ATTR_NONBLOCK);
1796 : while (1)
1797 : {
1798 0 : if (svm_msg_q_is_empty (wrk->app_event_queue) && is_nonblocking)
1799 0 : return VPPCOM_EAGAIN;
1800 :
1801 0 : svm_msg_q_wait (wrk->app_event_queue, SVM_MQ_WAIT_EMPTY);
1802 0 : vcl_worker_flush_mq_events (wrk);
1803 0 : goto again;
1804 : }
1805 :
1806 37 : handle:
1807 :
1808 : client_session_index =
1809 37 : vcl_session_accepted_handler (wrk, &accepted_msg, ls_index);
1810 37 : if (client_session_index == VCL_INVALID_SESSION_INDEX)
1811 0 : return VPPCOM_ECONNABORTED;
1812 :
1813 37 : ls = vcl_session_get (wrk, ls_index);
1814 37 : client_session = vcl_session_get (wrk, client_session_index);
1815 :
1816 37 : if (flags & O_NONBLOCK)
1817 0 : vcl_session_set_attr (client_session, VCL_SESS_ATTR_NONBLOCK);
1818 :
1819 37 : VDBG (1,
1820 : "listener %u [0x%llx]: Got a connect request! session %u [0x%llx],"
1821 : " flags %d, is_nonblocking %u",
1822 : ls->session_index, ls->vpp_handle, client_session_index,
1823 : client_session->vpp_handle, flags,
1824 : vcl_session_has_attr (client_session, VCL_SESS_ATTR_NONBLOCK));
1825 :
1826 37 : if (ep)
1827 : {
1828 37 : ep->is_ip4 = client_session->transport.is_ip4;
1829 37 : ep->port = client_session->transport.rmt_port;
1830 37 : if (client_session->transport.is_ip4)
1831 33 : clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip4,
1832 : sizeof (ip4_address_t));
1833 : else
1834 4 : clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip6,
1835 : sizeof (ip6_address_t));
1836 : }
1837 :
1838 37 : VDBG (0,
1839 : "listener %u [0x%llx] accepted %u [0x%llx] peer: %U:%u "
1840 : "local: %U:%u",
1841 : ls_handle, ls->vpp_handle, client_session_index,
1842 : client_session->vpp_handle, vcl_format_ip46_address,
1843 : &client_session->transport.rmt_ip,
1844 : client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1845 : clib_net_to_host_u16 (client_session->transport.rmt_port),
1846 : vcl_format_ip46_address, &client_session->transport.lcl_ip,
1847 : client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1848 : clib_net_to_host_u16 (client_session->transport.lcl_port));
1849 : vcl_evt (VCL_EVT_ACCEPT, client_session, ls, client_session_index);
1850 :
1851 : /*
1852 : * Session might have been closed already
1853 : */
1854 37 : if (accept_flags)
1855 : {
1856 0 : if (accept_flags & VCL_ACCEPTED_F_CLOSED)
1857 0 : client_session->session_state = VCL_STATE_VPP_CLOSING;
1858 0 : else if (accept_flags & VCL_ACCEPTED_F_RESET)
1859 0 : client_session->session_state = VCL_STATE_DISCONNECT;
1860 : }
1861 37 : return vcl_session_handle (client_session);
1862 : }
1863 :
1864 : int
1865 47 : vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep)
1866 : {
1867 47 : vcl_worker_t *wrk = vcl_worker_get_current ();
1868 47 : vcl_session_t *session = 0;
1869 : u32 session_index;
1870 : int rv;
1871 :
1872 47 : session = vcl_session_get_w_handle (wrk, session_handle);
1873 47 : if (!session)
1874 0 : return VPPCOM_EBADFD;
1875 47 : session_index = session->session_index;
1876 :
1877 47 : if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
1878 : {
1879 0 : VWRN ("cannot connect epoll session %u!", session->session_index);
1880 0 : return VPPCOM_EBADFD;
1881 : }
1882 :
1883 47 : if (PREDICT_FALSE (vcl_session_is_ready (session)))
1884 : {
1885 0 : VDBG (0,
1886 : "session %u [0x%llx]: already connected to %U:%d proto %s,"
1887 : " state (%s)",
1888 : session->session_index, session->vpp_handle,
1889 : vcl_format_ip46_address, &session->transport.rmt_ip,
1890 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1891 : clib_net_to_host_u16 (session->transport.rmt_port),
1892 : vppcom_proto_str (session->session_type),
1893 : vcl_session_state_str (session->session_state));
1894 0 : return VPPCOM_OK;
1895 : }
1896 :
1897 : /* Attempt to connect a connectionless listener */
1898 47 : if (PREDICT_FALSE (session->session_state == VCL_STATE_LISTEN))
1899 : {
1900 2 : if (session->session_type != VPPCOM_PROTO_UDP)
1901 0 : return VPPCOM_EINVAL;
1902 2 : vcl_send_session_unlisten (wrk, session);
1903 2 : session->session_state = VCL_STATE_CLOSED;
1904 : }
1905 :
1906 47 : session->transport.is_ip4 = server_ep->is_ip4;
1907 47 : vcl_ip_copy_from_ep (&session->transport.rmt_ip, server_ep);
1908 47 : session->transport.rmt_port = server_ep->port;
1909 47 : session->parent_handle = VCL_INVALID_SESSION_HANDLE;
1910 47 : session->flags |= VCL_SESSION_F_CONNECTED;
1911 :
1912 47 : VDBG (0, "session %u: connecting to peer %U:%d proto %s",
1913 : session->session_index, vcl_format_ip46_address,
1914 : &session->transport.rmt_ip,
1915 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1916 : clib_net_to_host_u16 (session->transport.rmt_port),
1917 : vppcom_proto_str (session->session_type));
1918 :
1919 47 : vcl_send_session_connect (wrk, session);
1920 :
1921 47 : if (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK))
1922 : {
1923 : /* State set to STATE_UPDATED to ensure the session is not assumed
1924 : * to be ready and to also allow the app to close it prior to vpp's
1925 : * connected reply. */
1926 0 : session->session_state = VCL_STATE_UPDATED;
1927 0 : return VPPCOM_EINPROGRESS;
1928 : }
1929 :
1930 : /*
1931 : * Wait for reply from vpp if blocking
1932 : */
1933 47 : rv = vppcom_wait_for_session_state_change (session_index, VCL_STATE_READY,
1934 47 : vcm->cfg.session_timeout);
1935 :
1936 47 : session = vcl_session_get (wrk, session_index);
1937 47 : VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1938 : session->vpp_handle, rv ? "failed" : "succeeded");
1939 :
1940 47 : return rv;
1941 : }
1942 :
1943 : int
1944 0 : vppcom_session_stream_connect (uint32_t session_handle,
1945 : uint32_t parent_session_handle)
1946 : {
1947 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
1948 : vcl_session_t *session, *parent_session;
1949 : u32 session_index, parent_session_index;
1950 : int rv;
1951 :
1952 0 : session = vcl_session_get_w_handle (wrk, session_handle);
1953 0 : if (!session)
1954 0 : return VPPCOM_EBADFD;
1955 0 : parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1956 0 : if (!parent_session)
1957 0 : return VPPCOM_EBADFD;
1958 :
1959 0 : session_index = session->session_index;
1960 0 : parent_session_index = parent_session->session_index;
1961 0 : if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
1962 : {
1963 0 : VDBG (0, "ERROR: cannot connect epoll session %u!",
1964 : session->session_index);
1965 0 : return VPPCOM_EBADFD;
1966 : }
1967 :
1968 0 : if (PREDICT_FALSE (vcl_session_is_ready (session)))
1969 : {
1970 0 : VDBG (0,
1971 : "session handle %u [0x%llx]: session already "
1972 : "connected to session %u [0x%llx] proto %s, state 0x%x (%s)",
1973 : session_handle, session->vpp_handle, parent_session_handle,
1974 : parent_session->vpp_handle,
1975 : vppcom_proto_str (session->session_type), session->session_state,
1976 : vcl_session_state_str (session->session_state));
1977 0 : return VPPCOM_OK;
1978 : }
1979 :
1980 : /* Connect to quic session specifics */
1981 0 : session->transport.is_ip4 = parent_session->transport.is_ip4;
1982 0 : session->transport.rmt_ip.ip4.as_u32 = (uint32_t) 1;
1983 0 : session->transport.rmt_port = 0;
1984 0 : session->parent_handle = parent_session->vpp_handle;
1985 :
1986 0 : VDBG (0, "session handle %u: connecting to session %u [0x%llx]",
1987 : session_handle, parent_session_handle, parent_session->vpp_handle);
1988 :
1989 : /*
1990 : * Send connect request and wait for reply from vpp
1991 : */
1992 0 : vcl_send_session_connect (wrk, session);
1993 0 : rv = vppcom_wait_for_session_state_change (session_index, VCL_STATE_READY,
1994 0 : vcm->cfg.session_timeout);
1995 :
1996 0 : session->listener_index = parent_session_index;
1997 0 : parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1998 0 : if (parent_session)
1999 0 : parent_session->n_accepted_sessions++;
2000 :
2001 0 : session = vcl_session_get (wrk, session_index);
2002 0 : VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
2003 : session->vpp_handle, rv ? "failed" : "succeeded");
2004 :
2005 0 : return rv;
2006 : }
2007 :
2008 : static inline int
2009 737655 : vppcom_session_read_internal (uint32_t session_handle, void *buf, int n,
2010 : u8 peek)
2011 : {
2012 737655 : vcl_worker_t *wrk = vcl_worker_get_current ();
2013 737655 : int rv, n_read = 0, is_nonblocking;
2014 737655 : vcl_session_t *s = 0;
2015 : svm_fifo_t *rx_fifo;
2016 : session_event_t *e;
2017 : svm_msg_q_t *mq;
2018 : u8 is_ct;
2019 :
2020 737655 : if (PREDICT_FALSE (!buf))
2021 0 : return VPPCOM_EFAULT;
2022 :
2023 737655 : s = vcl_session_get_w_handle (wrk, session_handle);
2024 737655 : if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2025 0 : return VPPCOM_EBADFD;
2026 :
2027 737655 : if (PREDICT_FALSE (!vcl_session_is_open (s)))
2028 : {
2029 0 : VDBG (0, "session %u[0x%llx] is not open! state 0x%x (%s)",
2030 : s->session_index, s->vpp_handle, s->session_state,
2031 : vcl_session_state_str (s->session_state));
2032 0 : return vcl_session_closed_error (s);
2033 : }
2034 :
2035 737655 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_RD_SHUTDOWN))
2036 : {
2037 : /* Vpp would ack the incoming data and enqueue it for reading.
2038 : * So even if SHUT_RD is set, we can still read() the data if
2039 : * the session is ready.
2040 : */
2041 0 : if (!vcl_session_read_ready (s))
2042 : {
2043 0 : return 0;
2044 : }
2045 : }
2046 :
2047 737655 : is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2048 737655 : is_ct = vcl_session_is_ct (s);
2049 737655 : mq = wrk->app_event_queue;
2050 737655 : rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
2051 737655 : s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
2052 :
2053 737655 : if (svm_fifo_is_empty_cons (rx_fifo))
2054 : {
2055 267937 : if (is_ct)
2056 14 : svm_fifo_unset_event (s->rx_fifo);
2057 267937 : svm_fifo_unset_event (rx_fifo);
2058 267937 : if (is_nonblocking)
2059 : {
2060 267865 : if (vcl_session_is_closing (s))
2061 0 : return vcl_session_closing_error (s);
2062 267865 : return VPPCOM_EWOULDBLOCK;
2063 : }
2064 169 : while (svm_fifo_is_empty_cons (rx_fifo))
2065 : {
2066 97 : if (vcl_session_is_closing (s))
2067 0 : return vcl_session_closing_error (s);
2068 :
2069 97 : if (is_ct)
2070 21 : svm_fifo_unset_event (s->rx_fifo);
2071 97 : svm_fifo_unset_event (rx_fifo);
2072 :
2073 97 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2074 97 : vcl_worker_flush_mq_events (wrk);
2075 : }
2076 : }
2077 :
2078 469790 : read_again:
2079 :
2080 808177 : if (s->is_dgram)
2081 1380 : rv = app_recv_dgram_raw (rx_fifo, buf, n, &s->transport, 0, peek);
2082 : else
2083 806797 : rv = app_recv_stream_raw (rx_fifo, buf, n, 0, peek);
2084 :
2085 808177 : ASSERT (rv >= 0);
2086 :
2087 808177 : if (peek)
2088 0 : return rv;
2089 :
2090 808177 : n_read += rv;
2091 :
2092 808177 : if (svm_fifo_is_empty_cons (rx_fifo))
2093 : {
2094 370309 : if (is_ct)
2095 790 : svm_fifo_unset_event (s->rx_fifo);
2096 370309 : svm_fifo_unset_event (rx_fifo);
2097 370309 : if (!svm_fifo_is_empty_cons (rx_fifo)
2098 13136 : && svm_fifo_set_event (rx_fifo) && is_nonblocking)
2099 : {
2100 13120 : vec_add2 (wrk->unhandled_evts_vector, e, 1);
2101 13120 : e->event_type = SESSION_IO_EVT_RX;
2102 13120 : e->session_index = s->session_index;
2103 : }
2104 : }
2105 437868 : else if (PREDICT_FALSE (rv < n && !s->is_dgram))
2106 : {
2107 : /* More data enqueued while reading. Try to drain it
2108 : * or fill the buffer. Avoid doing that for dgrams */
2109 338387 : buf += rv;
2110 338387 : n -= rv;
2111 338387 : goto read_again;
2112 : }
2113 :
2114 469790 : if (PREDICT_FALSE (svm_fifo_needs_deq_ntf (rx_fifo, n_read)))
2115 : {
2116 4987 : svm_fifo_clear_deq_ntf (rx_fifo);
2117 4987 : app_send_io_evt_to_vpp (s->vpp_evt_q,
2118 4987 : s->rx_fifo->shr->master_session_index,
2119 : SESSION_IO_EVT_RX, SVM_Q_WAIT);
2120 : }
2121 :
2122 469790 : VDBG (2, "session %u[0x%llx]: read %d bytes from (%p)", s->session_index,
2123 : s->vpp_handle, n_read, rx_fifo);
2124 :
2125 469790 : return n_read;
2126 : }
2127 :
2128 : int
2129 737655 : vppcom_session_read (uint32_t session_handle, void *buf, size_t n)
2130 : {
2131 737655 : return (vppcom_session_read_internal (session_handle, buf, n, 0));
2132 : }
2133 :
2134 : static int
2135 0 : vppcom_session_peek (uint32_t session_handle, void *buf, int n)
2136 : {
2137 0 : return (vppcom_session_read_internal (session_handle, buf, n, 1));
2138 : }
2139 :
2140 : int
2141 0 : vppcom_session_read_segments (uint32_t session_handle,
2142 : vppcom_data_segment_t * ds, uint32_t n_segments,
2143 : uint32_t max_bytes)
2144 : {
2145 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
2146 0 : int n_read = 0, is_nonblocking;
2147 0 : vcl_session_t *s = 0;
2148 : svm_fifo_t *rx_fifo;
2149 : svm_msg_q_t *mq;
2150 : u8 is_ct;
2151 :
2152 0 : s = vcl_session_get_w_handle (wrk, session_handle);
2153 0 : if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2154 0 : return VPPCOM_EBADFD;
2155 :
2156 0 : if (PREDICT_FALSE (!vcl_session_is_open (s)))
2157 0 : return vcl_session_closed_error (s);
2158 :
2159 0 : is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2160 0 : is_ct = vcl_session_is_ct (s);
2161 0 : mq = wrk->app_event_queue;
2162 0 : rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
2163 0 : s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
2164 :
2165 0 : if (svm_fifo_is_empty_cons (rx_fifo))
2166 : {
2167 0 : if (is_ct)
2168 0 : svm_fifo_unset_event (s->rx_fifo);
2169 0 : svm_fifo_unset_event (rx_fifo);
2170 0 : if (is_nonblocking)
2171 : {
2172 0 : if (vcl_session_is_closing (s))
2173 0 : return vcl_session_closing_error (s);
2174 0 : return VPPCOM_EWOULDBLOCK;
2175 : }
2176 0 : while (svm_fifo_is_empty_cons (rx_fifo))
2177 : {
2178 0 : if (vcl_session_is_closing (s))
2179 0 : return vcl_session_closing_error (s);
2180 :
2181 0 : if (is_ct)
2182 0 : svm_fifo_unset_event (s->rx_fifo);
2183 0 : svm_fifo_unset_event (rx_fifo);
2184 :
2185 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2186 0 : vcl_worker_flush_mq_events (wrk);
2187 : }
2188 : }
2189 :
2190 0 : n_read = svm_fifo_segments (rx_fifo, s->rx_bytes_pending,
2191 : (svm_fifo_seg_t *) ds, &n_segments, max_bytes);
2192 0 : if (n_read < 0)
2193 0 : return VPPCOM_EAGAIN;
2194 :
2195 0 : if (svm_fifo_max_dequeue_cons (rx_fifo) == n_read)
2196 : {
2197 0 : if (is_ct)
2198 0 : svm_fifo_unset_event (s->rx_fifo);
2199 0 : svm_fifo_unset_event (rx_fifo);
2200 0 : if (svm_fifo_max_dequeue_cons (rx_fifo) != n_read
2201 0 : && svm_fifo_set_event (rx_fifo)
2202 0 : && vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
2203 : {
2204 : session_event_t *e;
2205 0 : vec_add2 (wrk->unhandled_evts_vector, e, 1);
2206 0 : e->event_type = SESSION_IO_EVT_RX;
2207 0 : e->session_index = s->session_index;
2208 : }
2209 : }
2210 :
2211 0 : s->rx_bytes_pending += n_read;
2212 0 : return n_read;
2213 : }
2214 :
2215 : void
2216 0 : vppcom_session_free_segments (uint32_t session_handle, uint32_t n_bytes)
2217 : {
2218 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
2219 : vcl_session_t *s;
2220 : u8 is_ct;
2221 :
2222 0 : s = vcl_session_get_w_handle (wrk, session_handle);
2223 0 : if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2224 0 : return;
2225 :
2226 0 : is_ct = vcl_session_is_ct (s);
2227 0 : svm_fifo_dequeue_drop (is_ct ? s->ct_rx_fifo : s->rx_fifo, n_bytes);
2228 :
2229 0 : ASSERT (s->rx_bytes_pending >= n_bytes);
2230 0 : s->rx_bytes_pending -= n_bytes;
2231 : }
2232 :
2233 : always_inline u8
2234 1128250 : vcl_fifo_is_writeable (svm_fifo_t * f, u32 len, u8 is_dgram)
2235 : {
2236 1128250 : u32 max_enq = svm_fifo_max_enqueue_prod (f);
2237 1128250 : if (is_dgram)
2238 1380 : return max_enq >= (sizeof (session_dgram_hdr_t) + len);
2239 : else
2240 1126870 : return max_enq > 0;
2241 : }
2242 :
2243 : always_inline int
2244 1128250 : vppcom_session_write_inline (vcl_worker_t *wrk, vcl_session_t *s, void *buf,
2245 : size_t n, u8 is_flush, u8 is_dgram)
2246 : {
2247 : int n_write, is_nonblocking;
2248 : session_evt_type_t et;
2249 : svm_fifo_t *tx_fifo;
2250 : svm_msg_q_t *mq;
2251 : u8 is_ct;
2252 :
2253 : /* Accept zero length writes but just return */
2254 1128250 : if (PREDICT_FALSE (!n))
2255 0 : return VPPCOM_OK;
2256 :
2257 1128250 : if (PREDICT_FALSE (!buf))
2258 0 : return VPPCOM_EFAULT;
2259 :
2260 1128250 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2261 : {
2262 0 : VDBG (0, "ERROR: session %u [0x%llx]: cannot write to an epoll"
2263 : " session!", s->session_index, s->vpp_handle);
2264 0 : return VPPCOM_EBADFD;
2265 : }
2266 :
2267 1128250 : if (PREDICT_FALSE (!vcl_session_is_open (s)))
2268 : {
2269 0 : VDBG (1, "session %u [0x%llx]: is not open! state 0x%x (%s)",
2270 : s->session_index, s->vpp_handle, s->session_state,
2271 : vcl_session_state_str (s->session_state));
2272 0 : return vcl_session_closed_error (s);;
2273 : }
2274 :
2275 1128250 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_WR_SHUTDOWN))
2276 : {
2277 0 : VDBG (1, "session %u [0x%llx]: is shutdown! state 0x%x (%s)",
2278 : s->session_index, s->vpp_handle, s->session_state,
2279 : vcl_session_state_str (s->session_state));
2280 0 : return VPPCOM_EPIPE;
2281 : }
2282 :
2283 1128250 : is_ct = vcl_session_is_ct (s);
2284 1128250 : tx_fifo = is_ct ? s->ct_tx_fifo : s->tx_fifo;
2285 1128250 : is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2286 :
2287 1128250 : mq = wrk->app_event_queue;
2288 1128250 : if (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2289 : {
2290 951232 : if (is_nonblocking)
2291 : {
2292 951232 : return VPPCOM_EWOULDBLOCK;
2293 : }
2294 0 : while (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2295 : {
2296 0 : svm_fifo_add_want_deq_ntf (tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
2297 0 : if (vcl_session_is_closing (s))
2298 0 : return vcl_session_closing_error (s);
2299 :
2300 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2301 0 : vcl_worker_flush_mq_events (wrk);
2302 : }
2303 : }
2304 :
2305 177022 : et = SESSION_IO_EVT_TX;
2306 177022 : if (is_flush && !is_ct)
2307 28404 : et = SESSION_IO_EVT_TX_FLUSH;
2308 :
2309 177022 : if (is_dgram)
2310 : {
2311 1380 : et = vcl_session_dgram_tx_evt (s, et);
2312 : n_write =
2313 1380 : app_send_dgram_raw_gso (tx_fifo, &s->transport, s->vpp_evt_q, buf, n,
2314 1380 : s->gso_size, et, 0 /* do_evt */, SVM_Q_WAIT);
2315 : }
2316 : else
2317 : {
2318 175642 : n_write = app_send_stream_raw (tx_fifo, s->vpp_evt_q, buf, n, et,
2319 : 0 /* do_evt */, SVM_Q_WAIT);
2320 : }
2321 :
2322 177022 : if (svm_fifo_set_event (s->tx_fifo))
2323 138551 : app_send_io_evt_to_vpp (
2324 138551 : s->vpp_evt_q, s->tx_fifo->shr->master_session_index, et, SVM_Q_WAIT);
2325 :
2326 : /* The underlying fifo segment can run out of memory */
2327 177022 : if (PREDICT_FALSE (n_write < 0))
2328 0 : return VPPCOM_EAGAIN;
2329 :
2330 177022 : VDBG (2, "session %u [0x%llx]: wrote %d bytes", s->session_index,
2331 : s->vpp_handle, n_write);
2332 :
2333 177022 : return n_write;
2334 : }
2335 :
2336 : int
2337 18482 : vppcom_session_write (uint32_t session_handle, void *buf, size_t n)
2338 : {
2339 18482 : vcl_worker_t *wrk = vcl_worker_get_current ();
2340 : vcl_session_t *s;
2341 :
2342 18482 : s = vcl_session_get_w_handle (wrk, session_handle);
2343 18482 : if (PREDICT_FALSE (!s))
2344 0 : return VPPCOM_EBADFD;
2345 :
2346 18482 : return vppcom_session_write_inline (wrk, s, buf, n, 0 /* is_flush */,
2347 18482 : s->is_dgram ? 1 : 0);
2348 : }
2349 :
2350 : int
2351 1109770 : vppcom_session_write_msg (uint32_t session_handle, void *buf, size_t n)
2352 : {
2353 1109770 : vcl_worker_t *wrk = vcl_worker_get_current ();
2354 : vcl_session_t *s;
2355 :
2356 1109770 : s = vcl_session_get_w_handle (wrk, session_handle);
2357 1109770 : if (PREDICT_FALSE (!s))
2358 0 : return VPPCOM_EBADFD;
2359 :
2360 1109770 : return vppcom_session_write_inline (wrk, s, buf, n, 1 /* is_flush */,
2361 1109770 : s->is_dgram ? 1 : 0);
2362 : }
2363 :
2364 : #define vcl_fifo_rx_evt_valid_or_break(_s) \
2365 : if (PREDICT_FALSE (!_s->rx_fifo)) \
2366 : break; \
2367 : if (PREDICT_FALSE (svm_fifo_is_empty (_s->rx_fifo))) \
2368 : { \
2369 : if (!vcl_session_is_ct (_s)) \
2370 : { \
2371 : svm_fifo_unset_event (_s->rx_fifo); \
2372 : if (svm_fifo_is_empty (_s->rx_fifo)) \
2373 : break; \
2374 : } \
2375 : else if (svm_fifo_is_empty (_s->ct_rx_fifo)) \
2376 : { \
2377 : svm_fifo_unset_event (_s->rx_fifo); /* rx evts on actual fifo*/ \
2378 : if (svm_fifo_is_empty (_s->ct_rx_fifo)) \
2379 : break; \
2380 : } \
2381 : } \
2382 :
2383 : static void
2384 97325 : vcl_select_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
2385 : unsigned long n_bits, unsigned long *read_map,
2386 : unsigned long *write_map,
2387 : unsigned long *except_map, u32 * bits_set)
2388 : {
2389 : session_disconnected_msg_t *disconnected_msg;
2390 : session_connected_msg_t *connected_msg;
2391 : vcl_session_t *s;
2392 : u32 sid;
2393 :
2394 97325 : switch (e->event_type)
2395 : {
2396 70016 : case SESSION_IO_EVT_RX:
2397 70016 : sid = e->session_index;
2398 70016 : s = vcl_session_get (wrk, sid);
2399 70016 : if (!s || !vcl_session_is_open (s))
2400 : break;
2401 70016 : vcl_fifo_rx_evt_valid_or_break (s);
2402 34918 : if (sid < n_bits && read_map)
2403 : {
2404 34918 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2405 34918 : *bits_set += 1;
2406 : }
2407 34918 : break;
2408 27293 : case SESSION_IO_EVT_TX:
2409 27293 : sid = e->session_index;
2410 27293 : s = vcl_session_get (wrk, sid);
2411 27293 : if (!s || !vcl_session_is_open (s))
2412 : break;
2413 27293 : if (sid < n_bits && write_map)
2414 : {
2415 27293 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2416 27293 : *bits_set += 1;
2417 : }
2418 27293 : break;
2419 9 : case SESSION_CTRL_EVT_ACCEPTED:
2420 9 : if (!e->postponed)
2421 9 : s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
2422 : else
2423 0 : s = vcl_session_get (wrk, e->session_index);
2424 9 : if (!s)
2425 0 : break;
2426 9 : sid = s->session_index;
2427 9 : if (sid < n_bits && read_map)
2428 : {
2429 9 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2430 9 : *bits_set += 1;
2431 : }
2432 9 : break;
2433 0 : case SESSION_CTRL_EVT_CONNECTED:
2434 0 : if (!e->postponed)
2435 : {
2436 0 : connected_msg = (session_connected_msg_t *) e->data;
2437 0 : sid = vcl_session_connected_handler (wrk, connected_msg);
2438 : }
2439 : else
2440 0 : sid = e->session_index;
2441 0 : if (sid == VCL_INVALID_SESSION_INDEX)
2442 0 : break;
2443 0 : if (!(sid < n_bits && write_map))
2444 : break;
2445 0 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2446 0 : *bits_set += 1;
2447 0 : s = vcl_session_get (wrk, sid);
2448 : /* We didn't have a fifo when the event was added */
2449 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2450 0 : break;
2451 0 : case SESSION_CTRL_EVT_DISCONNECTED:
2452 0 : if (!e->postponed)
2453 : {
2454 0 : disconnected_msg = (session_disconnected_msg_t *) e->data;
2455 0 : s = vcl_session_disconnected_handler (wrk, disconnected_msg);
2456 0 : if (!s)
2457 0 : break;
2458 : }
2459 : else
2460 : {
2461 0 : s = vcl_session_get (wrk, e->session_index);
2462 0 : s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
2463 : }
2464 0 : if (vcl_session_is_closed (s))
2465 : {
2466 0 : if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
2467 0 : vcl_session_free (wrk, s);
2468 0 : break;
2469 : }
2470 0 : sid = s->session_index;
2471 0 : if (sid < n_bits && except_map)
2472 : {
2473 0 : clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2474 0 : *bits_set += 1;
2475 : }
2476 0 : break;
2477 0 : case SESSION_CTRL_EVT_RESET:
2478 0 : if (!e->postponed)
2479 : {
2480 : sid =
2481 0 : vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
2482 0 : s = vcl_session_get (wrk, sid);
2483 : }
2484 : else
2485 : {
2486 0 : sid = e->session_index;
2487 0 : s = vcl_session_get (wrk, sid);
2488 0 : s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
2489 : }
2490 0 : if (vcl_session_is_closed (s))
2491 : {
2492 0 : if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
2493 0 : vcl_session_free (wrk, s);
2494 0 : break;
2495 : }
2496 0 : if (sid < n_bits && except_map)
2497 : {
2498 0 : clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2499 0 : *bits_set += 1;
2500 : }
2501 0 : break;
2502 1 : case SESSION_CTRL_EVT_UNLISTEN_REPLY:
2503 1 : vcl_session_unlisten_reply_handler (wrk, e->data);
2504 1 : break;
2505 0 : case SESSION_CTRL_EVT_MIGRATED:
2506 0 : vcl_session_migrated_handler (wrk, e->data);
2507 0 : break;
2508 0 : case SESSION_CTRL_EVT_CLEANUP:
2509 0 : vcl_session_cleanup_handler (wrk, e->data);
2510 0 : break;
2511 0 : case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
2512 0 : vcl_session_worker_update_reply_handler (wrk, e->data);
2513 0 : break;
2514 0 : case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
2515 0 : vcl_session_req_worker_update_handler (wrk, e->data);
2516 0 : break;
2517 6 : case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
2518 6 : vcl_session_app_add_segment_handler (wrk, e->data);
2519 6 : break;
2520 0 : case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
2521 0 : vcl_session_app_del_segment_handler (wrk, e->data);
2522 0 : break;
2523 0 : case SESSION_CTRL_EVT_APP_WRK_RPC:
2524 0 : vcl_worker_rpc_handler (wrk, e->data);
2525 0 : break;
2526 0 : default:
2527 0 : clib_warning ("unhandled: %u", e->event_type);
2528 0 : break;
2529 : }
2530 0 : }
2531 :
2532 : static int
2533 26709800 : vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
2534 : unsigned long n_bits, unsigned long *read_map,
2535 : unsigned long *write_map, unsigned long *except_map,
2536 : double time_to_wait, u32 * bits_set)
2537 : {
2538 : svm_msg_q_msg_t *msg;
2539 : session_event_t *e;
2540 : u32 i;
2541 :
2542 26709800 : if (svm_msg_q_is_empty (mq))
2543 : {
2544 26626500 : if (*bits_set)
2545 459085 : return 0;
2546 :
2547 26167400 : if (!time_to_wait)
2548 26167400 : return 0;
2549 0 : else if (time_to_wait < 0)
2550 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2551 : else
2552 : {
2553 0 : if (svm_msg_q_timedwait (mq, time_to_wait))
2554 0 : return 0;
2555 : }
2556 : }
2557 83335 : vcl_mq_dequeue_batch (wrk, mq, ~0);
2558 :
2559 167481 : for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
2560 : {
2561 84146 : msg = vec_elt_at_index (wrk->mq_msg_vector, i);
2562 84146 : e = svm_msg_q_msg_data (mq, msg);
2563 84146 : vcl_select_handle_mq_event (wrk, e, n_bits, read_map, write_map,
2564 : except_map, bits_set);
2565 84146 : svm_msg_q_free_msg (mq, msg);
2566 : }
2567 83335 : vec_reset_length (wrk->mq_msg_vector);
2568 83335 : vcl_handle_pending_wrk_updates (wrk);
2569 83335 : return *bits_set;
2570 : }
2571 :
2572 : static int
2573 26709800 : vppcom_select_condvar (vcl_worker_t * wrk, int n_bits,
2574 : vcl_si_set * read_map, vcl_si_set * write_map,
2575 : vcl_si_set * except_map, double time_to_wait,
2576 : u32 * bits_set)
2577 : {
2578 26709800 : double wait = 0, start = 0;
2579 :
2580 26709800 : if (!*bits_set)
2581 : {
2582 26227200 : wait = time_to_wait;
2583 26227200 : start = clib_time_now (&wrk->clib_time);
2584 : }
2585 :
2586 : do
2587 : {
2588 26709800 : vcl_select_handle_mq (wrk, wrk->app_event_queue, n_bits, read_map,
2589 : write_map, except_map, wait, bits_set);
2590 26709800 : if (*bits_set)
2591 511926 : return *bits_set;
2592 26197900 : if (wait == -1)
2593 0 : continue;
2594 :
2595 26197900 : wait = wait - (clib_time_now (&wrk->clib_time) - start);
2596 : }
2597 26197900 : while (wait > 0);
2598 :
2599 26197900 : return 0;
2600 : }
2601 :
2602 : static int
2603 0 : vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits,
2604 : vcl_si_set * read_map, vcl_si_set * write_map,
2605 : vcl_si_set * except_map, double time_to_wait,
2606 : u32 * bits_set)
2607 : {
2608 : vcl_mq_evt_conn_t *mqc;
2609 : int __clib_unused n_read;
2610 : int n_mq_evts, i;
2611 : u64 buf;
2612 :
2613 0 : if (PREDICT_FALSE (wrk->api_client_handle == ~0))
2614 : {
2615 0 : vcl_api_retry_attach (wrk);
2616 0 : return 0;
2617 : }
2618 :
2619 0 : vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
2620 0 : n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
2621 0 : vec_len (wrk->mq_events), time_to_wait);
2622 0 : for (i = 0; i < n_mq_evts; i++)
2623 : {
2624 0 : if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0))
2625 : {
2626 0 : vcl_api_handle_disconnect (wrk);
2627 0 : continue;
2628 : }
2629 :
2630 0 : mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
2631 0 : n_read = read (mqc->mq_fd, &buf, sizeof (buf));
2632 0 : vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map,
2633 : except_map, 0, bits_set);
2634 : }
2635 :
2636 0 : return (n_mq_evts > 0 ? (int) *bits_set : 0);
2637 : }
2638 :
2639 : int
2640 26709800 : vppcom_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
2641 : vcl_si_set * except_map, double time_to_wait)
2642 : {
2643 26709800 : u32 sid, minbits = clib_max (n_bits, BITS (uword)), bits_set = 0;
2644 26709800 : vcl_worker_t *wrk = vcl_worker_get_current ();
2645 26709800 : vcl_session_t *s = 0;
2646 : int i;
2647 :
2648 26709800 : if (n_bits && read_map)
2649 : {
2650 26708800 : clib_bitmap_validate (wrk->rd_bitmap, minbits);
2651 26708800 : clib_memcpy_fast (wrk->rd_bitmap, read_map,
2652 26708800 : vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2653 26708800 : memset (read_map, 0, vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2654 : }
2655 26709800 : if (n_bits && write_map)
2656 : {
2657 26709800 : clib_bitmap_validate (wrk->wr_bitmap, minbits);
2658 26709800 : clib_memcpy_fast (wrk->wr_bitmap, write_map,
2659 26709800 : vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2660 26709800 : memset (write_map, 0, vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2661 : }
2662 26709800 : if (n_bits && except_map)
2663 : {
2664 0 : clib_bitmap_validate (wrk->ex_bitmap, minbits);
2665 0 : clib_memcpy_fast (wrk->ex_bitmap, except_map,
2666 0 : vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2667 0 : memset (except_map, 0, vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2668 : }
2669 :
2670 26709800 : if (!n_bits)
2671 0 : return 0;
2672 :
2673 26709800 : if (!write_map)
2674 6 : goto check_rd;
2675 :
2676 34793700 : clib_bitmap_foreach (sid, wrk->wr_bitmap)
2677 : {
2678 8083930 : if (!(s = vcl_session_get (wrk, sid)))
2679 : {
2680 0 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2681 0 : bits_set++;
2682 0 : continue;
2683 : }
2684 :
2685 8083930 : if (vcl_session_write_ready (s))
2686 : {
2687 106500 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2688 106500 : bits_set++;
2689 : }
2690 : else
2691 : {
2692 7977430 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF);
2693 : }
2694 : }
2695 :
2696 26709800 : check_rd:
2697 26709800 : if (!read_map)
2698 1000 : goto check_mq;
2699 :
2700 75921100 : clib_bitmap_foreach (sid, wrk->rd_bitmap)
2701 : {
2702 49212300 : if (!(s = vcl_session_get (wrk, sid)))
2703 : {
2704 0 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2705 0 : bits_set++;
2706 0 : continue;
2707 : }
2708 :
2709 49212300 : if (vcl_session_read_ready (s))
2710 : {
2711 421082 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2712 421082 : bits_set++;
2713 : }
2714 : }
2715 :
2716 26708800 : check_mq:
2717 :
2718 26723000 : for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
2719 : {
2720 13179 : vcl_select_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i], n_bits,
2721 : read_map, write_map, except_map, &bits_set);
2722 : }
2723 26709800 : vec_reset_length (wrk->unhandled_evts_vector);
2724 :
2725 26709800 : if (vcm->cfg.use_mq_eventfd)
2726 0 : vppcom_select_eventfd (wrk, n_bits, read_map, write_map, except_map,
2727 : time_to_wait, &bits_set);
2728 : else
2729 26709800 : vppcom_select_condvar (wrk, n_bits, read_map, write_map, except_map,
2730 : time_to_wait, &bits_set);
2731 :
2732 26709800 : return (bits_set);
2733 : }
2734 :
2735 : static inline void
2736 94 : vep_verify_epoll_chain (vcl_worker_t * wrk, u32 vep_handle)
2737 : {
2738 : vppcom_epoll_t *vep;
2739 94 : u32 sh = vep_handle;
2740 : vcl_session_t *s;
2741 :
2742 94 : if (VPPCOM_DEBUG <= 3)
2743 94 : return;
2744 :
2745 0 : s = vcl_session_get_w_handle (wrk, vep_handle);
2746 0 : if (PREDICT_FALSE (!s))
2747 : {
2748 0 : VDBG (0, "ERROR: Invalid vep_sh (%u)!", vep_handle);
2749 0 : goto done;
2750 : }
2751 0 : if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP)))
2752 : {
2753 0 : VDBG (0, "ERROR: vep_sh (%u) is not a vep!", vep_handle);
2754 0 : goto done;
2755 : }
2756 0 : vep = &s->vep;
2757 0 : VDBG (0, "vep_sh (%u): Dumping epoll chain\n"
2758 : "{\n"
2759 : " is_vep = %u\n"
2760 : " is_vep_session = %u\n"
2761 : " next_sh = 0x%x (%u)\n"
2762 : "}\n", vep_handle, s->flags & VCL_SESSION_F_IS_VEP,
2763 : s->flags & VCL_SESSION_F_IS_VEP_SESSION, vep->next_sh, vep->next_sh);
2764 :
2765 0 : for (sh = vep->next_sh; sh != ~0; sh = vep->next_sh)
2766 : {
2767 0 : s = vcl_session_get_w_handle (wrk, sh);
2768 0 : if (PREDICT_FALSE (!s))
2769 : {
2770 0 : VDBG (0, "ERROR: Invalid sh (%u)!", sh);
2771 0 : goto done;
2772 : }
2773 0 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2774 : {
2775 0 : VDBG (0, "ERROR: sh (%u) is a vep!", vep_handle);
2776 : }
2777 0 : else if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2778 : {
2779 0 : VDBG (0, "ERROR: sh (%u) is not a vep session handle!", sh);
2780 0 : goto done;
2781 : }
2782 0 : vep = &s->vep;
2783 0 : if (PREDICT_FALSE (vep->vep_sh != vep_handle))
2784 0 : VDBG (0, "ERROR: session (%u) vep_sh (%u) != vep_sh (%u)!",
2785 : sh, s->vep.vep_sh, vep_handle);
2786 0 : if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
2787 : {
2788 0 : VDBG (0, "vep_sh[%u]: sh 0x%x (%u)\n"
2789 : "{\n"
2790 : " next_sh = 0x%x (%u)\n"
2791 : " prev_sh = 0x%x (%u)\n"
2792 : " vep_sh = 0x%x (%u)\n"
2793 : " ev.events = 0x%x\n"
2794 : " ev.data.u64 = 0x%llx\n"
2795 : " et_mask = 0x%x\n"
2796 : "}\n",
2797 : vep_handle, sh, sh, vep->next_sh, vep->next_sh, vep->prev_sh,
2798 : vep->prev_sh, vep->vep_sh, vep->vep_sh, vep->ev.events,
2799 : vep->ev.data.u64, vep->et_mask);
2800 : }
2801 : }
2802 :
2803 0 : done:
2804 0 : VDBG (0, "vep_sh (%u): Dump complete!\n", vep_handle);
2805 : }
2806 :
2807 : int
2808 13 : vppcom_epoll_create (void)
2809 : {
2810 13 : vcl_worker_t *wrk = vcl_worker_get_current ();
2811 : vcl_session_t *vep_session;
2812 :
2813 13 : vep_session = vcl_session_alloc (wrk);
2814 :
2815 13 : vep_session->flags |= VCL_SESSION_F_IS_VEP;
2816 13 : vep_session->vep.vep_sh = ~0;
2817 13 : vep_session->vep.next_sh = ~0;
2818 13 : vep_session->vep.prev_sh = ~0;
2819 13 : vep_session->vpp_handle = ~0;
2820 :
2821 : vcl_evt (VCL_EVT_EPOLL_CREATE, vep_session, vep_session->session_index);
2822 13 : VDBG (0, "Created vep_idx %u", vep_session->session_index);
2823 :
2824 13 : return vcl_session_handle (vep_session);
2825 : }
2826 :
2827 : static void
2828 0 : vcl_epoll_ctl_add_unhandled_event (vcl_worker_t *wrk, vcl_session_t *s,
2829 : u8 is_epollet, session_evt_type_t evt)
2830 : {
2831 0 : if (!is_epollet)
2832 : {
2833 0 : if (s->vep.lt_next == VCL_INVALID_SESSION_INDEX)
2834 0 : vcl_epoll_lt_add (wrk, s);
2835 0 : return;
2836 : }
2837 :
2838 0 : session_event_t e = { 0 };
2839 0 : e.session_index = s->session_index;
2840 0 : e.event_type = evt;
2841 0 : if (evt == SESSION_IO_EVT_RX)
2842 0 : s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
2843 0 : vec_add1 (wrk->unhandled_evts_vector, e);
2844 : }
2845 :
2846 : int
2847 94 : vppcom_epoll_ctl (uint32_t vep_handle, int op, uint32_t session_handle,
2848 : struct epoll_event *event)
2849 : {
2850 94 : vcl_worker_t *wrk = vcl_worker_get_current ();
2851 94 : int rv = VPPCOM_OK, add_evt = 0;
2852 : vcl_session_t *vep_session;
2853 : vcl_session_t *s;
2854 :
2855 94 : if (vep_handle == session_handle)
2856 : {
2857 0 : VDBG (0, "vep_sh == session handle (%u)!", vep_handle);
2858 0 : return VPPCOM_EINVAL;
2859 : }
2860 :
2861 94 : vep_session = vcl_session_get_w_handle (wrk, vep_handle);
2862 94 : if (PREDICT_FALSE (!vep_session))
2863 : {
2864 0 : VDBG (0, "Invalid vep_sh (%u)!", vep_handle);
2865 0 : return VPPCOM_EBADFD;
2866 : }
2867 94 : if (PREDICT_FALSE (!(vep_session->flags & VCL_SESSION_F_IS_VEP)))
2868 : {
2869 0 : VDBG (0, "vep_sh (%u) is not a vep!", vep_handle);
2870 0 : return VPPCOM_EINVAL;
2871 : }
2872 :
2873 94 : ASSERT (vep_session->vep.vep_sh == ~0);
2874 94 : ASSERT (vep_session->vep.prev_sh == ~0);
2875 :
2876 94 : s = vcl_session_get_w_handle (wrk, session_handle);
2877 94 : if (PREDICT_FALSE (!s))
2878 : {
2879 0 : VDBG (0, "Invalid session_handle (%u)!", session_handle);
2880 0 : return VPPCOM_EBADFD;
2881 : }
2882 94 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2883 : {
2884 0 : VDBG (0, "session_handle (%u) is a vep!", vep_handle);
2885 0 : return VPPCOM_EINVAL;
2886 : }
2887 :
2888 94 : switch (op)
2889 : {
2890 53 : case EPOLL_CTL_ADD:
2891 53 : if (PREDICT_FALSE (!event))
2892 : {
2893 0 : VDBG (0, "EPOLL_CTL_ADD: NULL pointer to epoll_event structure!");
2894 0 : return VPPCOM_EINVAL;
2895 : }
2896 53 : if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
2897 : {
2898 0 : VDBG (0, "EPOLL_CTL_ADD: %u already epolled!", s->session_index);
2899 0 : rv = VPPCOM_EEXIST;
2900 0 : goto done;
2901 : }
2902 53 : if (vep_session->vep.next_sh != ~0)
2903 : {
2904 : vcl_session_t *next_session;
2905 40 : next_session = vcl_session_get_w_handle (wrk,
2906 : vep_session->vep.next_sh);
2907 40 : if (PREDICT_FALSE (!next_session))
2908 : {
2909 0 : VDBG (0, "EPOLL_CTL_ADD: Invalid vep.next_sh (%u) on "
2910 : "vep_idx (%u)!", vep_session->vep.next_sh, vep_handle);
2911 0 : return VPPCOM_EBADFD;
2912 : }
2913 40 : ASSERT (next_session->vep.prev_sh == vep_handle);
2914 40 : next_session->vep.prev_sh = session_handle;
2915 : }
2916 53 : s->vep.next_sh = vep_session->vep.next_sh;
2917 53 : s->vep.prev_sh = vep_handle;
2918 53 : s->vep.vep_sh = vep_handle;
2919 53 : s->vep.et_mask = VEP_DEFAULT_ET_MASK;
2920 53 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
2921 53 : s->vep.ev = *event;
2922 53 : s->vep.ev.events |= EPOLLHUP | EPOLLERR;
2923 53 : s->flags &= ~VCL_SESSION_F_IS_VEP;
2924 53 : s->flags |= VCL_SESSION_F_IS_VEP_SESSION;
2925 53 : vep_session->vep.next_sh = session_handle;
2926 :
2927 53 : if ((event->events & EPOLLOUT))
2928 : {
2929 0 : int write_ready = vcl_session_write_ready (s);
2930 :
2931 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2932 0 : if (write_ready > 0)
2933 : {
2934 : /* Generate EPOLLOUT if tx fifo not full */
2935 0 : vcl_epoll_ctl_add_unhandled_event (
2936 : wrk, s, event->events & EPOLLET, SESSION_IO_EVT_TX);
2937 0 : add_evt = 1;
2938 : }
2939 : else
2940 : {
2941 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF);
2942 : }
2943 : }
2944 : /* Generate EPOLLIN if rx fifo has data */
2945 53 : if ((event->events & EPOLLIN) && (vcl_session_read_ready (s) > 0))
2946 : {
2947 0 : vcl_epoll_ctl_add_unhandled_event (wrk, s, event->events & EPOLLET,
2948 : SESSION_IO_EVT_RX);
2949 0 : add_evt = 1;
2950 : }
2951 53 : if (!add_evt && vcl_session_is_closing (s))
2952 : {
2953 0 : session_event_t e = { 0 };
2954 0 : if (s->session_state == VCL_STATE_VPP_CLOSING)
2955 0 : e.event_type = SESSION_CTRL_EVT_DISCONNECTED;
2956 : else
2957 0 : e.event_type = SESSION_CTRL_EVT_RESET;
2958 0 : e.session_index = s->session_index;
2959 0 : e.postponed = 1;
2960 0 : vec_add1 (wrk->unhandled_evts_vector, e);
2961 : }
2962 53 : VDBG (1, "EPOLL_CTL_ADD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2963 : vep_handle, session_handle, event->events, event->data.u64);
2964 : vcl_evt (VCL_EVT_EPOLL_CTLADD, s, event->events, event->data.u64);
2965 53 : break;
2966 :
2967 0 : case EPOLL_CTL_MOD:
2968 0 : if (PREDICT_FALSE (!event))
2969 : {
2970 0 : VDBG (0, "EPOLL_CTL_MOD: NULL pointer to epoll_event structure!");
2971 0 : rv = VPPCOM_EINVAL;
2972 0 : goto done;
2973 : }
2974 0 : else if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2975 : {
2976 0 : VDBG (0, "sh %u EPOLL_CTL_MOD: not a vep session!", session_handle);
2977 0 : rv = VPPCOM_ENOENT;
2978 0 : goto done;
2979 : }
2980 0 : else if (PREDICT_FALSE (s->vep.vep_sh != vep_handle))
2981 : {
2982 0 : VDBG (0, "EPOLL_CTL_MOD: sh %u vep_sh (%u) != vep_sh (%u)!",
2983 : session_handle, s->vep.vep_sh, vep_handle);
2984 0 : rv = VPPCOM_EINVAL;
2985 0 : goto done;
2986 : }
2987 :
2988 : /* Generate EPOLLOUT if session write ready and event was not on */
2989 0 : if ((event->events & EPOLLOUT) && !(s->vep.ev.events & EPOLLOUT))
2990 0 : {
2991 : /* Fifo size load acq synchronized with update store rel */
2992 0 : int write_ready = vcl_session_write_ready (s);
2993 :
2994 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2995 0 : if (write_ready > 0)
2996 0 : vcl_epoll_ctl_add_unhandled_event (wrk, s, event->events & EPOLLET,
2997 : SESSION_IO_EVT_TX);
2998 : else
2999 : /* Request deq ntf in case dequeue happened while updating flag */
3000 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF);
3001 : }
3002 0 : else if (!(event->events & EPOLLOUT))
3003 0 : vcl_session_del_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
3004 :
3005 : /* Generate EPOLLIN if session read ready and event was not on */
3006 0 : if ((event->events & EPOLLIN) && !(s->vep.ev.events & EPOLLIN) &&
3007 0 : (vcl_session_read_ready (s) > 0))
3008 : {
3009 0 : vcl_epoll_ctl_add_unhandled_event (wrk, s, event->events & EPOLLET,
3010 : SESSION_IO_EVT_RX);
3011 : }
3012 0 : s->vep.et_mask = VEP_DEFAULT_ET_MASK;
3013 0 : s->vep.ev = *event;
3014 0 : s->vep.ev.events |= EPOLLHUP | EPOLLERR;
3015 :
3016 0 : VDBG (1, "EPOLL_CTL_MOD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
3017 : vep_handle, session_handle, event->events, event->data.u64);
3018 0 : break;
3019 :
3020 41 : case EPOLL_CTL_DEL:
3021 41 : if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
3022 : {
3023 0 : VDBG (0, "EPOLL_CTL_DEL: %u not a vep session!", session_handle);
3024 0 : rv = VPPCOM_ENOENT;
3025 0 : goto done;
3026 : }
3027 41 : else if (PREDICT_FALSE (s->vep.vep_sh != vep_handle))
3028 : {
3029 0 : VDBG (0, "EPOLL_CTL_DEL: sh %u vep_sh (%u) != vep_sh (%u)!",
3030 : session_handle, s->vep.vep_sh, vep_handle);
3031 0 : rv = VPPCOM_EINVAL;
3032 0 : goto done;
3033 : }
3034 :
3035 41 : if (s->vep.prev_sh == vep_handle)
3036 34 : vep_session->vep.next_sh = s->vep.next_sh;
3037 : else
3038 : {
3039 : vcl_session_t *prev_session;
3040 7 : prev_session = vcl_session_get_w_handle (wrk, s->vep.prev_sh);
3041 7 : if (PREDICT_FALSE (!prev_session))
3042 : {
3043 0 : VDBG (0, "EPOLL_CTL_DEL: Invalid prev_sh (%u) on sh (%u)!",
3044 : s->vep.prev_sh, session_handle);
3045 0 : return VPPCOM_EBADFD;
3046 : }
3047 7 : ASSERT (prev_session->vep.next_sh == session_handle);
3048 7 : prev_session->vep.next_sh = s->vep.next_sh;
3049 : }
3050 41 : if (s->vep.next_sh != ~0)
3051 : {
3052 : vcl_session_t *next_session;
3053 40 : next_session = vcl_session_get_w_handle (wrk, s->vep.next_sh);
3054 40 : if (PREDICT_FALSE (!next_session))
3055 : {
3056 0 : VDBG (0, "EPOLL_CTL_DEL: Invalid next_sh (%u) on sh (%u)!",
3057 : s->vep.next_sh, session_handle);
3058 0 : return VPPCOM_EBADFD;
3059 : }
3060 40 : ASSERT (next_session->vep.prev_sh == session_handle);
3061 40 : next_session->vep.prev_sh = s->vep.prev_sh;
3062 : }
3063 :
3064 41 : if (s->vep.lt_next != VCL_INVALID_SESSION_INDEX)
3065 0 : vcl_epoll_lt_del (wrk, s);
3066 :
3067 41 : memset (&s->vep, 0, sizeof (s->vep));
3068 41 : s->vep.next_sh = ~0;
3069 41 : s->vep.prev_sh = ~0;
3070 41 : s->vep.vep_sh = ~0;
3071 41 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
3072 41 : s->flags &= ~VCL_SESSION_F_IS_VEP_SESSION;
3073 :
3074 41 : if (vcl_session_is_open (s))
3075 26 : vcl_session_del_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
3076 :
3077 41 : VDBG (1, "EPOLL_CTL_DEL: vep_idx %u, sh %u!", vep_handle,
3078 : session_handle);
3079 : vcl_evt (VCL_EVT_EPOLL_CTLDEL, s, vep_sh);
3080 41 : break;
3081 :
3082 0 : default:
3083 0 : VDBG (0, "Invalid operation (%d)!", op);
3084 0 : rv = VPPCOM_EINVAL;
3085 : }
3086 :
3087 94 : vep_verify_epoll_chain (wrk, vep_handle);
3088 :
3089 94 : done:
3090 94 : return rv;
3091 : }
3092 :
3093 : always_inline u8
3094 4832 : vcl_ep_session_needs_evt (vcl_session_t *s, u32 evt)
3095 : {
3096 : /* No event if not epolled / events reset on hup or level-trigger on */
3097 9664 : return ((s->vep.ev.events & evt) &&
3098 4832 : s->vep.lt_next == VCL_INVALID_SESSION_INDEX);
3099 : }
3100 :
3101 : static inline void
3102 5610 : vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
3103 : struct epoll_event *events, u32 * num_ev)
3104 : {
3105 : session_disconnected_msg_t *disconnected_msg;
3106 : session_connected_msg_t *connected_msg;
3107 5610 : u32 sid = ~0, session_events;
3108 5610 : u64 session_evt_data = ~0;
3109 : vcl_session_t *s;
3110 5610 : u8 add_event = 0;
3111 :
3112 5610 : switch (e->event_type)
3113 : {
3114 5538 : case SESSION_IO_EVT_RX:
3115 5538 : sid = e->session_index;
3116 5538 : s = vcl_session_get (wrk, sid);
3117 5538 : if (vcl_session_is_closed (s))
3118 0 : break;
3119 5538 : vcl_fifo_rx_evt_valid_or_break (s);
3120 4797 : if (!vcl_ep_session_needs_evt (s, EPOLLIN) ||
3121 4797 : (s->flags & VCL_SESSION_F_HAS_RX_EVT))
3122 : break;
3123 4787 : session_events = s->vep.ev.events;
3124 4787 : add_event = 1;
3125 4787 : events[*num_ev].events = EPOLLIN;
3126 4787 : session_evt_data = s->vep.ev.data.u64;
3127 4787 : s->flags |= VCL_SESSION_F_HAS_RX_EVT;
3128 4787 : break;
3129 0 : case SESSION_IO_EVT_TX:
3130 0 : sid = e->session_index;
3131 0 : s = vcl_session_get (wrk, sid);
3132 0 : if (!s || !vcl_session_is_open (s))
3133 : break;
3134 0 : svm_fifo_reset_has_deq_ntf (vcl_session_is_ct (s) ? s->ct_tx_fifo :
3135 : s->tx_fifo);
3136 0 : if (!vcl_ep_session_needs_evt (s, EPOLLOUT))
3137 0 : break;
3138 0 : session_events = s->vep.ev.events;
3139 0 : add_event = 1;
3140 0 : events[*num_ev].events = EPOLLOUT;
3141 0 : session_evt_data = s->vep.ev.data.u64;
3142 0 : break;
3143 28 : case SESSION_CTRL_EVT_ACCEPTED:
3144 28 : if (!e->postponed)
3145 28 : s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
3146 : else
3147 0 : s = vcl_session_get (wrk, e->session_index);
3148 28 : if (!s || !vcl_ep_session_needs_evt (s, EPOLLIN))
3149 : break;
3150 28 : sid = s->session_index;
3151 28 : session_events = s->vep.ev.events;
3152 28 : add_event = 1;
3153 28 : events[*num_ev].events = EPOLLIN;
3154 28 : session_evt_data = s->vep.ev.data.u64;
3155 28 : break;
3156 0 : case SESSION_CTRL_EVT_CONNECTED:
3157 0 : if (!e->postponed)
3158 : {
3159 0 : connected_msg = (session_connected_msg_t *) e->data;
3160 0 : sid = vcl_session_connected_handler (wrk, connected_msg);
3161 : }
3162 : else
3163 0 : sid = e->session_index;
3164 0 : s = vcl_session_get (wrk, sid);
3165 0 : if (vcl_session_is_closed (s) || !vcl_ep_session_needs_evt (s, EPOLLOUT))
3166 : break;
3167 : /* We didn't have a fifo when the event was added */
3168 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
3169 0 : add_event = 1;
3170 0 : session_events = s->vep.ev.events;
3171 : /* Generate EPOLLOUT because there's no connected event */
3172 0 : events[*num_ev].events = EPOLLOUT;
3173 0 : session_evt_data = s->vep.ev.data.u64;
3174 0 : if (s->session_state == VCL_STATE_DETACHED)
3175 : {
3176 0 : events[*num_ev].events |= EPOLLHUP;
3177 0 : s->vep.ev.events = 0;
3178 : }
3179 0 : break;
3180 10 : case SESSION_CTRL_EVT_DISCONNECTED:
3181 10 : if (!e->postponed)
3182 : {
3183 10 : disconnected_msg = (session_disconnected_msg_t *) e->data;
3184 10 : s = vcl_session_disconnected_handler (wrk, disconnected_msg);
3185 : }
3186 : else
3187 : {
3188 0 : s = vcl_session_get (wrk, e->session_index);
3189 0 : s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
3190 : }
3191 10 : if (vcl_session_is_closed (s) || !vcl_ep_session_needs_evt (s, EPOLLHUP))
3192 : {
3193 5 : if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
3194 0 : vcl_session_free (wrk, s);
3195 5 : break;
3196 : }
3197 5 : sid = s->session_index;
3198 5 : session_events = s->vep.ev.events;
3199 5 : add_event = 1;
3200 5 : if (EPOLLRDHUP & session_events)
3201 : {
3202 : /* If app can distinguish between RDHUP and HUP,
3203 : * we make finer control */
3204 0 : events[*num_ev].events = EPOLLRDHUP;
3205 0 : if (s->flags & VCL_SESSION_F_WR_SHUTDOWN)
3206 : {
3207 0 : events[*num_ev].events |= EPOLLHUP;
3208 : }
3209 : }
3210 : else
3211 : {
3212 5 : events[*num_ev].events = EPOLLHUP;
3213 : }
3214 5 : session_evt_data = s->vep.ev.data.u64;
3215 5 : s->vep.ev.events = 0;
3216 5 : break;
3217 0 : case SESSION_CTRL_EVT_BOUND:
3218 0 : vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
3219 0 : break;
3220 2 : case SESSION_CTRL_EVT_RESET:
3221 2 : if (!e->postponed)
3222 : {
3223 : sid =
3224 2 : vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
3225 2 : s = vcl_session_get (wrk, sid);
3226 : }
3227 : else
3228 : {
3229 0 : sid = e->session_index;
3230 0 : s = vcl_session_get (wrk, sid);
3231 0 : s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
3232 : }
3233 2 : if (vcl_session_is_closed (s) || !vcl_ep_session_needs_evt (s, EPOLLHUP))
3234 : {
3235 0 : if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
3236 0 : vcl_session_free (wrk, s);
3237 0 : break;
3238 : }
3239 2 : session_events = s->vep.ev.events;
3240 2 : add_event = 1;
3241 2 : events[*num_ev].events = EPOLLERR | EPOLLHUP;
3242 2 : if ((EPOLLRDHUP & session_events) &&
3243 0 : (s->flags & VCL_SESSION_F_RD_SHUTDOWN))
3244 : {
3245 0 : events[*num_ev].events |= EPOLLRDHUP;
3246 : }
3247 2 : if ((EPOLLIN & session_events) && (s->flags & VCL_SESSION_F_RD_SHUTDOWN))
3248 : {
3249 2 : events[*num_ev].events |= EPOLLIN;
3250 : }
3251 2 : session_evt_data = s->vep.ev.data.u64;
3252 2 : s->vep.ev.events = 0;
3253 2 : break;
3254 0 : case SESSION_CTRL_EVT_UNLISTEN_REPLY:
3255 0 : vcl_session_unlisten_reply_handler (wrk, e->data);
3256 0 : break;
3257 0 : case SESSION_CTRL_EVT_MIGRATED:
3258 0 : vcl_session_migrated_handler (wrk, e->data);
3259 0 : break;
3260 10 : case SESSION_CTRL_EVT_CLEANUP:
3261 10 : vcl_session_cleanup_handler (wrk, e->data);
3262 10 : break;
3263 0 : case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
3264 0 : vcl_session_req_worker_update_handler (wrk, e->data);
3265 0 : break;
3266 0 : case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
3267 0 : vcl_session_worker_update_reply_handler (wrk, e->data);
3268 0 : break;
3269 20 : case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
3270 20 : vcl_session_app_add_segment_handler (wrk, e->data);
3271 20 : break;
3272 2 : case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
3273 2 : vcl_session_app_del_segment_handler (wrk, e->data);
3274 2 : break;
3275 0 : case SESSION_CTRL_EVT_APP_WRK_RPC:
3276 0 : vcl_worker_rpc_handler (wrk, e->data);
3277 0 : break;
3278 0 : default:
3279 0 : VDBG (0, "unhandled: %u", e->event_type);
3280 0 : break;
3281 : }
3282 :
3283 5610 : if (add_event)
3284 : {
3285 4822 : ASSERT (s->flags & VCL_SESSION_F_IS_VEP_SESSION);
3286 4822 : events[*num_ev].data.u64 = session_evt_data;
3287 4822 : if (EPOLLONESHOT & session_events)
3288 : {
3289 0 : s = vcl_session_get (wrk, sid);
3290 0 : if (!(events[*num_ev].events & EPOLLHUP))
3291 0 : s->vep.ev.events = EPOLLHUP | EPOLLERR;
3292 : }
3293 4822 : else if (!(EPOLLET & session_events))
3294 : {
3295 0 : s = vcl_session_get (wrk, sid);
3296 0 : if (s->vep.lt_next == VCL_INVALID_SESSION_INDEX)
3297 0 : vcl_epoll_lt_add (wrk, s);
3298 : }
3299 4822 : *num_ev += 1;
3300 : }
3301 5610 : }
3302 :
3303 : static int
3304 126734000 : vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
3305 : struct epoll_event *events, u32 maxevents,
3306 : double wait_for_time, u32 * num_ev)
3307 : {
3308 : svm_msg_q_msg_t *msg;
3309 : session_event_t *e;
3310 : int i;
3311 :
3312 126734000 : if (vec_len (wrk->mq_msg_vector) && svm_msg_q_is_empty (mq))
3313 0 : goto handle_dequeued;
3314 :
3315 126734000 : if (svm_msg_q_is_empty (mq))
3316 : {
3317 126729000 : if (!wait_for_time)
3318 126729000 : return 0;
3319 0 : else if (wait_for_time < 0)
3320 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
3321 : else
3322 : {
3323 0 : if (svm_msg_q_timedwait (mq, wait_for_time / 1e3))
3324 0 : return 0;
3325 : }
3326 : }
3327 4473 : ASSERT (maxevents > *num_ev);
3328 4473 : vcl_mq_dequeue_batch (wrk, mq, ~0);
3329 :
3330 4473 : handle_dequeued:
3331 10083 : for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
3332 : {
3333 5610 : msg = vec_elt_at_index (wrk->mq_msg_vector, i);
3334 5610 : e = svm_msg_q_msg_data (mq, msg);
3335 5610 : if (*num_ev < maxevents)
3336 5610 : vcl_epoll_wait_handle_mq_event (wrk, e, events, num_ev);
3337 : else
3338 0 : vcl_handle_mq_event (wrk, e);
3339 5610 : svm_msg_q_free_msg (mq, msg);
3340 : }
3341 4473 : vec_reset_length (wrk->mq_msg_vector);
3342 4473 : vcl_handle_pending_wrk_updates (wrk);
3343 4473 : return *num_ev;
3344 : }
3345 :
3346 : static int
3347 126734000 : vppcom_epoll_wait_condvar (vcl_worker_t *wrk, struct epoll_event *events,
3348 : int maxevents, u32 n_evts, double timeout_ms)
3349 : {
3350 126734000 : double end = -1;
3351 :
3352 126734000 : if (!n_evts)
3353 : {
3354 126734000 : if (timeout_ms > 0)
3355 0 : end = clib_time_now (&wrk->clib_time) + (timeout_ms / 1e3);
3356 : }
3357 :
3358 : do
3359 : {
3360 126734000 : vcl_epoll_wait_handle_mq (wrk, wrk->app_event_queue, events, maxevents,
3361 : timeout_ms, &n_evts);
3362 126734000 : if (n_evts || !timeout_ms)
3363 126734000 : return n_evts;
3364 : }
3365 0 : while (end == -1 || clib_time_now (&wrk->clib_time) < end);
3366 :
3367 0 : return 0;
3368 : }
3369 :
3370 : static int
3371 0 : vppcom_epoll_wait_eventfd (vcl_worker_t *wrk, struct epoll_event *events,
3372 : int maxevents, u32 n_evts, double timeout_ms)
3373 : {
3374 : int __clib_unused n_read;
3375 : vcl_mq_evt_conn_t *mqc;
3376 : int n_mq_evts, i;
3377 0 : double end = -1;
3378 : u64 buf;
3379 :
3380 0 : if (PREDICT_FALSE (wrk->api_client_handle == ~0))
3381 : {
3382 0 : vcl_api_retry_attach (wrk);
3383 0 : return n_evts;
3384 : }
3385 :
3386 0 : vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
3387 0 : if (!n_evts)
3388 : {
3389 0 : if (timeout_ms > 0)
3390 0 : end = clib_time_now (&wrk->clib_time) + (timeout_ms / 1e3);
3391 : }
3392 :
3393 : do
3394 : {
3395 0 : n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
3396 0 : vec_len (wrk->mq_events), timeout_ms);
3397 0 : if (n_mq_evts < 0)
3398 : {
3399 0 : VDBG (0, "epoll_wait error %u", errno);
3400 0 : return n_evts;
3401 : }
3402 :
3403 0 : for (i = 0; i < n_mq_evts; i++)
3404 : {
3405 0 : if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0))
3406 : {
3407 : /* api socket was closed */
3408 0 : vcl_api_handle_disconnect (wrk);
3409 0 : continue;
3410 : }
3411 :
3412 0 : mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
3413 0 : n_read = read (mqc->mq_fd, &buf, sizeof (buf));
3414 0 : vcl_epoll_wait_handle_mq (wrk, mqc->mq, events, maxevents, 0,
3415 : &n_evts);
3416 : }
3417 :
3418 0 : if (n_evts || !timeout_ms)
3419 0 : return n_evts;
3420 : }
3421 0 : while (end == -1 || clib_time_now (&wrk->clib_time) < end);
3422 :
3423 0 : return 0;
3424 : }
3425 :
3426 : static void
3427 0 : vcl_epoll_wait_handle_lt (vcl_worker_t *wrk, struct epoll_event *events,
3428 : int maxevents, u32 *n_evts)
3429 : {
3430 0 : u32 add_event = 0, evt_flags = 0, next, *to_remove = 0, *si;
3431 : vcl_session_t *s;
3432 : u64 evt_data;
3433 : int rv;
3434 :
3435 0 : ASSERT (wrk->ep_lt_current != VCL_INVALID_SESSION_INDEX);
3436 0 : if (*n_evts >= maxevents)
3437 0 : return;
3438 :
3439 0 : next = wrk->ep_lt_current;
3440 : do
3441 : {
3442 0 : s = vcl_session_get (wrk, next);
3443 0 : next = s->vep.lt_next;
3444 :
3445 0 : if (s->vep.ev.events == 0)
3446 : {
3447 0 : vec_add1 (to_remove, s->session_index);
3448 0 : continue;
3449 : }
3450 0 : if ((s->vep.ev.events & EPOLLIN) && (rv = vcl_session_read_ready (s)))
3451 : {
3452 0 : add_event = 1;
3453 0 : evt_flags |= rv > 0 ? EPOLLIN : EPOLLHUP | EPOLLRDHUP;
3454 0 : evt_data = s->vep.ev.data.u64;
3455 : }
3456 0 : if ((s->vep.ev.events & EPOLLOUT) && (rv = vcl_session_write_ready (s)))
3457 : {
3458 0 : add_event = 1;
3459 0 : evt_flags |= rv > 0 ? EPOLLOUT : EPOLLHUP | EPOLLRDHUP;
3460 0 : evt_data = s->vep.ev.data.u64;
3461 : }
3462 0 : if (!add_event && s->session_state > VCL_STATE_READY)
3463 : {
3464 0 : add_event = 1;
3465 0 : evt_flags |= EPOLLHUP | EPOLLRDHUP;
3466 0 : evt_data = s->vep.ev.data.u64;
3467 : }
3468 0 : if (add_event)
3469 : {
3470 0 : events[*n_evts].events = evt_flags;
3471 0 : events[*n_evts].data.u64 = evt_data;
3472 0 : if (EPOLLONESHOT & s->vep.ev.events)
3473 0 : s->vep.ev.events = EPOLLHUP | EPOLLERR;
3474 0 : if (evt_flags & EPOLLHUP)
3475 0 : s->vep.ev.events = 0;
3476 0 : *n_evts += 1;
3477 0 : add_event = 0;
3478 0 : evt_flags = 0;
3479 0 : if (*n_evts == maxevents)
3480 : {
3481 0 : wrk->ep_lt_current = next;
3482 0 : break;
3483 : }
3484 : }
3485 : else
3486 : {
3487 0 : vec_add1 (to_remove, s->session_index);
3488 : }
3489 : }
3490 0 : while (next != wrk->ep_lt_current);
3491 :
3492 0 : vec_foreach (si, to_remove)
3493 : {
3494 0 : s = vcl_session_get (wrk, *si);
3495 0 : vcl_epoll_lt_del (wrk, s);
3496 : }
3497 0 : vec_free (to_remove);
3498 : }
3499 :
3500 : int
3501 126734000 : vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
3502 : int maxevents, double wait_for_time)
3503 : {
3504 126734000 : vcl_worker_t *wrk = vcl_worker_get_current ();
3505 : vcl_session_t *vep_session;
3506 126734000 : u32 n_evts = 0;
3507 : int i;
3508 :
3509 126734000 : if (PREDICT_FALSE (maxevents <= 0))
3510 : {
3511 0 : VDBG (0, "ERROR: Invalid maxevents (%d)!", maxevents);
3512 0 : return VPPCOM_EINVAL;
3513 : }
3514 :
3515 126734000 : vep_session = vcl_session_get_w_handle (wrk, vep_handle);
3516 126734000 : if (!vep_session)
3517 0 : return VPPCOM_EBADFD;
3518 :
3519 126734000 : if (PREDICT_FALSE (!(vep_session->flags & VCL_SESSION_F_IS_VEP)))
3520 : {
3521 0 : VDBG (0, "ERROR: vep_idx (%u) is not a vep!", vep_handle);
3522 0 : return VPPCOM_EINVAL;
3523 : }
3524 :
3525 126734000 : if (vec_len (wrk->unhandled_evts_vector))
3526 : {
3527 0 : for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
3528 : {
3529 0 : vcl_epoll_wait_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i],
3530 : events, &n_evts);
3531 0 : if (n_evts == maxevents)
3532 : {
3533 0 : vec_delete (wrk->unhandled_evts_vector, i + 1, 0);
3534 0 : return n_evts;
3535 : }
3536 : }
3537 0 : vec_reset_length (wrk->unhandled_evts_vector);
3538 : }
3539 :
3540 126734000 : if (PREDICT_FALSE (wrk->ep_lt_current != VCL_INVALID_SESSION_INDEX))
3541 0 : vcl_epoll_wait_handle_lt (wrk, events, maxevents, &n_evts);
3542 :
3543 : /* Request to only drain unhandled */
3544 126734000 : if ((int) wait_for_time == -2)
3545 0 : return n_evts;
3546 :
3547 :
3548 126734000 : if (vcm->cfg.use_mq_eventfd)
3549 0 : n_evts = vppcom_epoll_wait_eventfd (wrk, events, maxevents, n_evts,
3550 : wait_for_time);
3551 : else
3552 126734000 : n_evts = vppcom_epoll_wait_condvar (wrk, events, maxevents, n_evts,
3553 : wait_for_time);
3554 :
3555 126734000 : return n_evts;
3556 : }
3557 :
3558 : int
3559 6402 : vppcom_session_attr (uint32_t session_handle, uint32_t op,
3560 : void *buffer, uint32_t * buflen)
3561 : {
3562 6402 : vcl_worker_t *wrk = vcl_worker_get_current ();
3563 6402 : u32 *flags = buffer;
3564 6402 : vppcom_endpt_t *ep = buffer;
3565 : transport_endpt_attr_t tea;
3566 : vcl_session_t *session;
3567 6402 : int rv = VPPCOM_OK;
3568 :
3569 6402 : session = vcl_session_get_w_handle (wrk, session_handle);
3570 6402 : if (!session)
3571 0 : return VPPCOM_EBADFD;
3572 :
3573 6402 : switch (op)
3574 : {
3575 5966 : case VPPCOM_ATTR_GET_NREAD:
3576 5966 : rv = vcl_session_read_ready (session);
3577 5966 : VDBG (2, "VPPCOM_ATTR_GET_NREAD: sh %u, nread = %d", session_handle,
3578 : rv);
3579 5966 : break;
3580 :
3581 0 : case VPPCOM_ATTR_GET_NWRITE:
3582 0 : rv = vcl_session_write_ready (session);
3583 0 : VDBG (2, "VPPCOM_ATTR_GET_NWRITE: sh %u, nwrite = %d", session_handle,
3584 : rv);
3585 0 : break;
3586 :
3587 14 : case VPPCOM_ATTR_GET_FLAGS:
3588 14 : if (PREDICT_TRUE (buffer && buflen && (*buflen >= sizeof (*flags))))
3589 : {
3590 14 : *flags =
3591 14 : O_RDWR |
3592 14 : (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK) ?
3593 : O_NONBLOCK : 0);
3594 14 : *buflen = sizeof (*flags);
3595 14 : VDBG (2, "VPPCOM_ATTR_GET_FLAGS: sh %u, flags = 0x%08x, "
3596 : "is_nonblocking = %u", session_handle, *flags,
3597 : vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK));
3598 : }
3599 : else
3600 0 : rv = VPPCOM_EINVAL;
3601 14 : break;
3602 :
3603 37 : case VPPCOM_ATTR_SET_FLAGS:
3604 37 : if (PREDICT_TRUE (buffer && buflen && (*buflen == sizeof (*flags))))
3605 : {
3606 37 : if (*flags & O_NONBLOCK)
3607 33 : vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
3608 : else
3609 4 : vcl_session_clear_attr (session, VCL_SESS_ATTR_NONBLOCK);
3610 :
3611 37 : VDBG (2, "VPPCOM_ATTR_SET_FLAGS: sh %u, flags = 0x%08x,"
3612 : " is_nonblocking = %u", session_handle, *flags,
3613 : vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK));
3614 : }
3615 : else
3616 0 : rv = VPPCOM_EINVAL;
3617 37 : break;
3618 :
3619 17 : case VPPCOM_ATTR_GET_PEER_ADDR:
3620 17 : if (PREDICT_TRUE (buffer && buflen &&
3621 : (*buflen >= sizeof (*ep)) && ep->ip))
3622 : {
3623 17 : ep->is_ip4 = session->transport.is_ip4;
3624 17 : ep->port = session->transport.rmt_port;
3625 17 : if (session->transport.is_ip4)
3626 14 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
3627 : sizeof (ip4_address_t));
3628 : else
3629 3 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
3630 : sizeof (ip6_address_t));
3631 17 : *buflen = sizeof (*ep);
3632 17 : VDBG (1,
3633 : "VPPCOM_ATTR_GET_PEER_ADDR: sh %u, is_ip4 = %u, "
3634 : "addr = %U, port %u",
3635 : session_handle, ep->is_ip4, vcl_format_ip46_address,
3636 : &session->transport.rmt_ip,
3637 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3638 : clib_net_to_host_u16 (ep->port));
3639 : }
3640 : else
3641 0 : rv = VPPCOM_EINVAL;
3642 17 : break;
3643 :
3644 29 : case VPPCOM_ATTR_GET_LCL_ADDR:
3645 29 : if (PREDICT_TRUE (buffer && buflen &&
3646 : (*buflen >= sizeof (*ep)) && ep->ip))
3647 : {
3648 29 : ep->is_ip4 = session->transport.is_ip4;
3649 29 : ep->port = session->transport.lcl_port;
3650 29 : if (session->transport.is_ip4)
3651 24 : clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip4,
3652 : sizeof (ip4_address_t));
3653 : else
3654 5 : clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip6,
3655 : sizeof (ip6_address_t));
3656 29 : *buflen = sizeof (*ep);
3657 29 : VDBG (1,
3658 : "VPPCOM_ATTR_GET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3659 : " port %d",
3660 : session_handle, ep->is_ip4, vcl_format_ip46_address,
3661 : &session->transport.lcl_ip,
3662 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3663 : clib_net_to_host_u16 (ep->port));
3664 : }
3665 : else
3666 0 : rv = VPPCOM_EINVAL;
3667 29 : break;
3668 :
3669 0 : case VPPCOM_ATTR_GET_ORIGINAL_DST:
3670 0 : if (!session->transport.is_ip4)
3671 : {
3672 : /* now original dst only support ipv4*/
3673 0 : rv = VPPCOM_EAFNOSUPPORT;
3674 0 : break;
3675 : }
3676 0 : if (PREDICT_TRUE (buffer && buflen && (*buflen >= sizeof (*ep)) &&
3677 : ep->ip))
3678 : {
3679 0 : ep->is_ip4 = session->transport.is_ip4;
3680 0 : ep->port = session->original_dst_port;
3681 0 : clib_memcpy_fast (ep->ip, &session->original_dst_ip4,
3682 : sizeof (ip4_address_t));
3683 0 : *buflen = sizeof (*ep);
3684 0 : VDBG (1,
3685 : "VPPCOM_ATTR_GET_ORIGINAL_DST: sh %u, is_ip4 = %u, addr = %U"
3686 : " port %d",
3687 : session_handle, ep->is_ip4, vcl_format_ip4_address,
3688 : (ip4_address_t *) (&session->original_dst_ip4),
3689 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3690 : clib_net_to_host_u16 (ep->port));
3691 : }
3692 : else
3693 0 : rv = VPPCOM_EINVAL;
3694 0 : break;
3695 :
3696 0 : case VPPCOM_ATTR_SET_LCL_ADDR:
3697 0 : if (PREDICT_TRUE (buffer && buflen &&
3698 : (*buflen >= sizeof (*ep)) && ep->ip))
3699 : {
3700 0 : session->transport.is_ip4 = ep->is_ip4;
3701 0 : session->transport.lcl_port = ep->port;
3702 0 : vcl_ip_copy_from_ep (&session->transport.lcl_ip, ep);
3703 0 : *buflen = sizeof (*ep);
3704 0 : VDBG (1,
3705 : "VPPCOM_ATTR_SET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3706 : " port %d",
3707 : session_handle, ep->is_ip4, vcl_format_ip46_address,
3708 : &session->transport.lcl_ip,
3709 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3710 : clib_net_to_host_u16 (ep->port));
3711 : }
3712 : else
3713 0 : rv = VPPCOM_EINVAL;
3714 0 : break;
3715 :
3716 284 : case VPPCOM_ATTR_GET_LIBC_EPFD:
3717 284 : rv = session->libc_epfd;
3718 284 : VDBG (2, "VPPCOM_ATTR_GET_LIBC_EPFD: libc_epfd %d", rv);
3719 284 : break;
3720 :
3721 1 : case VPPCOM_ATTR_SET_LIBC_EPFD:
3722 1 : if (PREDICT_TRUE (buffer && buflen &&
3723 : (*buflen == sizeof (session->libc_epfd))))
3724 : {
3725 1 : session->libc_epfd = *(int *) buffer;
3726 1 : *buflen = sizeof (session->libc_epfd);
3727 :
3728 1 : VDBG (2, "VPPCOM_ATTR_SET_LIBC_EPFD: libc_epfd %d, buflen %d",
3729 : session->libc_epfd, *buflen);
3730 : }
3731 : else
3732 0 : rv = VPPCOM_EINVAL;
3733 1 : break;
3734 :
3735 0 : case VPPCOM_ATTR_GET_PROTOCOL:
3736 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3737 : {
3738 0 : *(int *) buffer = session->session_type;
3739 0 : *buflen = sizeof (int);
3740 :
3741 0 : VDBG (2, "VPPCOM_ATTR_GET_PROTOCOL: %d (%s), buflen %d",
3742 : *(int *) buffer, *(int *) buffer ? "UDP" : "TCP", *buflen);
3743 : }
3744 : else
3745 0 : rv = VPPCOM_EINVAL;
3746 0 : break;
3747 :
3748 0 : case VPPCOM_ATTR_GET_LISTEN:
3749 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3750 : {
3751 0 : *(int *) buffer = vcl_session_has_attr (session,
3752 : VCL_SESS_ATTR_LISTEN);
3753 0 : *buflen = sizeof (int);
3754 :
3755 0 : VDBG (2, "VPPCOM_ATTR_GET_LISTEN: %d, buflen %d", *(int *) buffer,
3756 : *buflen);
3757 : }
3758 : else
3759 0 : rv = VPPCOM_EINVAL;
3760 0 : break;
3761 :
3762 0 : case VPPCOM_ATTR_GET_ERROR:
3763 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3764 : {
3765 0 : *(int *) buffer = 0;
3766 0 : *buflen = sizeof (int);
3767 :
3768 0 : VDBG (2, "VPPCOM_ATTR_GET_ERROR: %d, buflen %d, #VPP-TBD#",
3769 : *(int *) buffer, *buflen);
3770 : }
3771 : else
3772 0 : rv = VPPCOM_EINVAL;
3773 0 : break;
3774 :
3775 12 : case VPPCOM_ATTR_GET_TX_FIFO_LEN:
3776 12 : if (buffer && buflen && (*buflen >= sizeof (u32)))
3777 : {
3778 :
3779 : /* VPP-TBD */
3780 24 : *(size_t *) buffer = (session->sndbuf_size ? session->sndbuf_size :
3781 12 : session->tx_fifo ?
3782 4 : svm_fifo_size (session->tx_fifo) :
3783 8 : vcm->cfg.tx_fifo_size);
3784 12 : *buflen = sizeof (u32);
3785 :
3786 12 : VDBG (2, "VPPCOM_ATTR_GET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3787 : " #VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer,
3788 : *buflen);
3789 : }
3790 : else
3791 0 : rv = VPPCOM_EINVAL;
3792 12 : break;
3793 :
3794 0 : case VPPCOM_ATTR_SET_DSCP:
3795 0 : if (buffer && buflen && (*buflen >= sizeof (u8)))
3796 : {
3797 0 : session->dscp = *(u8 *) buffer;
3798 :
3799 0 : VDBG (2, "VPPCOM_ATTR_SET_DSCP: %u (0x%x), buflen %d,",
3800 : *(u8 *) buffer, *(u8 *) buffer, *buflen);
3801 : }
3802 : else
3803 0 : rv = VPPCOM_EINVAL;
3804 0 : break;
3805 :
3806 0 : case VPPCOM_ATTR_SET_TX_FIFO_LEN:
3807 0 : if (buffer && buflen && (*buflen == sizeof (u32)))
3808 : {
3809 : /* VPP-TBD */
3810 0 : session->sndbuf_size = *(u32 *) buffer;
3811 0 : VDBG (2, "VPPCOM_ATTR_SET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3812 : " #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3813 : *buflen);
3814 : }
3815 : else
3816 0 : rv = VPPCOM_EINVAL;
3817 0 : break;
3818 :
3819 12 : case VPPCOM_ATTR_GET_RX_FIFO_LEN:
3820 12 : if (buffer && buflen && (*buflen >= sizeof (u32)))
3821 : {
3822 :
3823 : /* VPP-TBD */
3824 24 : *(size_t *) buffer = (session->rcvbuf_size ? session->rcvbuf_size :
3825 12 : session->rx_fifo ?
3826 4 : svm_fifo_size (session->rx_fifo) :
3827 8 : vcm->cfg.rx_fifo_size);
3828 12 : *buflen = sizeof (u32);
3829 :
3830 12 : VDBG (2, "VPPCOM_ATTR_GET_RX_FIFO_LEN: %u (0x%x), buflen %d, "
3831 : "#VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer, *buflen);
3832 : }
3833 : else
3834 0 : rv = VPPCOM_EINVAL;
3835 12 : break;
3836 :
3837 0 : case VPPCOM_ATTR_SET_RX_FIFO_LEN:
3838 0 : if (buffer && buflen && (*buflen == sizeof (u32)))
3839 : {
3840 : /* VPP-TBD */
3841 0 : session->rcvbuf_size = *(u32 *) buffer;
3842 0 : VDBG (2, "VPPCOM_ATTR_SET_RX_FIFO_LEN: %u (0x%x), buflen %d,"
3843 : " #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3844 : *buflen);
3845 : }
3846 : else
3847 0 : rv = VPPCOM_EINVAL;
3848 0 : break;
3849 :
3850 0 : case VPPCOM_ATTR_GET_REUSEADDR:
3851 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3852 : {
3853 : /* VPP-TBD */
3854 0 : *(int *) buffer = vcl_session_has_attr (session,
3855 : VCL_SESS_ATTR_REUSEADDR);
3856 0 : *buflen = sizeof (int);
3857 :
3858 0 : VDBG (2, "VPPCOM_ATTR_GET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3859 : *(int *) buffer, *buflen);
3860 : }
3861 : else
3862 0 : rv = VPPCOM_EINVAL;
3863 0 : break;
3864 :
3865 15 : case VPPCOM_ATTR_SET_REUSEADDR:
3866 30 : if (buffer && buflen && (*buflen == sizeof (int)) &&
3867 15 : !vcl_session_has_attr (session, VCL_SESS_ATTR_LISTEN))
3868 : {
3869 : /* VPP-TBD */
3870 15 : if (*(int *) buffer)
3871 15 : vcl_session_set_attr (session, VCL_SESS_ATTR_REUSEADDR);
3872 : else
3873 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_REUSEADDR);
3874 :
3875 15 : VDBG (2, "VPPCOM_ATTR_SET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3876 : vcl_session_has_attr (session, VCL_SESS_ATTR_REUSEADDR),
3877 : *buflen);
3878 : }
3879 : else
3880 0 : rv = VPPCOM_EINVAL;
3881 15 : break;
3882 :
3883 0 : case VPPCOM_ATTR_GET_REUSEPORT:
3884 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3885 : {
3886 : /* VPP-TBD */
3887 0 : *(int *) buffer = vcl_session_has_attr (session,
3888 : VCL_SESS_ATTR_REUSEPORT);
3889 0 : *buflen = sizeof (int);
3890 :
3891 0 : VDBG (2, "VPPCOM_ATTR_GET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3892 : *(int *) buffer, *buflen);
3893 : }
3894 : else
3895 0 : rv = VPPCOM_EINVAL;
3896 0 : break;
3897 :
3898 0 : case VPPCOM_ATTR_SET_REUSEPORT:
3899 0 : if (buffer && buflen && (*buflen == sizeof (int)) &&
3900 0 : !vcl_session_has_attr (session, VCL_SESS_ATTR_LISTEN))
3901 : {
3902 : /* VPP-TBD */
3903 0 : if (*(int *) buffer)
3904 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_REUSEPORT);
3905 : else
3906 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_REUSEPORT);
3907 :
3908 0 : VDBG (2, "VPPCOM_ATTR_SET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3909 : vcl_session_has_attr (session, VCL_SESS_ATTR_REUSEPORT),
3910 : *buflen);
3911 : }
3912 : else
3913 0 : rv = VPPCOM_EINVAL;
3914 0 : break;
3915 :
3916 0 : case VPPCOM_ATTR_GET_BROADCAST:
3917 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3918 : {
3919 : /* VPP-TBD */
3920 0 : *(int *) buffer = vcl_session_has_attr (session,
3921 : VCL_SESS_ATTR_BROADCAST);
3922 0 : *buflen = sizeof (int);
3923 :
3924 0 : VDBG (2, "VPPCOM_ATTR_GET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3925 : *(int *) buffer, *buflen);
3926 : }
3927 : else
3928 0 : rv = VPPCOM_EINVAL;
3929 0 : break;
3930 :
3931 0 : case VPPCOM_ATTR_SET_BROADCAST:
3932 0 : if (buffer && buflen && (*buflen == sizeof (int)))
3933 : {
3934 : /* VPP-TBD */
3935 0 : if (*(int *) buffer)
3936 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_BROADCAST);
3937 : else
3938 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_BROADCAST);
3939 :
3940 0 : VDBG (2, "VPPCOM_ATTR_SET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3941 : vcl_session_has_attr (session, VCL_SESS_ATTR_BROADCAST),
3942 : *buflen);
3943 : }
3944 : else
3945 0 : rv = VPPCOM_EINVAL;
3946 0 : break;
3947 :
3948 0 : case VPPCOM_ATTR_GET_V6ONLY:
3949 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3950 : {
3951 : /* VPP-TBD */
3952 0 : *(int *) buffer = vcl_session_has_attr (session,
3953 : VCL_SESS_ATTR_V6ONLY);
3954 0 : *buflen = sizeof (int);
3955 :
3956 0 : VDBG (2, "VPPCOM_ATTR_GET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3957 : *(int *) buffer, *buflen);
3958 : }
3959 : else
3960 0 : rv = VPPCOM_EINVAL;
3961 0 : break;
3962 :
3963 2 : case VPPCOM_ATTR_SET_V6ONLY:
3964 2 : if (buffer && buflen && (*buflen == sizeof (int)))
3965 : {
3966 : /* VPP-TBD */
3967 2 : if (*(int *) buffer)
3968 2 : vcl_session_set_attr (session, VCL_SESS_ATTR_V6ONLY);
3969 : else
3970 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_V6ONLY);
3971 :
3972 2 : VDBG (2, "VPPCOM_ATTR_SET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3973 : vcl_session_has_attr (session, VCL_SESS_ATTR_V6ONLY),
3974 : *buflen);
3975 : }
3976 : else
3977 0 : rv = VPPCOM_EINVAL;
3978 2 : break;
3979 :
3980 0 : case VPPCOM_ATTR_GET_KEEPALIVE:
3981 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3982 : {
3983 : /* VPP-TBD */
3984 0 : *(int *) buffer = vcl_session_has_attr (session,
3985 : VCL_SESS_ATTR_KEEPALIVE);
3986 0 : *buflen = sizeof (int);
3987 :
3988 0 : VDBG (2, "VPPCOM_ATTR_GET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3989 : *(int *) buffer, *buflen);
3990 : }
3991 : else
3992 0 : rv = VPPCOM_EINVAL;
3993 0 : break;
3994 :
3995 0 : case VPPCOM_ATTR_SET_KEEPALIVE:
3996 0 : if (buffer && buflen && (*buflen == sizeof (int)))
3997 : {
3998 : /* VPP-TBD */
3999 0 : if (*(int *) buffer)
4000 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_KEEPALIVE);
4001 : else
4002 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_KEEPALIVE);
4003 :
4004 0 : VDBG (2, "VPPCOM_ATTR_SET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
4005 : vcl_session_has_attr (session, VCL_SESS_ATTR_KEEPALIVE),
4006 : *buflen);
4007 : }
4008 : else
4009 0 : rv = VPPCOM_EINVAL;
4010 0 : break;
4011 :
4012 0 : case VPPCOM_ATTR_GET_TCP_NODELAY:
4013 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
4014 : {
4015 : /* VPP-TBD */
4016 0 : *(int *) buffer = vcl_session_has_attr (session,
4017 : VCL_SESS_ATTR_TCP_NODELAY);
4018 0 : *buflen = sizeof (int);
4019 :
4020 0 : VDBG (2, "VPPCOM_ATTR_GET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
4021 : *(int *) buffer, *buflen);
4022 : }
4023 : else
4024 0 : rv = VPPCOM_EINVAL;
4025 0 : break;
4026 :
4027 0 : case VPPCOM_ATTR_SET_TCP_NODELAY:
4028 0 : if (buffer && buflen && (*buflen == sizeof (int)))
4029 : {
4030 : /* VPP-TBD */
4031 0 : if (*(int *) buffer)
4032 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_NODELAY);
4033 : else
4034 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_NODELAY);
4035 :
4036 0 : VDBG (2, "VPPCOM_ATTR_SET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
4037 : vcl_session_has_attr (session, VCL_SESS_ATTR_TCP_NODELAY),
4038 : *buflen);
4039 : }
4040 : else
4041 0 : rv = VPPCOM_EINVAL;
4042 0 : break;
4043 :
4044 0 : case VPPCOM_ATTR_GET_TCP_KEEPIDLE:
4045 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
4046 : {
4047 : /* VPP-TBD */
4048 0 : *(int *) buffer = vcl_session_has_attr (session,
4049 : VCL_SESS_ATTR_TCP_KEEPIDLE);
4050 0 : *buflen = sizeof (int);
4051 :
4052 0 : VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
4053 : *(int *) buffer, *buflen);
4054 : }
4055 : else
4056 0 : rv = VPPCOM_EINVAL;
4057 0 : break;
4058 :
4059 0 : case VPPCOM_ATTR_SET_TCP_KEEPIDLE:
4060 0 : if (buffer && buflen && (*buflen == sizeof (int)))
4061 : {
4062 : /* VPP-TBD */
4063 0 : if (*(int *) buffer)
4064 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_KEEPIDLE);
4065 : else
4066 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_KEEPIDLE);
4067 :
4068 0 : VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
4069 : vcl_session_has_attr (session,
4070 : VCL_SESS_ATTR_TCP_KEEPIDLE), *buflen);
4071 : }
4072 : else
4073 0 : rv = VPPCOM_EINVAL;
4074 0 : break;
4075 :
4076 0 : case VPPCOM_ATTR_GET_TCP_KEEPINTVL:
4077 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
4078 : {
4079 : /* VPP-TBD */
4080 0 : *(int *) buffer = vcl_session_has_attr (session,
4081 : VCL_SESS_ATTR_TCP_KEEPINTVL);
4082 0 : *buflen = sizeof (int);
4083 :
4084 0 : VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
4085 : *(int *) buffer, *buflen);
4086 : }
4087 : else
4088 0 : rv = VPPCOM_EINVAL;
4089 0 : break;
4090 :
4091 0 : case VPPCOM_ATTR_SET_TCP_KEEPINTVL:
4092 0 : if (buffer && buflen && (*buflen == sizeof (int)))
4093 : {
4094 : /* VPP-TBD */
4095 0 : if (*(int *) buffer)
4096 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_KEEPINTVL);
4097 : else
4098 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_KEEPINTVL);
4099 :
4100 0 : VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
4101 : vcl_session_has_attr (session,
4102 : VCL_SESS_ATTR_TCP_KEEPINTVL), *buflen);
4103 : }
4104 : else
4105 0 : rv = VPPCOM_EINVAL;
4106 0 : break;
4107 :
4108 5 : case VPPCOM_ATTR_GET_TCP_USER_MSS:
4109 5 : if (!(buffer && buflen && (*buflen >= sizeof (u32))))
4110 : {
4111 0 : rv = VPPCOM_EINVAL;
4112 0 : break;
4113 : }
4114 :
4115 5 : tea.type = TRANSPORT_ENDPT_ATTR_MSS;
4116 5 : tea.mss = *(u32 *) buffer;
4117 5 : if (vcl_session_transport_attr (wrk, session, 1 /* is_get */, &tea))
4118 2 : rv = VPPCOM_ENOPROTOOPT;
4119 :
4120 5 : if (!rv)
4121 : {
4122 3 : *(u32 *) buffer = tea.mss;
4123 3 : *buflen = sizeof (int);
4124 : }
4125 :
4126 5 : VDBG (2, "VPPCOM_ATTR_GET_TCP_USER_MSS: %d, buflen %d", *(int *) buffer,
4127 : *buflen);
4128 5 : break;
4129 2 : case VPPCOM_ATTR_SET_TCP_USER_MSS:
4130 2 : if (!(buffer && buflen && (*buflen == sizeof (u32))))
4131 : {
4132 0 : rv = VPPCOM_EINVAL;
4133 0 : break;
4134 : }
4135 :
4136 2 : tea.type = TRANSPORT_ENDPT_ATTR_MSS;
4137 2 : tea.mss = *(u32 *) buffer;
4138 2 : if (vcl_session_transport_attr (wrk, session, 0 /* is_get */, &tea))
4139 0 : rv = VPPCOM_ENOPROTOOPT;
4140 :
4141 2 : VDBG (2, "VPPCOM_ATTR_SET_TCP_USER_MSS: %u, buflen %d", tea.mss,
4142 : *buflen);
4143 2 : break;
4144 :
4145 0 : case VPPCOM_ATTR_SET_CONNECTED:
4146 0 : session->flags |= VCL_SESSION_F_CONNECTED;
4147 0 : break;
4148 :
4149 6 : case VPPCOM_ATTR_SET_CKPAIR:
4150 12 : if (!(buffer && buflen && (*buflen == sizeof (int))) ||
4151 6 : !vcl_session_has_crypto (session))
4152 : {
4153 0 : rv = VPPCOM_EINVAL;
4154 0 : break;
4155 : }
4156 6 : if (!session->ext_config)
4157 : {
4158 6 : vcl_session_alloc_ext_cfg (session, TRANSPORT_ENDPT_EXT_CFG_CRYPTO,
4159 : sizeof (transport_endpt_ext_cfg_t));
4160 : }
4161 0 : else if (session->ext_config->type != TRANSPORT_ENDPT_EXT_CFG_CRYPTO)
4162 : {
4163 0 : rv = VPPCOM_EINVAL;
4164 0 : break;
4165 : }
4166 :
4167 6 : session->ext_config->crypto.ckpair_index = *(uint32_t *) buffer;
4168 6 : break;
4169 :
4170 0 : case VPPCOM_ATTR_SET_VRF:
4171 0 : if (!(buffer && buflen && (*buflen == sizeof (u32))))
4172 : {
4173 0 : rv = VPPCOM_EINVAL;
4174 0 : break;
4175 : }
4176 0 : session->vrf = *(u32 *) buffer;
4177 0 : break;
4178 :
4179 0 : case VPPCOM_ATTR_GET_VRF:
4180 0 : if (!(buffer && buflen && (*buflen >= sizeof (u32))))
4181 : {
4182 0 : rv = VPPCOM_EINVAL;
4183 0 : break;
4184 : }
4185 0 : *(u32 *) buffer = session->vrf;
4186 0 : *buflen = sizeof (u32);
4187 0 : break;
4188 :
4189 0 : case VPPCOM_ATTR_GET_DOMAIN:
4190 0 : if (!(buffer && buflen && (*buflen >= sizeof (int))))
4191 : {
4192 0 : rv = VPPCOM_EINVAL;
4193 0 : break;
4194 : }
4195 :
4196 0 : if (session->transport.is_ip4)
4197 0 : *(int *) buffer = AF_INET;
4198 : else
4199 0 : *(int *) buffer = AF_INET6;
4200 0 : *buflen = sizeof (int);
4201 :
4202 0 : VDBG (2, "VPPCOM_ATTR_GET_DOMAIN: %d, buflen %u", *(int *) buffer,
4203 : *buflen);
4204 0 : break;
4205 :
4206 0 : case VPPCOM_ATTR_SET_ENDPT_EXT_CFG:
4207 0 : if (!(buffer && buflen && (*buflen > 0)))
4208 : {
4209 0 : rv = VPPCOM_EINVAL;
4210 0 : break;
4211 : }
4212 0 : if (session->ext_config)
4213 : {
4214 0 : rv = VPPCOM_EINVAL;
4215 0 : break;
4216 : }
4217 0 : vcl_session_alloc_ext_cfg (session, TRANSPORT_ENDPT_EXT_CFG_NONE,
4218 0 : *buflen + sizeof (u32));
4219 0 : clib_memcpy (session->ext_config->data, buffer, *buflen);
4220 0 : session->ext_config->len = *buflen;
4221 0 : break;
4222 0 : case VPPCOM_ATTR_SET_IP_PKTINFO:
4223 0 : if (buffer && buflen && (*buflen == sizeof (int)) &&
4224 0 : !vcl_session_has_attr (session, VCL_SESS_ATTR_IP_PKTINFO))
4225 : {
4226 0 : if (*(int *) buffer)
4227 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_IP_PKTINFO);
4228 : else
4229 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_IP_PKTINFO);
4230 :
4231 0 : VDBG (2, "VCL_SESS_ATTR_IP_PKTINFO: %d, buflen %d",
4232 : vcl_session_has_attr (session, VCL_SESS_ATTR_IP_PKTINFO),
4233 : *buflen);
4234 : }
4235 : else
4236 0 : rv = VPPCOM_EINVAL;
4237 0 : break;
4238 :
4239 0 : case VPPCOM_ATTR_GET_IP_PKTINFO:
4240 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
4241 : {
4242 0 : *(int *) buffer =
4243 0 : vcl_session_has_attr (session, VCL_SESS_ATTR_IP_PKTINFO);
4244 0 : *buflen = sizeof (int);
4245 :
4246 0 : VDBG (2, "VCL_SESS_ATTR_IP_PKTINFO: %d, buflen %d", *(int *) buffer,
4247 : *buflen);
4248 : }
4249 : else
4250 0 : rv = VPPCOM_EINVAL;
4251 0 : break;
4252 :
4253 0 : default:
4254 0 : rv = VPPCOM_EINVAL;
4255 0 : break;
4256 : }
4257 :
4258 6402 : return rv;
4259 : }
4260 :
4261 : int
4262 4 : vppcom_session_recvfrom (uint32_t session_handle, void *buffer,
4263 : uint32_t buflen, int flags, vppcom_endpt_t * ep)
4264 : {
4265 4 : vcl_worker_t *wrk = vcl_worker_get_current ();
4266 : vcl_session_t *session;
4267 4 : int rv = VPPCOM_OK;
4268 :
4269 4 : if (flags == 0)
4270 4 : rv = vppcom_session_read (session_handle, buffer, buflen);
4271 0 : else if (flags & MSG_PEEK)
4272 0 : rv = vppcom_session_peek (session_handle, buffer, buflen);
4273 : else
4274 : {
4275 0 : VDBG (0, "Unsupport flags for recvfrom %d", flags);
4276 0 : return VPPCOM_EAFNOSUPPORT;
4277 : }
4278 :
4279 4 : if (ep && rv > 0)
4280 : {
4281 2 : session = vcl_session_get_w_handle (wrk, session_handle);
4282 2 : if (session->transport.is_ip4)
4283 2 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
4284 : sizeof (ip4_address_t));
4285 : else
4286 0 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
4287 : sizeof (ip6_address_t));
4288 2 : ep->is_ip4 = session->transport.is_ip4;
4289 2 : ep->port = session->transport.rmt_port;
4290 : }
4291 :
4292 4 : return rv;
4293 : }
4294 :
4295 : static void
4296 0 : vcl_handle_ep_app_tlvs (vcl_session_t *s, vppcom_endpt_t *ep)
4297 : {
4298 0 : vppcom_endpt_tlv_t *tlv = ep->app_tlvs;
4299 :
4300 : do
4301 : {
4302 0 : switch (tlv->data_type)
4303 : {
4304 0 : case VCL_UDP_SEGMENT:
4305 0 : s->gso_size = *(u16 *) tlv->data;
4306 0 : break;
4307 0 : case VCL_IP_PKTINFO:
4308 0 : clib_memcpy_fast (&s->transport.lcl_ip, (ip4_address_t *) tlv->data,
4309 : sizeof (ip4_address_t));
4310 0 : break;
4311 0 : default:
4312 0 : VDBG (0, "Ignorning unsupported app tlv %u", tlv->data_type);
4313 0 : break;
4314 : }
4315 0 : tlv = VCL_EP_NEXT_APP_TLV (ep, tlv);
4316 : }
4317 0 : while (tlv);
4318 0 : }
4319 :
4320 : int
4321 0 : vppcom_session_sendto (uint32_t session_handle, void *buffer,
4322 : uint32_t buflen, int flags, vppcom_endpt_t * ep)
4323 : {
4324 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4325 : vcl_session_t *s;
4326 :
4327 0 : s = vcl_session_get_w_handle (wrk, session_handle);
4328 0 : if (PREDICT_FALSE (!s))
4329 0 : return VPPCOM_EBADFD;
4330 :
4331 0 : if (ep)
4332 : {
4333 0 : if (!vcl_session_is_cl (s))
4334 0 : return VPPCOM_EINVAL;
4335 :
4336 0 : s->transport.is_ip4 = ep->is_ip4;
4337 0 : s->transport.rmt_port = ep->port;
4338 0 : vcl_ip_copy_from_ep (&s->transport.rmt_ip, ep);
4339 :
4340 0 : if (ep->app_tlvs)
4341 0 : vcl_handle_ep_app_tlvs (s, ep);
4342 :
4343 : /* Session not connected/bound in vpp. Create it by 'connecting' it */
4344 0 : if (PREDICT_FALSE (s->session_state == VCL_STATE_CLOSED))
4345 : {
4346 0 : u32 session_index = s->session_index;
4347 0 : f64 timeout = vcm->cfg.session_timeout;
4348 : int rv;
4349 :
4350 0 : vcl_send_session_connect (wrk, s);
4351 0 : rv = vppcom_wait_for_session_state_change (session_index,
4352 : VCL_STATE_READY,
4353 : timeout);
4354 0 : if (rv < 0)
4355 0 : return rv;
4356 0 : s = vcl_session_get (wrk, session_index);
4357 : }
4358 : }
4359 :
4360 0 : if (flags)
4361 : {
4362 : // TBD check the flags and do the right thing
4363 0 : VDBG (2, "handling flags 0x%u (%d) not implemented yet.", flags, flags);
4364 : }
4365 :
4366 0 : return (vppcom_session_write_inline (wrk, s, buffer, buflen, 1,
4367 0 : s->is_dgram ? 1 : 0));
4368 : }
4369 :
4370 : int
4371 0 : vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
4372 : {
4373 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4374 0 : f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
4375 0 : u32 i, keep_trying = 1;
4376 : svm_msg_q_msg_t msg;
4377 : session_event_t *e;
4378 0 : int rv, num_ev = 0;
4379 :
4380 0 : VDBG (3, "vp %p, nsids %u, wait_for_time %f", vp, n_sids, wait_for_time);
4381 :
4382 0 : if (!vp)
4383 0 : return VPPCOM_EFAULT;
4384 :
4385 : do
4386 : {
4387 : vcl_session_t *session;
4388 :
4389 : /* Dequeue all events and drop all unhandled io events */
4390 0 : while (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0) == 0)
4391 : {
4392 0 : e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
4393 0 : vcl_handle_mq_event (wrk, e);
4394 0 : svm_msg_q_free_msg (wrk->app_event_queue, &msg);
4395 : }
4396 0 : vec_reset_length (wrk->unhandled_evts_vector);
4397 :
4398 0 : for (i = 0; i < n_sids; i++)
4399 : {
4400 0 : session = vcl_session_get (wrk, vp[i].sh);
4401 0 : if (!session)
4402 : {
4403 0 : vp[i].revents = POLLHUP;
4404 0 : num_ev++;
4405 0 : continue;
4406 : }
4407 :
4408 0 : vp[i].revents = 0;
4409 :
4410 0 : if (POLLIN & vp[i].events)
4411 : {
4412 0 : rv = vcl_session_read_ready (session);
4413 0 : if (rv > 0)
4414 : {
4415 0 : vp[i].revents |= POLLIN;
4416 0 : num_ev++;
4417 : }
4418 0 : else if (rv < 0)
4419 : {
4420 0 : switch (rv)
4421 : {
4422 0 : case VPPCOM_ECONNRESET:
4423 0 : vp[i].revents = POLLHUP;
4424 0 : break;
4425 :
4426 0 : default:
4427 0 : vp[i].revents = POLLERR;
4428 0 : break;
4429 : }
4430 0 : num_ev++;
4431 : }
4432 : }
4433 :
4434 0 : if (POLLOUT & vp[i].events)
4435 : {
4436 0 : rv = vcl_session_write_ready (session);
4437 0 : if (rv > 0)
4438 : {
4439 0 : vp[i].revents |= POLLOUT;
4440 0 : num_ev++;
4441 : }
4442 0 : else if (rv < 0)
4443 : {
4444 0 : switch (rv)
4445 : {
4446 0 : case VPPCOM_ECONNRESET:
4447 0 : vp[i].revents = POLLHUP;
4448 0 : break;
4449 :
4450 0 : default:
4451 0 : vp[i].revents = POLLERR;
4452 0 : break;
4453 : }
4454 0 : num_ev++;
4455 : }
4456 : }
4457 :
4458 : if (0) // Note "done:" label used by VCL_SESSION_LOCK_AND_GET()
4459 : {
4460 : vp[i].revents = POLLNVAL;
4461 : num_ev++;
4462 : }
4463 : }
4464 0 : if (wait_for_time != -1)
4465 0 : keep_trying = (clib_time_now (&wrk->clib_time) <= timeout) ? 1 : 0;
4466 : }
4467 0 : while ((num_ev == 0) && keep_trying);
4468 :
4469 0 : return num_ev;
4470 : }
4471 :
4472 : int
4473 0 : vppcom_mq_epoll_fd (void)
4474 : {
4475 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4476 0 : return wrk->mqs_epfd;
4477 : }
4478 :
4479 : int
4480 1620060 : vppcom_session_index (vcl_session_handle_t session_handle)
4481 : {
4482 1620060 : return session_handle & 0xFFFFFF;
4483 : }
4484 :
4485 : int
4486 47 : vppcom_session_worker (vcl_session_handle_t session_handle)
4487 : {
4488 47 : return session_handle >> 24;
4489 : }
4490 :
4491 : int
4492 0 : vppcom_worker_register (void)
4493 : {
4494 0 : if (!vcl_worker_alloc_and_init ())
4495 0 : return VPPCOM_EEXIST;
4496 :
4497 0 : if (vcl_worker_register_with_vpp ())
4498 0 : return VPPCOM_EEXIST;
4499 :
4500 0 : return VPPCOM_OK;
4501 : }
4502 :
4503 : void
4504 0 : vppcom_worker_unregister (void)
4505 : {
4506 0 : vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
4507 0 : vcl_set_worker_index (~0);
4508 0 : }
4509 :
4510 : void
4511 0 : vppcom_worker_index_set (int index)
4512 : {
4513 0 : vcl_set_worker_index (index);
4514 0 : }
4515 :
4516 : int
4517 470513 : vppcom_worker_index (void)
4518 : {
4519 470513 : return vcl_get_worker_index ();
4520 : }
4521 :
4522 : int
4523 0 : vppcom_worker_mqs_epfd (void)
4524 : {
4525 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4526 0 : if (!vcm->cfg.use_mq_eventfd)
4527 0 : return -1;
4528 0 : return wrk->mqs_epfd;
4529 : }
4530 :
4531 : int
4532 4540 : vppcom_session_is_connectable_listener (uint32_t session_handle)
4533 : {
4534 : vcl_session_t *session;
4535 4540 : vcl_worker_t *wrk = vcl_worker_get_current ();
4536 4540 : session = vcl_session_get_w_handle (wrk, session_handle);
4537 4540 : if (!session)
4538 0 : return VPPCOM_EBADFD;
4539 4540 : return vcl_session_is_connectable_listener (wrk, session);
4540 : }
4541 :
4542 : int
4543 0 : vppcom_session_listener (uint32_t session_handle)
4544 : {
4545 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4546 : vcl_session_t *listen_session, *session;
4547 0 : session = vcl_session_get_w_handle (wrk, session_handle);
4548 0 : if (!session)
4549 0 : return VPPCOM_EBADFD;
4550 0 : if (session->listener_index == VCL_INVALID_SESSION_INDEX)
4551 0 : return VPPCOM_EBADFD;
4552 0 : listen_session = vcl_session_get_w_handle (wrk, session->listener_index);
4553 0 : if (!listen_session)
4554 0 : return VPPCOM_EBADFD;
4555 0 : return vcl_session_handle (listen_session);
4556 : }
4557 :
4558 : int
4559 0 : vppcom_session_n_accepted (uint32_t session_handle)
4560 : {
4561 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4562 0 : vcl_session_t *session = vcl_session_get_w_handle (wrk, session_handle);
4563 0 : if (!session)
4564 0 : return VPPCOM_EBADFD;
4565 0 : return session->n_accepted_sessions;
4566 : }
4567 :
4568 : const char *
4569 122 : vppcom_proto_str (vppcom_proto_t proto)
4570 : {
4571 : char const *proto_str;
4572 :
4573 122 : switch (proto)
4574 : {
4575 104 : case VPPCOM_PROTO_TCP:
4576 104 : proto_str = "TCP";
4577 104 : break;
4578 12 : case VPPCOM_PROTO_UDP:
4579 12 : proto_str = "UDP";
4580 12 : break;
4581 4 : case VPPCOM_PROTO_TLS:
4582 4 : proto_str = "TLS";
4583 4 : break;
4584 0 : case VPPCOM_PROTO_QUIC:
4585 0 : proto_str = "QUIC";
4586 0 : break;
4587 2 : case VPPCOM_PROTO_DTLS:
4588 2 : proto_str = "DTLS";
4589 2 : break;
4590 0 : case VPPCOM_PROTO_SRTP:
4591 0 : proto_str = "SRTP";
4592 0 : break;
4593 0 : default:
4594 0 : proto_str = "UNKNOWN";
4595 0 : break;
4596 : }
4597 122 : return proto_str;
4598 : }
4599 :
4600 : const char *
4601 0 : vppcom_retval_str (int retval)
4602 : {
4603 : char const *st;
4604 :
4605 0 : switch (retval)
4606 : {
4607 0 : case VPPCOM_OK:
4608 0 : st = "VPPCOM_OK";
4609 0 : break;
4610 :
4611 0 : case VPPCOM_EAGAIN:
4612 0 : st = "VPPCOM_EAGAIN";
4613 0 : break;
4614 :
4615 0 : case VPPCOM_EFAULT:
4616 0 : st = "VPPCOM_EFAULT";
4617 0 : break;
4618 :
4619 0 : case VPPCOM_ENOMEM:
4620 0 : st = "VPPCOM_ENOMEM";
4621 0 : break;
4622 :
4623 0 : case VPPCOM_EINVAL:
4624 0 : st = "VPPCOM_EINVAL";
4625 0 : break;
4626 :
4627 0 : case VPPCOM_EBADFD:
4628 0 : st = "VPPCOM_EBADFD";
4629 0 : break;
4630 :
4631 0 : case VPPCOM_EAFNOSUPPORT:
4632 0 : st = "VPPCOM_EAFNOSUPPORT";
4633 0 : break;
4634 :
4635 0 : case VPPCOM_ECONNABORTED:
4636 0 : st = "VPPCOM_ECONNABORTED";
4637 0 : break;
4638 :
4639 0 : case VPPCOM_ECONNRESET:
4640 0 : st = "VPPCOM_ECONNRESET";
4641 0 : break;
4642 :
4643 0 : case VPPCOM_ENOTCONN:
4644 0 : st = "VPPCOM_ENOTCONN";
4645 0 : break;
4646 :
4647 0 : case VPPCOM_ECONNREFUSED:
4648 0 : st = "VPPCOM_ECONNREFUSED";
4649 0 : break;
4650 :
4651 0 : case VPPCOM_ETIMEDOUT:
4652 0 : st = "VPPCOM_ETIMEDOUT";
4653 0 : break;
4654 :
4655 0 : case VPPCOM_EADDRINUSE:
4656 0 : st = "VPPCOM_EADDRINUSE";
4657 0 : break;
4658 :
4659 0 : default:
4660 0 : st = "UNKNOWN_STATE";
4661 0 : break;
4662 : }
4663 :
4664 0 : return st;
4665 : }
4666 :
4667 : int
4668 6 : vppcom_add_cert_key_pair (vppcom_cert_key_pair_t *ckpair)
4669 : {
4670 6 : if (vcm->cfg.vpp_app_socket_api)
4671 4 : return vcl_sapi_add_cert_key_pair (ckpair);
4672 : else
4673 2 : return vcl_bapi_add_cert_key_pair (ckpair);
4674 : }
4675 :
4676 : int
4677 0 : vppcom_del_cert_key_pair (uint32_t ckpair_index)
4678 : {
4679 0 : if (vcm->cfg.vpp_app_socket_api)
4680 0 : return vcl_sapi_del_cert_key_pair (ckpair_index);
4681 : else
4682 0 : return vcl_bapi_del_cert_key_pair (ckpair_index);
4683 : }
4684 :
4685 : int
4686 0 : vppcom_session_get_error (uint32_t session_handle)
4687 : {
4688 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4689 0 : vcl_session_t *session = 0;
4690 :
4691 0 : session = vcl_session_get_w_handle (wrk, session_handle);
4692 0 : if (!session)
4693 0 : return VPPCOM_EBADFD;
4694 :
4695 0 : if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
4696 : {
4697 0 : VWRN ("epoll session %u! will not have connect", session->session_index);
4698 0 : return VPPCOM_EBADFD;
4699 : }
4700 :
4701 0 : if (session->vpp_error == SESSION_E_PORTINUSE)
4702 0 : return VPPCOM_EADDRINUSE;
4703 0 : else if (session->vpp_error == SESSION_E_REFUSED)
4704 0 : return VPPCOM_ECONNREFUSED;
4705 0 : else if (session->vpp_error != SESSION_E_NONE)
4706 0 : return VPPCOM_EFAULT;
4707 : else
4708 0 : return VPPCOM_OK;
4709 : }
4710 :
4711 : int
4712 0 : vppcom_worker_is_detached (void)
4713 : {
4714 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4715 :
4716 0 : if (!vcm->cfg.use_mq_eventfd)
4717 0 : return VPPCOM_ENOTSUP;
4718 :
4719 0 : return wrk->api_client_handle == ~0;
4720 : }
4721 :
4722 : /*
4723 : * fd.io coding-style-patch-verification: ON
4724 : *
4725 : * Local Variables:
4726 : * eval: (c-set-style "gnu")
4727 : * End:
4728 : */
|
