Line data Source code
1 : /*
2 : * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #include <stdio.h>
17 : #include <stdlib.h>
18 : #include <vcl/vppcom.h>
19 : #include <vcl/vcl_debug.h>
20 : #include <vcl/vcl_private.h>
21 : #include <svm/fifo_segment.h>
22 :
23 : __thread uword __vcl_worker_index = ~0;
24 :
25 : static inline int
26 111766 : vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq, u32 n_max_msg)
27 : {
28 111766 : u32 n_msgs = 0, sz, len;
29 :
30 222332 : while ((sz = svm_msg_q_size (mq)))
31 : {
32 110566 : len = vec_len (wrk->mq_msg_vector);
33 110566 : vec_validate (wrk->mq_msg_vector, len + sz - 1);
34 110566 : svm_msg_q_sub_raw_batch (mq, wrk->mq_msg_vector + len, sz);
35 110566 : n_msgs += sz;
36 : }
37 111766 : return n_msgs;
38 : }
39 :
40 :
41 :
42 : static void
43 6 : vcl_msg_add_ext_config (vcl_session_t *s, uword *offset)
44 : {
45 : svm_fifo_chunk_t *c;
46 :
47 6 : c = vcl_segment_alloc_chunk (vcl_vpp_worker_segment_handle (0),
48 6 : 0 /* one slice only */, s->ext_config->len,
49 : offset);
50 6 : if (c)
51 6 : clib_memcpy_fast (c->data, s->ext_config, s->ext_config->len);
52 6 : }
53 :
54 : void
55 40 : vcl_send_session_listen (vcl_worker_t *wrk, vcl_session_t *s)
56 : {
57 40 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
58 : session_listen_msg_t *mp;
59 : svm_msg_q_t *mq;
60 :
61 40 : mq = vcl_worker_ctrl_mq (wrk);
62 40 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_LISTEN);
63 40 : mp = (session_listen_msg_t *) app_evt->evt->data;
64 40 : memset (mp, 0, sizeof (*mp));
65 40 : mp->client_index = wrk->api_client_handle;
66 40 : mp->context = s->session_index;
67 40 : mp->wrk_index = wrk->vpp_wrk_index;
68 40 : mp->is_ip4 = s->transport.is_ip4;
69 40 : clib_memcpy_fast (&mp->ip, &s->transport.lcl_ip, sizeof (mp->ip));
70 40 : mp->port = s->transport.lcl_port;
71 40 : mp->proto = s->session_type;
72 40 : mp->vrf = s->vrf;
73 40 : if (s->flags & VCL_SESSION_F_CONNECTED)
74 0 : mp->flags = TRANSPORT_CFG_F_CONNECTED;
75 40 : if (s->ext_config)
76 3 : vcl_msg_add_ext_config (s, &mp->ext_config);
77 40 : app_send_ctrl_evt_to_vpp (mq, app_evt);
78 40 : if (s->ext_config)
79 : {
80 3 : clib_mem_free (s->ext_config);
81 3 : s->ext_config = 0;
82 : }
83 40 : s->flags |= VCL_SESSION_F_PENDING_LISTEN;
84 40 : }
85 :
86 : static void
87 47 : vcl_send_session_connect (vcl_worker_t * wrk, vcl_session_t * s)
88 : {
89 47 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
90 : session_connect_msg_t *mp;
91 : svm_msg_q_t *mq;
92 :
93 47 : mq = vcl_worker_ctrl_mq (wrk);
94 47 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_CONNECT);
95 47 : mp = (session_connect_msg_t *) app_evt->evt->data;
96 47 : memset (mp, 0, sizeof (*mp));
97 47 : mp->client_index = wrk->api_client_handle;
98 47 : mp->context = s->session_index;
99 47 : mp->dscp = s->dscp;
100 47 : mp->wrk_index = wrk->vpp_wrk_index;
101 47 : mp->is_ip4 = s->transport.is_ip4;
102 47 : mp->parent_handle = s->parent_handle;
103 47 : clib_memcpy_fast (&mp->ip, &s->transport.rmt_ip, sizeof (mp->ip));
104 47 : clib_memcpy_fast (&mp->lcl_ip, &s->transport.lcl_ip, sizeof (mp->lcl_ip));
105 47 : mp->port = s->transport.rmt_port;
106 47 : mp->lcl_port = s->transport.lcl_port;
107 47 : mp->proto = s->session_type;
108 47 : mp->vrf = s->vrf;
109 47 : if (s->flags & VCL_SESSION_F_CONNECTED)
110 47 : mp->flags |= TRANSPORT_CFG_F_CONNECTED;
111 47 : if (s->ext_config)
112 3 : vcl_msg_add_ext_config (s, &mp->ext_config);
113 47 : app_send_ctrl_evt_to_vpp (mq, app_evt);
114 :
115 47 : if (s->ext_config)
116 : {
117 3 : clib_mem_free (s->ext_config);
118 3 : s->ext_config = 0;
119 : }
120 47 : }
121 :
122 : void
123 23 : vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s)
124 : {
125 23 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
126 : session_unlisten_msg_t *mp;
127 : svm_msg_q_t *mq;
128 :
129 23 : mq = vcl_worker_ctrl_mq (wrk);
130 23 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_UNLISTEN);
131 23 : mp = (session_unlisten_msg_t *) app_evt->evt->data;
132 23 : memset (mp, 0, sizeof (*mp));
133 23 : mp->client_index = wrk->api_client_handle;
134 23 : mp->wrk_index = wrk->vpp_wrk_index;
135 23 : mp->handle = s->vpp_handle;
136 23 : mp->context = wrk->wrk_index;
137 23 : app_send_ctrl_evt_to_vpp (mq, app_evt);
138 23 : }
139 :
140 : static void
141 0 : vcl_send_session_shutdown (vcl_worker_t *wrk, vcl_session_t *s)
142 : {
143 0 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
144 : session_shutdown_msg_t *mp;
145 : svm_msg_q_t *mq;
146 :
147 : /* Send to thread that owns the session */
148 0 : mq = s->vpp_evt_q;
149 0 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_SHUTDOWN);
150 0 : mp = (session_shutdown_msg_t *) app_evt->evt->data;
151 0 : memset (mp, 0, sizeof (*mp));
152 0 : mp->client_index = wrk->api_client_handle;
153 0 : mp->handle = s->vpp_handle;
154 0 : app_send_ctrl_evt_to_vpp (mq, app_evt);
155 0 : }
156 :
157 : static void
158 56 : vcl_send_session_disconnect (vcl_worker_t * wrk, vcl_session_t * s)
159 : {
160 56 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
161 : session_disconnect_msg_t *mp;
162 : svm_msg_q_t *mq;
163 :
164 : /* Send to thread that owns the session */
165 56 : mq = s->vpp_evt_q;
166 56 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_DISCONNECT);
167 56 : mp = (session_disconnect_msg_t *) app_evt->evt->data;
168 56 : memset (mp, 0, sizeof (*mp));
169 56 : mp->client_index = wrk->api_client_handle;
170 56 : mp->handle = s->vpp_handle;
171 56 : app_send_ctrl_evt_to_vpp (mq, app_evt);
172 56 : }
173 :
174 : static void
175 24 : vcl_send_app_detach (vcl_worker_t * wrk)
176 : {
177 24 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
178 : session_app_detach_msg_t *mp;
179 : svm_msg_q_t *mq;
180 :
181 24 : mq = vcl_worker_ctrl_mq (wrk);
182 24 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_DETACH);
183 24 : mp = (session_app_detach_msg_t *) app_evt->evt->data;
184 24 : memset (mp, 0, sizeof (*mp));
185 24 : mp->client_index = wrk->api_client_handle;
186 24 : app_send_ctrl_evt_to_vpp (mq, app_evt);
187 24 : }
188 :
189 : static void
190 37 : vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context,
191 : session_handle_t handle, int retval)
192 : {
193 37 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
194 : session_accepted_reply_msg_t *rmp;
195 37 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_ACCEPTED_REPLY);
196 37 : rmp = (session_accepted_reply_msg_t *) app_evt->evt->data;
197 37 : rmp->handle = handle;
198 37 : rmp->context = context;
199 37 : rmp->retval = retval;
200 37 : app_send_ctrl_evt_to_vpp (mq, app_evt);
201 37 : }
202 :
203 : static void
204 6 : vcl_send_session_disconnected_reply (vcl_worker_t * wrk, vcl_session_t * s,
205 : int retval)
206 : {
207 6 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
208 : session_disconnected_reply_msg_t *rmp;
209 6 : app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
210 : SESSION_CTRL_EVT_DISCONNECTED_REPLY);
211 6 : rmp = (session_disconnected_reply_msg_t *) app_evt->evt->data;
212 6 : rmp->handle = s->vpp_handle;
213 6 : rmp->context = wrk->api_client_handle;
214 6 : rmp->retval = retval;
215 6 : app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
216 6 : }
217 :
218 : static void
219 4 : vcl_send_session_reset_reply (vcl_worker_t * wrk, vcl_session_t * s,
220 : int retval)
221 : {
222 4 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
223 : session_reset_reply_msg_t *rmp;
224 4 : app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
225 : SESSION_CTRL_EVT_RESET_REPLY);
226 4 : rmp = (session_reset_reply_msg_t *) app_evt->evt->data;
227 4 : rmp->handle = s->vpp_handle;
228 4 : rmp->context = wrk->api_client_handle;
229 4 : rmp->retval = retval;
230 4 : app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
231 4 : }
232 :
233 : void
234 0 : vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
235 : u32 wrk_index)
236 : {
237 0 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
238 : session_worker_update_msg_t *mp;
239 :
240 0 : app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
241 : SESSION_CTRL_EVT_WORKER_UPDATE);
242 0 : mp = (session_worker_update_msg_t *) app_evt->evt->data;
243 0 : mp->client_index = wrk->api_client_handle;
244 0 : mp->handle = s->vpp_handle;
245 0 : mp->req_wrk_index = wrk->vpp_wrk_index;
246 0 : mp->wrk_index = wrk_index;
247 0 : app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
248 0 : }
249 :
250 : int
251 0 : vcl_send_worker_rpc (u32 dst_wrk_index, void *data, u32 data_len)
252 : {
253 0 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
254 : session_app_wrk_rpc_msg_t *mp;
255 : vcl_worker_t *dst_wrk, *wrk;
256 : svm_msg_q_t *mq;
257 0 : int ret = -1;
258 :
259 0 : if (data_len > sizeof (mp->data))
260 0 : goto done;
261 :
262 0 : clib_spinlock_lock (&vcm->workers_lock);
263 :
264 0 : dst_wrk = vcl_worker_get_if_valid (dst_wrk_index);
265 0 : if (!dst_wrk)
266 0 : goto done;
267 :
268 0 : wrk = vcl_worker_get_current ();
269 0 : mq = vcl_worker_ctrl_mq (wrk);
270 0 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_WRK_RPC);
271 0 : mp = (session_app_wrk_rpc_msg_t *) app_evt->evt->data;
272 0 : mp->client_index = wrk->api_client_handle;
273 0 : mp->wrk_index = dst_wrk->vpp_wrk_index;
274 0 : clib_memcpy (mp->data, data, data_len);
275 0 : app_send_ctrl_evt_to_vpp (mq, app_evt);
276 0 : ret = 0;
277 :
278 0 : done:
279 0 : clib_spinlock_unlock (&vcm->workers_lock);
280 0 : return ret;
281 : }
282 :
283 : int
284 7 : vcl_session_transport_attr (vcl_worker_t *wrk, vcl_session_t *s, u8 is_get,
285 : transport_endpt_attr_t *attr)
286 : {
287 7 : app_session_evt_t _app_evt, *app_evt = &_app_evt;
288 : session_transport_attr_msg_t *mp;
289 : svm_msg_q_t *mq;
290 : f64 timeout;
291 :
292 7 : ASSERT (!wrk->session_attr_op);
293 7 : mq = s->vpp_evt_q;
294 7 : if (PREDICT_FALSE (!mq))
295 : {
296 : /* FIXME: attribute should be stored and sent once session is
297 : * bound/connected to vpp */
298 2 : return 0;
299 : }
300 :
301 5 : wrk->session_attr_op = 1;
302 5 : wrk->session_attr_op_rv = -1;
303 :
304 5 : app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_TRANSPORT_ATTR);
305 5 : mp = (session_transport_attr_msg_t *) app_evt->evt->data;
306 5 : memset (mp, 0, sizeof (*mp));
307 5 : mp->client_index = wrk->api_client_handle;
308 5 : mp->handle = s->vpp_handle;
309 5 : mp->is_get = is_get;
310 5 : mp->attr = *attr;
311 5 : app_send_ctrl_evt_to_vpp (mq, app_evt);
312 :
313 5 : timeout = clib_time_now (&wrk->clib_time) + 1;
314 :
315 1355 : while (wrk->session_attr_op && clib_time_now (&wrk->clib_time) < timeout)
316 1350 : vcl_flush_mq_events ();
317 :
318 5 : if (!wrk->session_attr_op_rv && is_get)
319 3 : *attr = wrk->session_attr_rv;
320 :
321 5 : wrk->session_attr_op = 0;
322 :
323 5 : return wrk->session_attr_op_rv;
324 : }
325 :
326 : static u32
327 37 : vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp,
328 : u32 ls_index)
329 : {
330 : vcl_session_t *session, *listen_session;
331 : svm_msg_q_t *evt_q;
332 :
333 37 : session = vcl_session_alloc (wrk);
334 :
335 37 : listen_session = vcl_session_get (wrk, ls_index);
336 37 : if (listen_session->vpp_handle != mp->listener_handle)
337 : {
338 0 : VDBG (0, "ERROR: listener handle %lu does not match session %u",
339 : mp->listener_handle, ls_index);
340 0 : goto error;
341 : }
342 :
343 37 : if (vcl_segment_attach_session (
344 : mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
345 : mp->vpp_event_queue_address, mp->mq_index, 0, session))
346 : {
347 0 : VDBG (0, "session %u [0x%llx]: failed to attach fifos",
348 : session->session_index, mp->handle);
349 0 : goto error;
350 : }
351 :
352 37 : session->vpp_handle = mp->handle;
353 37 : session->session_state = VCL_STATE_READY;
354 37 : session->transport.rmt_port = mp->rmt.port;
355 37 : session->transport.is_ip4 = mp->rmt.is_ip4;
356 37 : clib_memcpy_fast (&session->transport.rmt_ip, &mp->rmt.ip,
357 : sizeof (ip46_address_t));
358 :
359 37 : vcl_session_table_add_vpp_handle (wrk, mp->handle, session->session_index);
360 37 : session->transport.lcl_port = mp->lcl.port;
361 37 : session->transport.lcl_ip = mp->lcl.ip;
362 37 : session->session_type = listen_session->session_type;
363 37 : session->is_dgram = vcl_proto_is_dgram (session->session_type);
364 37 : if (session->is_dgram)
365 1 : session->flags |= (listen_session->flags & VCL_SESSION_F_CONNECTED);
366 37 : session->listener_index = listen_session->session_index;
367 37 : listen_session->n_accepted_sessions++;
368 :
369 : vcl_evt (VCL_EVT_ACCEPT, session, listen_session, session_index);
370 :
371 37 : vcl_send_session_accepted_reply (session->vpp_evt_q, mp->context,
372 : session->vpp_handle, 0);
373 :
374 37 : return session->session_index;
375 :
376 0 : error:
377 0 : vcl_segment_attach_mq (vcl_vpp_worker_segment_handle (0),
378 : mp->vpp_event_queue_address, mp->mq_index, &evt_q);
379 0 : vcl_send_session_accepted_reply (evt_q, mp->context, mp->handle,
380 : VNET_API_ERROR_INVALID_ARGUMENT);
381 0 : vcl_session_free (wrk, session);
382 0 : return VCL_INVALID_SESSION_INDEX;
383 : }
384 :
385 : static u32
386 47 : vcl_session_connected_handler (vcl_worker_t * wrk,
387 : session_connected_msg_t * mp)
388 : {
389 47 : vcl_session_t *session = 0;
390 : u32 session_index;
391 :
392 47 : session_index = mp->context;
393 47 : session = vcl_session_get (wrk, session_index);
394 47 : if (PREDICT_FALSE (!session))
395 : {
396 0 : VERR ("vpp handle 0x%llx has no session index (%u)!", mp->handle,
397 : session_index);
398 : /* Should not happen but if it does, force vpp session cleanup */
399 0 : vcl_session_t tmp_session = {
400 0 : .vpp_handle = mp->handle,
401 : .vpp_evt_q = 0,
402 : };
403 0 : vcl_segment_attach_session (
404 : mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
405 : mp->vpp_event_queue_address, mp->mq_index, 0, session);
406 0 : if (tmp_session.vpp_evt_q)
407 0 : vcl_send_session_disconnect (wrk, &tmp_session);
408 0 : return VCL_INVALID_SESSION_INDEX;
409 : }
410 :
411 47 : if (mp->retval)
412 : {
413 0 : VDBG (0, "session %u: connect failed! %U", session_index,
414 : format_session_error, mp->retval);
415 0 : session->session_state = VCL_STATE_DETACHED;
416 0 : session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
417 0 : session->vpp_error = mp->retval;
418 0 : return session_index;
419 : }
420 :
421 47 : session->vpp_handle = mp->handle;
422 :
423 : /* Add to lookup table. Even if something fails, session cannot be
424 : * cleaned up prior to notifying vpp and going through the cleanup
425 : * "procedure" see @ref vcl_session_cleanup_handler */
426 47 : vcl_session_table_add_vpp_handle (wrk, mp->handle, session_index);
427 :
428 47 : if (vcl_segment_attach_session (
429 : mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
430 : mp->vpp_event_queue_address, mp->mq_index, 0, session))
431 : {
432 0 : VDBG (0, "session %u [0x%llx]: failed to attach fifos",
433 : session->session_index, session->vpp_handle);
434 0 : session->session_state = VCL_STATE_UPDATED;
435 0 : vcl_send_session_disconnect (wrk, session);
436 0 : return session_index;
437 : }
438 :
439 47 : if (mp->ct_rx_fifo)
440 : {
441 12 : if (vcl_segment_attach_session (mp->ct_segment_handle, mp->ct_rx_fifo,
442 : mp->ct_tx_fifo, (uword) ~0, ~0, 1,
443 : session))
444 : {
445 0 : VDBG (0, "session %u [0x%llx]: failed to attach ct fifos",
446 : session->session_index, session->vpp_handle);
447 0 : session->session_state = VCL_STATE_UPDATED;
448 0 : vcl_send_session_disconnect (wrk, session);
449 0 : return session_index;
450 : }
451 : }
452 :
453 47 : session->transport.is_ip4 = mp->lcl.is_ip4;
454 47 : clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip,
455 : sizeof (session->transport.lcl_ip));
456 47 : session->transport.lcl_port = mp->lcl.port;
457 :
458 : /* Application closed session before connect reply */
459 47 : if (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK)
460 0 : && session->session_state == VCL_STATE_CLOSED)
461 0 : vcl_send_session_disconnect (wrk, session);
462 : else
463 47 : session->session_state = VCL_STATE_READY;
464 :
465 47 : VDBG (0, "session %u [0x%llx] connected local: %U:%u remote %U:%u",
466 : session->session_index, session->vpp_handle, vcl_format_ip46_address,
467 : &session->transport.lcl_ip,
468 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
469 : clib_net_to_host_u16 (session->transport.lcl_port),
470 : vcl_format_ip46_address, &session->transport.rmt_ip,
471 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
472 : clib_net_to_host_u16 (session->transport.rmt_port));
473 :
474 47 : return session_index;
475 : }
476 :
477 : static int
478 0 : vcl_flag_accepted_session (vcl_session_t * session, u64 handle, u32 flags)
479 : {
480 : vcl_session_msg_t *accepted_msg;
481 : int i;
482 :
483 0 : for (i = 0; i < vec_len (session->accept_evts_fifo); i++)
484 : {
485 0 : accepted_msg = &session->accept_evts_fifo[i];
486 0 : if (accepted_msg->accepted_msg.handle == handle)
487 : {
488 0 : accepted_msg->flags |= flags;
489 0 : return 1;
490 : }
491 : }
492 0 : return 0;
493 : }
494 :
495 : static u32
496 5 : vcl_session_reset_handler (vcl_worker_t * wrk,
497 : session_reset_msg_t * reset_msg)
498 : {
499 : vcl_session_t *session;
500 : u32 sid;
501 :
502 5 : sid = vcl_session_index_from_vpp_handle (wrk, reset_msg->handle);
503 5 : session = vcl_session_get (wrk, sid);
504 5 : if (!session)
505 : {
506 0 : VDBG (0, "request to reset unknown handle 0x%llx", reset_msg->handle);
507 0 : return VCL_INVALID_SESSION_INDEX;
508 : }
509 :
510 : /* Caught a reset before actually accepting the session */
511 5 : if (session->session_state == VCL_STATE_LISTEN)
512 : {
513 :
514 0 : if (!vcl_flag_accepted_session (session, reset_msg->handle,
515 : VCL_ACCEPTED_F_RESET))
516 0 : VDBG (0, "session was not accepted!");
517 0 : return VCL_INVALID_SESSION_INDEX;
518 : }
519 :
520 5 : if (session->session_state != VCL_STATE_CLOSED)
521 4 : session->session_state = VCL_STATE_DISCONNECT;
522 :
523 5 : session->flags |= (VCL_SESSION_F_RD_SHUTDOWN | VCL_SESSION_F_WR_SHUTDOWN);
524 5 : VDBG (0, "session %u [0x%llx]: reset", sid, reset_msg->handle);
525 5 : return sid;
526 : }
527 :
528 : static u32
529 40 : vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
530 : {
531 : vcl_session_t *session;
532 40 : u32 sid = mp->context;
533 :
534 40 : session = vcl_session_get (wrk, sid);
535 40 : if (mp->retval)
536 : {
537 0 : VERR ("session %u [0x%llx]: bind failed: %U", sid, mp->handle,
538 : format_session_error, mp->retval);
539 0 : if (session)
540 : {
541 0 : session->session_state = VCL_STATE_DETACHED;
542 0 : session->vpp_handle = mp->handle;
543 0 : return sid;
544 : }
545 : else
546 : {
547 0 : VDBG (0, "session %u [0x%llx]: Invalid session index!", sid,
548 : mp->handle);
549 0 : return VCL_INVALID_SESSION_INDEX;
550 : }
551 : }
552 :
553 40 : session->vpp_handle = mp->handle;
554 40 : session->transport.is_ip4 = mp->lcl_is_ip4;
555 40 : clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip,
556 : sizeof (ip46_address_t));
557 40 : session->transport.lcl_port = mp->lcl_port;
558 40 : vcl_session_table_add_listener (wrk, mp->handle, sid);
559 40 : session->session_state = VCL_STATE_LISTEN;
560 40 : session->flags &= ~VCL_SESSION_F_PENDING_LISTEN;
561 :
562 40 : if (vcl_session_is_cl (session))
563 : {
564 3 : if (vcl_segment_attach_session (mp->segment_handle, mp->rx_fifo,
565 : mp->tx_fifo, mp->vpp_evt_q, mp->mq_index,
566 : 0, session))
567 : {
568 0 : VDBG (0, "session %u [0x%llx]: failed to attach fifos",
569 : session->session_index, session->vpp_handle);
570 0 : session->session_state = VCL_STATE_DETACHED;
571 0 : return VCL_INVALID_SESSION_INDEX;
572 : }
573 : }
574 :
575 40 : VDBG (0, "session %u [0x%llx]: listen succeeded!", sid, mp->handle);
576 40 : return sid;
577 : }
578 :
579 : static void
580 10 : vcl_session_unlisten_reply_handler (vcl_worker_t * wrk, void *data)
581 : {
582 10 : session_unlisten_reply_msg_t *mp = (session_unlisten_reply_msg_t *) data;
583 : vcl_session_t *s;
584 :
585 10 : s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
586 10 : if (!s)
587 : {
588 0 : VDBG (0, "Unlisten reply with wrong handle %llx", mp->handle);
589 0 : return;
590 : }
591 10 : if (s->session_state != VCL_STATE_DISCONNECT)
592 : {
593 : /* Connected udp listener */
594 2 : if (s->session_type == VPPCOM_PROTO_UDP
595 2 : && s->session_state == VCL_STATE_CLOSED)
596 2 : return;
597 :
598 0 : VDBG (0, "Unlisten session in wrong state %llx", mp->handle);
599 0 : return;
600 : }
601 :
602 8 : if (mp->retval)
603 0 : VDBG (0, "ERROR: session %u [0xllx]: unlisten failed: %U",
604 : s->session_index, mp->handle, format_session_error, mp->retval);
605 :
606 8 : if (mp->context != wrk->wrk_index)
607 0 : VDBG (0, "wrong context");
608 :
609 8 : vcl_session_table_del_vpp_handle (wrk, mp->handle);
610 8 : vcl_session_free (wrk, s);
611 : }
612 :
613 : static void
614 0 : vcl_session_migrated_handler (vcl_worker_t * wrk, void *data)
615 : {
616 0 : session_migrated_msg_t *mp = (session_migrated_msg_t *) data;
617 : vcl_session_t *s;
618 : u32 fs_index;
619 :
620 0 : s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
621 0 : if (!s)
622 : {
623 0 : VDBG (0, "Migrated notification with wrong handle %llx", mp->handle);
624 0 : return;
625 : }
626 :
627 : /* Only validate if a value is provided */
628 0 : if (mp->segment_handle != SESSION_INVALID_HANDLE)
629 : {
630 0 : fs_index = vcl_segment_table_lookup (mp->segment_handle);
631 0 : if (fs_index == VCL_INVALID_SEGMENT_INDEX)
632 : {
633 0 : VDBG (0, "segment %lx for session %u is not mounted!",
634 : mp->segment_handle, s->session_index);
635 0 : s->session_state = VCL_STATE_DETACHED;
636 0 : return;
637 : }
638 : }
639 :
640 0 : s->vpp_handle = mp->new_handle;
641 :
642 0 : vcl_segment_attach_mq (vcl_vpp_worker_segment_handle (0), mp->vpp_evt_q,
643 : mp->vpp_thread_index, &s->vpp_evt_q);
644 :
645 0 : vcl_session_table_del_vpp_handle (wrk, mp->handle);
646 0 : vcl_session_table_add_vpp_handle (wrk, mp->new_handle, s->session_index);
647 :
648 : /* Generate new tx event if we have outstanding data */
649 0 : if (svm_fifo_has_event (s->tx_fifo))
650 0 : app_send_io_evt_to_vpp (s->vpp_evt_q,
651 0 : s->tx_fifo->shr->master_session_index,
652 : SESSION_IO_EVT_TX, SVM_Q_WAIT);
653 :
654 0 : VDBG (0, "Migrated 0x%lx to thread %u 0x%lx", mp->handle,
655 : mp->vpp_thread_index, mp->new_handle);
656 : }
657 :
658 : static vcl_session_t *
659 37 : vcl_session_accepted (vcl_worker_t * wrk, session_accepted_msg_t * msg)
660 : {
661 : vcl_session_msg_t *vcl_msg;
662 : vcl_session_t *session;
663 :
664 37 : session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
665 37 : if (PREDICT_FALSE (session != 0))
666 0 : VWRN ("session overlap handle %lu state %u!", msg->handle,
667 : session->session_state);
668 :
669 37 : session = vcl_session_table_lookup_listener (wrk, msg->listener_handle);
670 37 : if (!session)
671 : {
672 0 : VERR ("couldn't find listen session: listener handle %llx",
673 : msg->listener_handle);
674 0 : return 0;
675 : }
676 :
677 37 : clib_fifo_add2 (session->accept_evts_fifo, vcl_msg);
678 37 : vcl_msg->flags = 0;
679 37 : vcl_msg->accepted_msg = *msg;
680 : /* Session handle points to listener until fully accepted by app */
681 37 : vcl_session_table_add_vpp_handle (wrk, msg->handle, session->session_index);
682 :
683 37 : return session;
684 : }
685 :
686 : static vcl_session_t *
687 11 : vcl_session_disconnected_handler (vcl_worker_t * wrk,
688 : session_disconnected_msg_t * msg)
689 : {
690 : vcl_session_t *session;
691 :
692 11 : session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
693 11 : if (!session)
694 : {
695 0 : VWRN ("request to disconnect unknown handle 0x%llx", msg->handle);
696 0 : return 0;
697 : }
698 :
699 : /* Late disconnect notification on a session that has been closed */
700 11 : if (session->session_state == VCL_STATE_CLOSED)
701 5 : return 0;
702 :
703 : /* Caught a disconnect before actually accepting the session */
704 6 : if (session->session_state == VCL_STATE_LISTEN)
705 : {
706 0 : if (!vcl_flag_accepted_session (session, msg->handle,
707 : VCL_ACCEPTED_F_CLOSED))
708 0 : VDBG (0, "session was not accepted!");
709 0 : return 0;
710 : }
711 :
712 : /* If not already reset change state */
713 6 : if (session->session_state != VCL_STATE_DISCONNECT)
714 6 : session->session_state = VCL_STATE_VPP_CLOSING;
715 :
716 6 : return session;
717 : }
718 :
719 : int
720 0 : vppcom_session_shutdown (uint32_t session_handle, int how)
721 : {
722 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
723 : vcl_session_t *session;
724 : vcl_session_state_t state;
725 : u64 vpp_handle;
726 :
727 0 : session = vcl_session_get_w_handle (wrk, session_handle);
728 0 : if (PREDICT_FALSE (!session))
729 0 : return VPPCOM_EBADFD;
730 :
731 0 : vpp_handle = session->vpp_handle;
732 0 : state = session->session_state;
733 :
734 0 : VDBG (1, "session %u [0x%llx] state 0x%x (%s)", session->session_index,
735 : vpp_handle, state, vcl_session_state_str (state));
736 :
737 0 : if (PREDICT_FALSE (state == VCL_STATE_LISTEN))
738 : {
739 0 : VDBG (0, "ERROR: Cannot shutdown a listen socket!");
740 0 : return VPPCOM_EBADFD;
741 : }
742 :
743 0 : if (how == SHUT_RD || how == SHUT_RDWR)
744 : {
745 0 : session->flags |= VCL_SESSION_F_RD_SHUTDOWN;
746 0 : if (how == SHUT_RD)
747 0 : return VPPCOM_OK;
748 : }
749 0 : session->flags |= VCL_SESSION_F_WR_SHUTDOWN;
750 :
751 0 : if (PREDICT_TRUE (state == VCL_STATE_READY))
752 : {
753 0 : VDBG (1, "session %u [0x%llx]: sending shutdown...",
754 : session->session_index, vpp_handle);
755 :
756 0 : vcl_send_session_shutdown (wrk, session);
757 : }
758 :
759 0 : return VPPCOM_OK;
760 : }
761 :
762 : static int
763 62 : vppcom_session_disconnect (u32 session_handle)
764 : {
765 62 : vcl_worker_t *wrk = vcl_worker_get_current ();
766 : vcl_session_t *session, *listen_session;
767 : vcl_session_state_t state;
768 : u64 vpp_handle;
769 :
770 62 : session = vcl_session_get_w_handle (wrk, session_handle);
771 62 : if (!session)
772 0 : return VPPCOM_EBADFD;
773 :
774 62 : vpp_handle = session->vpp_handle;
775 62 : state = session->session_state;
776 :
777 62 : VDBG (1, "session %u [0x%llx]: disconnecting state (%s)",
778 : session->session_index, vpp_handle, vcl_session_state_str (state));
779 :
780 62 : if (PREDICT_FALSE (state == VCL_STATE_LISTEN))
781 : {
782 0 : VDBG (0, "ERROR: Cannot disconnect a listen socket!");
783 0 : return VPPCOM_EBADFD;
784 : }
785 :
786 62 : if (state == VCL_STATE_VPP_CLOSING)
787 : {
788 6 : vcl_send_session_disconnected_reply (wrk, session, 0);
789 6 : VDBG (1, "session %u [0x%llx]: sending disconnect REPLY...",
790 : session->session_index, vpp_handle);
791 : }
792 : else
793 : {
794 : /* Session doesn't have an event queue yet. Probably a non-blocking
795 : * connect. Wait for the reply */
796 56 : if (PREDICT_FALSE (!session->vpp_evt_q))
797 0 : return VPPCOM_OK;
798 :
799 56 : VDBG (1, "session %u [0x%llx]: sending disconnect",
800 : session->session_index, vpp_handle);
801 56 : vcl_send_session_disconnect (wrk, session);
802 : }
803 :
804 62 : if (session->listener_index != VCL_INVALID_SESSION_INDEX)
805 : {
806 36 : listen_session = vcl_session_get (wrk, session->listener_index);
807 36 : if (listen_session)
808 35 : listen_session->n_accepted_sessions--;
809 : }
810 :
811 62 : return VPPCOM_OK;
812 : }
813 :
814 : static void
815 18 : vcl_session_cleanup_handler (vcl_worker_t * wrk, void *data)
816 : {
817 : session_cleanup_msg_t *msg;
818 : vcl_session_t *session;
819 :
820 18 : msg = (session_cleanup_msg_t *) data;
821 18 : session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
822 18 : if (!session)
823 : {
824 0 : VWRN ("disconnect confirmed for unknown handle 0x%llx", msg->handle);
825 0 : return;
826 : }
827 :
828 18 : if (msg->type == SESSION_CLEANUP_TRANSPORT)
829 : {
830 : /* Transport was cleaned up before we confirmed close. Probably the
831 : * app is still waiting for some data that cannot be delivered.
832 : * Confirm close to make sure everything is cleaned up.
833 : * Move to undetermined state to ensure that the session is not
834 : * removed before both vpp and the app cleanup.
835 : * - If the app closes first, the session is moved to CLOSED state
836 : * and the session cleanup notification from vpp removes the
837 : * session.
838 : * - If vpp cleans up the session first, the session is moved to
839 : * DETACHED state lower and subsequently the close from the app
840 : * frees the session
841 : */
842 2 : if (session->session_state == VCL_STATE_VPP_CLOSING)
843 : {
844 0 : vppcom_session_disconnect (vcl_session_handle (session));
845 0 : session->session_state = VCL_STATE_UPDATED;
846 : }
847 2 : else if (session->session_state == VCL_STATE_DISCONNECT)
848 : {
849 0 : vcl_send_session_reset_reply (wrk, session, 0);
850 0 : session->session_state = VCL_STATE_UPDATED;
851 : }
852 2 : return;
853 : }
854 :
855 : /* VPP will reuse the handle so clean it up now */
856 16 : vcl_session_table_del_vpp_handle (wrk, msg->handle);
857 :
858 : /* App did not close the connection yet so don't free it. */
859 16 : if (session->session_state != VCL_STATE_CLOSED)
860 : {
861 0 : VDBG (0, "session %u: app did not close", session->session_index);
862 0 : session->session_state = VCL_STATE_DETACHED;
863 0 : session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
864 0 : return;
865 : }
866 :
867 : /* Session probably tracked with epoll, disconnect not yet handled and
868 : * 1) both transport and session cleanup completed 2) app closed. Wait
869 : * until message is drained to free the session.
870 : * See @ref vcl_handle_mq_event */
871 16 : if (session->flags & VCL_SESSION_F_PENDING_DISCONNECT)
872 : {
873 0 : session->flags |= VCL_SESSION_F_PENDING_FREE;
874 0 : return;
875 : }
876 :
877 16 : vcl_session_free (wrk, session);
878 : }
879 :
880 : static void
881 0 : vcl_session_req_worker_update_handler (vcl_worker_t * wrk, void *data)
882 : {
883 : session_req_worker_update_msg_t *msg;
884 : vcl_session_t *s;
885 :
886 0 : msg = (session_req_worker_update_msg_t *) data;
887 0 : s = vcl_session_get_w_vpp_handle (wrk, msg->session_handle);
888 0 : if (!s)
889 0 : return;
890 :
891 0 : vec_add1 (wrk->pending_session_wrk_updates, s->session_index);
892 : }
893 :
894 : static void
895 0 : vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data)
896 : {
897 : session_worker_update_reply_msg_t *msg;
898 : vcl_session_t *s;
899 :
900 0 : msg = (session_worker_update_reply_msg_t *) data;
901 0 : s = vcl_session_get_w_vpp_handle (wrk, msg->handle);
902 0 : if (!s)
903 : {
904 0 : VDBG (0, "unknown handle 0x%llx", msg->handle);
905 0 : return;
906 : }
907 :
908 0 : if (s->rx_fifo)
909 : {
910 0 : if (vcl_segment_attach_session (msg->segment_handle, msg->rx_fifo,
911 : msg->tx_fifo, (uword) ~0, ~0, 0, s))
912 : {
913 0 : VDBG (0, "failed to attach fifos for %u", s->session_index);
914 0 : return;
915 : }
916 : }
917 0 : s->session_state = VCL_STATE_UPDATED;
918 :
919 0 : VDBG (0, "session %u[0x%llx] moved to worker %u", s->session_index,
920 : s->vpp_handle, wrk->wrk_index);
921 : }
922 :
923 : static int
924 37 : vcl_api_recv_fd (vcl_worker_t * wrk, int *fds, int n_fds)
925 : {
926 :
927 37 : if (vcm->cfg.vpp_app_socket_api)
928 7 : return vcl_sapi_recv_fds (wrk, fds, n_fds);
929 :
930 30 : return vcl_bapi_recv_fds (wrk, fds, n_fds);
931 : }
932 :
933 : static void
934 37 : vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data)
935 : {
936 37 : ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM;
937 : session_app_add_segment_msg_t *msg;
938 : u64 segment_handle;
939 37 : int fd = -1;
940 :
941 37 : msg = (session_app_add_segment_msg_t *) data;
942 :
943 37 : if (msg->fd_flags)
944 : {
945 37 : vcl_api_recv_fd (wrk, &fd, 1);
946 37 : seg_type = SSVM_SEGMENT_MEMFD;
947 : }
948 :
949 37 : segment_handle = msg->segment_handle;
950 37 : if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
951 : {
952 0 : clib_warning ("invalid segment handle");
953 0 : return;
954 : }
955 :
956 37 : if (vcl_segment_attach (segment_handle, (char *) msg->segment_name,
957 : seg_type, fd))
958 : {
959 0 : VDBG (0, "vcl_segment_attach ('%s') failed", msg->segment_name);
960 0 : return;
961 : }
962 :
963 37 : VDBG (1, "mapped new segment '%s' size %d", msg->segment_name,
964 : msg->segment_size);
965 : }
966 :
967 : static void
968 4 : vcl_session_app_del_segment_handler (vcl_worker_t * wrk, void *data)
969 : {
970 4 : session_app_del_segment_msg_t *msg = (session_app_del_segment_msg_t *) data;
971 4 : vcl_segment_detach (msg->segment_handle);
972 4 : VDBG (1, "Unmapped segment: %lx", msg->segment_handle);
973 4 : }
974 :
975 : static void
976 0 : vcl_worker_rpc_handler (vcl_worker_t * wrk, void *data)
977 : {
978 0 : if (!vcm->wrk_rpc_fn)
979 0 : return;
980 :
981 0 : (vcm->wrk_rpc_fn) (((session_app_wrk_rpc_msg_t *) data)->data);
982 : }
983 :
984 : static void
985 5 : vcl_session_transport_attr_reply_handler (vcl_worker_t *wrk, void *data)
986 : {
987 : session_transport_attr_reply_msg_t *mp;
988 :
989 5 : if (!wrk->session_attr_op)
990 0 : return;
991 :
992 5 : mp = (session_transport_attr_reply_msg_t *) data;
993 :
994 5 : wrk->session_attr_op_rv = mp->retval;
995 5 : wrk->session_attr_op = 0;
996 5 : wrk->session_attr_rv = mp->attr;
997 : }
998 :
999 : static int
1000 223 : vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
1001 : {
1002 : session_disconnected_msg_t *disconnected_msg;
1003 : session_connected_msg_t *connected_msg;
1004 : session_reset_msg_t *reset_msg;
1005 : session_event_t *ecpy;
1006 : vcl_session_t *s;
1007 : u32 sid;
1008 :
1009 223 : switch (e->event_type)
1010 : {
1011 94 : case SESSION_IO_EVT_RX:
1012 : case SESSION_IO_EVT_TX:
1013 94 : s = vcl_session_get (wrk, e->session_index);
1014 94 : if (!s || !vcl_session_is_open (s))
1015 : break;
1016 88 : vec_add1 (wrk->unhandled_evts_vector, *e);
1017 88 : break;
1018 40 : case SESSION_CTRL_EVT_BOUND:
1019 : /* We can only wait for only one listen so not postponed */
1020 40 : vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
1021 40 : break;
1022 0 : case SESSION_CTRL_EVT_ACCEPTED:
1023 0 : s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
1024 0 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1025 : {
1026 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1027 0 : *ecpy = *e;
1028 0 : ecpy->postponed = 1;
1029 0 : ecpy->session_index = s->session_index;
1030 : }
1031 0 : break;
1032 47 : case SESSION_CTRL_EVT_CONNECTED:
1033 47 : connected_msg = (session_connected_msg_t *) e->data;
1034 47 : sid = vcl_session_connected_handler (wrk, connected_msg);
1035 47 : if (!(s = vcl_session_get (wrk, sid)))
1036 0 : break;
1037 47 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1038 : {
1039 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1040 0 : *ecpy = *e;
1041 0 : ecpy->postponed = 1;
1042 0 : ecpy->session_index = s->session_index;
1043 : }
1044 47 : break;
1045 1 : case SESSION_CTRL_EVT_DISCONNECTED:
1046 1 : disconnected_msg = (session_disconnected_msg_t *) e->data;
1047 1 : if (!(s = vcl_session_get_w_vpp_handle (wrk, disconnected_msg->handle)))
1048 0 : break;
1049 1 : if (s->session_state == VCL_STATE_CLOSED)
1050 0 : break;
1051 1 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1052 : {
1053 0 : s->session_state = VCL_STATE_VPP_CLOSING;
1054 0 : s->flags |= VCL_SESSION_F_PENDING_DISCONNECT;
1055 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1056 0 : *ecpy = *e;
1057 0 : ecpy->postponed = 1;
1058 0 : ecpy->session_index = s->session_index;
1059 0 : break;
1060 : }
1061 1 : if (!(s = vcl_session_disconnected_handler (wrk, disconnected_msg)))
1062 0 : break;
1063 1 : VDBG (0, "disconnected session %u [0x%llx]", s->session_index,
1064 : s->vpp_handle);
1065 1 : break;
1066 3 : case SESSION_CTRL_EVT_RESET:
1067 3 : reset_msg = (session_reset_msg_t *) e->data;
1068 3 : if (!(s = vcl_session_get_w_vpp_handle (wrk, reset_msg->handle)))
1069 0 : break;
1070 3 : if (s->session_state == VCL_STATE_CLOSED)
1071 0 : break;
1072 3 : if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1073 : {
1074 0 : s->flags |= VCL_SESSION_F_PENDING_DISCONNECT;
1075 0 : s->session_state = VCL_STATE_DISCONNECT;
1076 0 : s->flags |= (VCL_SESSION_F_RD_SHUTDOWN | VCL_SESSION_F_WR_SHUTDOWN);
1077 0 : vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
1078 0 : *ecpy = *e;
1079 0 : ecpy->postponed = 1;
1080 0 : ecpy->session_index = s->session_index;
1081 0 : break;
1082 : }
1083 3 : vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
1084 3 : break;
1085 9 : case SESSION_CTRL_EVT_UNLISTEN_REPLY:
1086 9 : vcl_session_unlisten_reply_handler (wrk, e->data);
1087 9 : break;
1088 0 : case SESSION_CTRL_EVT_MIGRATED:
1089 0 : vcl_session_migrated_handler (wrk, e->data);
1090 0 : break;
1091 11 : case SESSION_CTRL_EVT_CLEANUP:
1092 11 : vcl_session_cleanup_handler (wrk, e->data);
1093 11 : break;
1094 0 : case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
1095 0 : vcl_session_req_worker_update_handler (wrk, e->data);
1096 0 : break;
1097 0 : case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
1098 0 : vcl_session_worker_update_reply_handler (wrk, e->data);
1099 0 : break;
1100 11 : case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
1101 11 : vcl_session_app_add_segment_handler (wrk, e->data);
1102 11 : break;
1103 2 : case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
1104 2 : vcl_session_app_del_segment_handler (wrk, e->data);
1105 2 : break;
1106 0 : case SESSION_CTRL_EVT_APP_WRK_RPC:
1107 0 : vcl_worker_rpc_handler (wrk, e->data);
1108 0 : break;
1109 5 : case SESSION_CTRL_EVT_TRANSPORT_ATTR_REPLY:
1110 5 : vcl_session_transport_attr_reply_handler (wrk, e->data);
1111 5 : break;
1112 0 : default:
1113 0 : clib_warning ("unhandled %u", e->event_type);
1114 : }
1115 223 : return VPPCOM_OK;
1116 : }
1117 :
1118 : static int
1119 87 : vppcom_wait_for_session_state_change (u32 session_index,
1120 : vcl_session_state_t state,
1121 : f64 wait_for_time)
1122 : {
1123 87 : vcl_worker_t *wrk = vcl_worker_get_current ();
1124 87 : f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
1125 : vcl_session_t *volatile session;
1126 : svm_msg_q_msg_t msg;
1127 : session_event_t *e;
1128 :
1129 : do
1130 : {
1131 1400 : session = vcl_session_get (wrk, session_index);
1132 1400 : if (PREDICT_FALSE (!session))
1133 : {
1134 0 : return VPPCOM_EBADFD;
1135 : }
1136 1400 : if (session->session_state == state)
1137 : {
1138 87 : return VPPCOM_OK;
1139 : }
1140 1313 : if (session->session_state == VCL_STATE_DETACHED)
1141 : {
1142 0 : return VPPCOM_ECONNREFUSED;
1143 : }
1144 :
1145 1313 : if (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0))
1146 : {
1147 1194 : usleep (100);
1148 1194 : continue;
1149 : }
1150 119 : e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
1151 119 : vcl_handle_mq_event (wrk, e);
1152 119 : svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1153 : }
1154 1313 : while (clib_time_now (&wrk->clib_time) < timeout);
1155 :
1156 0 : VDBG (0, "timeout waiting for state 0x%x (%s)", state,
1157 : vcl_session_state_str (state));
1158 : vcl_evt (VCL_EVT_SESSION_TIMEOUT, session, session_state);
1159 :
1160 0 : return VPPCOM_ETIMEDOUT;
1161 : }
1162 :
1163 : static void
1164 111766 : vcl_handle_pending_wrk_updates (vcl_worker_t * wrk)
1165 : {
1166 : vcl_session_state_t state;
1167 : vcl_session_t *s;
1168 : u32 *sip;
1169 :
1170 111766 : if (PREDICT_TRUE (vec_len (wrk->pending_session_wrk_updates) == 0))
1171 111766 : return;
1172 :
1173 0 : vec_foreach (sip, wrk->pending_session_wrk_updates)
1174 : {
1175 0 : s = vcl_session_get (wrk, *sip);
1176 0 : vcl_send_session_worker_update (wrk, s, wrk->wrk_index);
1177 0 : state = s->session_state;
1178 0 : vppcom_wait_for_session_state_change (s->session_index, VCL_STATE_UPDATED,
1179 : 5);
1180 0 : s->session_state = state;
1181 : }
1182 0 : vec_reset_length (wrk->pending_session_wrk_updates);
1183 : }
1184 :
1185 : void
1186 1445 : vcl_worker_flush_mq_events (vcl_worker_t *wrk)
1187 : {
1188 : svm_msg_q_msg_t *msg;
1189 : session_event_t *e;
1190 : svm_msg_q_t *mq;
1191 : int i;
1192 :
1193 1445 : mq = wrk->app_event_queue;
1194 1445 : vcl_mq_dequeue_batch (wrk, mq, ~0);
1195 :
1196 1549 : for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
1197 : {
1198 104 : msg = vec_elt_at_index (wrk->mq_msg_vector, i);
1199 104 : e = svm_msg_q_msg_data (mq, msg);
1200 104 : vcl_handle_mq_event (wrk, e);
1201 104 : svm_msg_q_free_msg (mq, msg);
1202 : }
1203 1445 : vec_reset_length (wrk->mq_msg_vector);
1204 1445 : vcl_handle_pending_wrk_updates (wrk);
1205 1445 : }
1206 :
1207 : void
1208 1350 : vcl_flush_mq_events (void)
1209 : {
1210 1350 : vcl_worker_flush_mq_events (vcl_worker_get_current ());
1211 1350 : }
1212 :
1213 : static int
1214 21 : vppcom_session_unbind (u32 session_handle)
1215 : {
1216 21 : vcl_worker_t *wrk = vcl_worker_get_current ();
1217 : session_accepted_msg_t *accepted_msg;
1218 21 : vcl_session_t *session = 0;
1219 : vcl_session_msg_t *evt;
1220 :
1221 21 : session = vcl_session_get_w_handle (wrk, session_handle);
1222 21 : if (!session)
1223 0 : return VPPCOM_EBADFD;
1224 :
1225 : /* Flush pending accept events, if any */
1226 21 : while (clib_fifo_elts (session->accept_evts_fifo))
1227 : {
1228 0 : clib_fifo_sub2 (session->accept_evts_fifo, evt);
1229 0 : accepted_msg = &evt->accepted_msg;
1230 0 : vcl_session_table_del_vpp_handle (wrk, accepted_msg->handle);
1231 0 : vcl_send_session_accepted_reply (session->vpp_evt_q,
1232 : accepted_msg->context,
1233 : accepted_msg->handle, -1);
1234 : }
1235 21 : clib_fifo_free (session->accept_evts_fifo);
1236 :
1237 21 : vcl_send_session_unlisten (wrk, session);
1238 :
1239 21 : VDBG (0, "session %u [0x%llx]: sending unbind!", session->session_index,
1240 : session->vpp_handle);
1241 : vcl_evt (VCL_EVT_UNBIND, session);
1242 :
1243 21 : session->vpp_handle = ~0;
1244 21 : session->session_state = VCL_STATE_DISCONNECT;
1245 :
1246 21 : return VPPCOM_OK;
1247 : }
1248 :
1249 : /**
1250 : * Handle app exit
1251 : *
1252 : * Notify vpp of the disconnect and mark the worker as free. If we're the
1253 : * last worker, do a full cleanup otherwise, since we're probably a forked
1254 : * child, avoid syscalls as much as possible. We might've lost privileges.
1255 : */
1256 : void
1257 37 : vppcom_app_exit (void)
1258 : {
1259 37 : if (!pool_elts (vcm->workers))
1260 24 : return;
1261 13 : vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
1262 13 : vcl_set_worker_index (~0);
1263 : vcl_elog_stop (vcm);
1264 : }
1265 :
1266 : static int
1267 37 : vcl_api_attach (void)
1268 : {
1269 37 : if (vcm->cfg.vpp_app_socket_api)
1270 8 : return vcl_sapi_attach ();
1271 :
1272 29 : return vcl_bapi_attach ();
1273 : }
1274 :
1275 : int
1276 0 : vcl_is_first_reattach_to_execute ()
1277 : {
1278 0 : if (vcm->reattach_count == 0)
1279 0 : return 1;
1280 :
1281 0 : return 0;
1282 : }
1283 :
1284 : void
1285 0 : vcl_set_reattach_counter ()
1286 : {
1287 0 : ++vcm->reattach_count;
1288 :
1289 0 : if (vcm->reattach_count == vec_len (vcm->workers))
1290 0 : vcm->reattach_count = 0;
1291 0 : }
1292 :
1293 : /**
1294 : * Reattach vcl to vpp after it has previously been disconnected.
1295 : *
1296 : * The logic should be:
1297 : * - first worker to hit `vcl_api_retry_attach` should attach to vpp,
1298 : * to reproduce the `vcl_api_attach` in `vppcom_app_create`.
1299 : * - the rest of the workers should `reproduce vcl_worker_register_with_vpp`
1300 : * from `vppcom_worker_register` since they were already allocated.
1301 : */
1302 :
1303 : static void
1304 0 : vcl_api_retry_attach (vcl_worker_t *wrk)
1305 : {
1306 : vcl_session_t *s;
1307 :
1308 0 : clib_spinlock_lock (&vcm->workers_lock);
1309 0 : if (vcl_is_first_reattach_to_execute ())
1310 : {
1311 0 : if (vcl_api_attach ())
1312 : {
1313 0 : clib_spinlock_unlock (&vcm->workers_lock);
1314 0 : return;
1315 : }
1316 0 : vcl_set_reattach_counter ();
1317 0 : clib_spinlock_unlock (&vcm->workers_lock);
1318 : }
1319 : else
1320 : {
1321 0 : vcl_set_reattach_counter ();
1322 0 : clib_spinlock_unlock (&vcm->workers_lock);
1323 0 : vcl_worker_register_with_vpp ();
1324 : }
1325 :
1326 : /* Treat listeners as configuration that needs to be re-added to vpp */
1327 0 : pool_foreach (s, wrk->sessions)
1328 : {
1329 0 : if (s->flags & VCL_SESSION_F_IS_VEP)
1330 0 : continue;
1331 0 : if (s->session_state == VCL_STATE_LISTEN_NO_MQ)
1332 0 : vppcom_session_listen (vcl_session_handle (s), 10);
1333 : else
1334 0 : VDBG (0, "internal error: unexpected state %d", s->session_state);
1335 : }
1336 : }
1337 :
1338 : static void
1339 0 : vcl_api_handle_disconnect (vcl_worker_t *wrk)
1340 : {
1341 0 : wrk->api_client_handle = ~0;
1342 0 : vcl_worker_detach_sessions (wrk);
1343 0 : }
1344 :
1345 : static void
1346 24 : vcl_api_detach (vcl_worker_t * wrk)
1347 : {
1348 24 : if (wrk->api_client_handle == ~0)
1349 0 : return;
1350 :
1351 24 : vcl_send_app_detach (wrk);
1352 :
1353 24 : if (vcm->cfg.vpp_app_socket_api)
1354 6 : return vcl_sapi_detach (wrk);
1355 :
1356 18 : return vcl_bapi_disconnect_from_vpp ();
1357 : }
1358 :
1359 : /*
1360 : * VPPCOM Public API functions
1361 : */
1362 : int
1363 37 : vppcom_app_create (const char *app_name)
1364 : {
1365 37 : vppcom_cfg_t *vcl_cfg = &vcm->cfg;
1366 : int rv;
1367 :
1368 37 : if (vcm->is_init)
1369 : {
1370 0 : VDBG (1, "already initialized");
1371 0 : return VPPCOM_EEXIST;
1372 : }
1373 :
1374 37 : vcm->is_init = 1;
1375 37 : vppcom_cfg (&vcm->cfg);
1376 37 : vcl_cfg = &vcm->cfg;
1377 :
1378 37 : vcm->main_cpu = pthread_self ();
1379 37 : vcm->main_pid = getpid ();
1380 37 : vcm->app_name = format (0, "%s", app_name);
1381 37 : fifo_segment_main_init (&vcm->segment_main, (uword) ~0,
1382 : 20 /* timeout in secs */);
1383 37 : pool_alloc (vcm->workers, vcl_cfg->max_workers);
1384 37 : clib_spinlock_init (&vcm->workers_lock);
1385 37 : clib_rwlock_init (&vcm->segment_table_lock);
1386 37 : atexit (vppcom_app_exit);
1387 : vcl_elog_init (vcm);
1388 :
1389 : /* Allocate default worker */
1390 37 : vcl_worker_alloc_and_init ();
1391 :
1392 37 : if ((rv = vcl_api_attach ()))
1393 : {
1394 0 : vppcom_app_destroy ();
1395 0 : return rv;
1396 : }
1397 :
1398 37 : VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
1399 : vcm->workers[0].api_client_handle, vcm->workers[0].api_client_handle);
1400 :
1401 37 : return VPPCOM_OK;
1402 : }
1403 :
1404 : void
1405 24 : vppcom_app_destroy (void)
1406 : {
1407 : vcl_worker_t *wrk, *current_wrk;
1408 : void *heap;
1409 :
1410 24 : if (!pool_elts (vcm->workers))
1411 0 : return;
1412 :
1413 : vcl_evt (VCL_EVT_DETACH, vcm);
1414 :
1415 24 : current_wrk = vcl_worker_get_current ();
1416 :
1417 : /* *INDENT-OFF* */
1418 48 : pool_foreach (wrk, vcm->workers) {
1419 24 : if (current_wrk != wrk)
1420 0 : vcl_worker_cleanup (wrk, 0 /* notify vpp */ );
1421 : }
1422 : /* *INDENT-ON* */
1423 :
1424 24 : vcl_api_detach (current_wrk);
1425 24 : vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ );
1426 24 : vcl_set_worker_index (~0);
1427 :
1428 : vcl_elog_stop (vcm);
1429 :
1430 : /*
1431 : * Free the heap and fix vcm
1432 : */
1433 24 : heap = clib_mem_get_heap ();
1434 24 : munmap (clib_mem_get_heap_base (heap), clib_mem_get_heap_size (heap));
1435 :
1436 24 : vcm = &_vppcom_main;
1437 24 : vcm->is_init = 0;
1438 : }
1439 :
1440 : int
1441 85 : vppcom_session_create (u8 proto, u8 is_nonblocking)
1442 : {
1443 85 : vcl_worker_t *wrk = vcl_worker_get_current ();
1444 : vcl_session_t *session;
1445 :
1446 85 : session = vcl_session_alloc (wrk);
1447 :
1448 85 : session->session_type = proto;
1449 85 : session->session_state = VCL_STATE_CLOSED;
1450 85 : session->vpp_handle = ~0;
1451 85 : session->is_dgram = vcl_proto_is_dgram (proto);
1452 85 : session->vpp_error = SESSION_E_NONE;
1453 :
1454 85 : if (is_nonblocking)
1455 12 : vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
1456 :
1457 : vcl_evt (VCL_EVT_CREATE, session, session_type, session->session_state,
1458 : is_nonblocking, session_index);
1459 :
1460 85 : VDBG (0, "created session %u", session->session_index);
1461 :
1462 85 : return vcl_session_handle (session);
1463 : }
1464 :
1465 : static void
1466 0 : vcl_epoll_lt_add (vcl_worker_t *wrk, vcl_session_t *s)
1467 : {
1468 : vcl_session_t *cur, *prev;
1469 :
1470 0 : ASSERT (s->vep.lt_next == VCL_INVALID_SESSION_INDEX);
1471 :
1472 0 : if (wrk->ep_lt_current == VCL_INVALID_SESSION_INDEX)
1473 : {
1474 0 : wrk->ep_lt_current = s->session_index;
1475 0 : s->vep.lt_next = s->session_index;
1476 0 : s->vep.lt_prev = s->session_index;
1477 0 : return;
1478 : }
1479 :
1480 0 : cur = vcl_session_get (wrk, wrk->ep_lt_current);
1481 0 : prev = vcl_session_get (wrk, cur->vep.lt_prev);
1482 :
1483 0 : prev->vep.lt_next = s->session_index;
1484 0 : s->vep.lt_prev = prev->session_index;
1485 :
1486 0 : s->vep.lt_next = cur->session_index;
1487 0 : cur->vep.lt_prev = s->session_index;
1488 : }
1489 :
1490 : static void
1491 0 : vcl_epoll_lt_del (vcl_worker_t *wrk, vcl_session_t *s)
1492 : {
1493 : vcl_session_t *prev, *next;
1494 :
1495 0 : ASSERT (s->vep.lt_next != VCL_INVALID_SESSION_INDEX);
1496 :
1497 0 : if (s->vep.lt_next == s->session_index)
1498 : {
1499 0 : wrk->ep_lt_current = VCL_INVALID_SESSION_INDEX;
1500 0 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
1501 0 : s->vep.lt_prev = VCL_INVALID_SESSION_INDEX;
1502 0 : return;
1503 : }
1504 :
1505 0 : prev = vcl_session_get (wrk, s->vep.lt_prev);
1506 0 : next = vcl_session_get (wrk, s->vep.lt_next);
1507 :
1508 0 : prev->vep.lt_next = next->session_index;
1509 0 : next->vep.lt_prev = prev->session_index;
1510 :
1511 0 : if (s->session_index == wrk->ep_lt_current)
1512 0 : wrk->ep_lt_current = s->vep.lt_next;
1513 :
1514 0 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
1515 0 : s->vep.lt_prev = VCL_INVALID_SESSION_INDEX;
1516 : }
1517 :
1518 : int
1519 87 : vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * s,
1520 : vcl_session_handle_t sh, u8 do_disconnect)
1521 : {
1522 87 : int rv = VPPCOM_OK;
1523 :
1524 87 : VDBG (1, "session %u [0x%llx] closing", s->session_index, s->vpp_handle);
1525 :
1526 87 : if (s->flags & VCL_SESSION_F_IS_VEP)
1527 : {
1528 0 : u32 next_sh = s->vep.next_sh;
1529 0 : while (next_sh != ~0)
1530 : {
1531 0 : rv = vppcom_epoll_ctl (sh, EPOLL_CTL_DEL, next_sh, 0);
1532 0 : if (PREDICT_FALSE (rv < 0))
1533 0 : VDBG (0, "vpp handle 0x%llx, sh %u: EPOLL_CTL_DEL vep_idx %u"
1534 : " failed! rv %d (%s)", s->vpp_handle, next_sh,
1535 : s->vep.vep_sh, rv, vppcom_retval_str (rv));
1536 0 : next_sh = s->vep.next_sh;
1537 : }
1538 0 : goto free_session;
1539 : }
1540 :
1541 87 : if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
1542 : {
1543 41 : rv = vppcom_epoll_ctl (s->vep.vep_sh, EPOLL_CTL_DEL, sh, 0);
1544 41 : if (rv < 0)
1545 0 : VDBG (0, "session %u [0x%llx]: EPOLL_CTL_DEL vep_idx %u "
1546 : "failed! rv %d (%s)", s->session_index, s->vpp_handle,
1547 : s->vep.vep_sh, rv, vppcom_retval_str (rv));
1548 : }
1549 :
1550 87 : if (!do_disconnect)
1551 : {
1552 0 : VDBG (1, "session %u [0x%llx] disconnect skipped",
1553 : s->session_index, s->vpp_handle);
1554 0 : goto cleanup;
1555 : }
1556 :
1557 87 : if (s->session_state == VCL_STATE_LISTEN)
1558 : {
1559 21 : rv = vppcom_session_unbind (sh);
1560 21 : if (PREDICT_FALSE (rv < 0))
1561 0 : VDBG (0, "session %u [0x%llx]: listener unbind failed! "
1562 : "rv %d (%s)", s->session_index, s->vpp_handle, rv,
1563 : vppcom_retval_str (rv));
1564 21 : return rv;
1565 : }
1566 66 : else if (vcl_session_is_ready (s)
1567 4 : || (vcl_session_is_connectable_listener (wrk, s)))
1568 : {
1569 62 : rv = vppcom_session_disconnect (sh);
1570 62 : if (PREDICT_FALSE (rv < 0))
1571 0 : VDBG (0, "ERROR: session %u [0x%llx]: disconnect failed!"
1572 : " rv %d (%s)", s->session_index, s->vpp_handle,
1573 : rv, vppcom_retval_str (rv));
1574 : }
1575 4 : else if (s->session_state == VCL_STATE_DISCONNECT)
1576 : {
1577 4 : vcl_send_session_reset_reply (wrk, s, 0);
1578 : }
1579 0 : else if (s->session_state == VCL_STATE_DETACHED)
1580 : {
1581 0 : VDBG (0, "vpp freed session %d before close", s->session_index);
1582 :
1583 0 : if (!(s->flags & VCL_SESSION_F_PENDING_DISCONNECT))
1584 0 : goto free_session;
1585 :
1586 : /* Disconnect/reset messages pending but vpp transport and session
1587 : * cleanups already done. Free only after messages drained. */
1588 0 : s->flags |= VCL_SESSION_F_PENDING_FREE;
1589 : }
1590 :
1591 66 : s->session_state = VCL_STATE_CLOSED;
1592 :
1593 : /* Session is removed only after vpp confirms the disconnect */
1594 66 : return rv;
1595 :
1596 0 : cleanup:
1597 0 : vcl_session_table_del_vpp_handle (wrk, s->vpp_handle);
1598 0 : free_session:
1599 0 : vcl_session_free (wrk, s);
1600 : vcl_evt (VCL_EVT_CLOSE, s, rv);
1601 :
1602 0 : return rv;
1603 : }
1604 :
1605 : int
1606 87 : vppcom_session_close (uint32_t session_handle)
1607 : {
1608 87 : vcl_worker_t *wrk = vcl_worker_get_current ();
1609 : vcl_session_t *session;
1610 :
1611 87 : session = vcl_session_get_w_handle (wrk, session_handle);
1612 87 : if (!session)
1613 0 : return VPPCOM_EBADFD;
1614 87 : return vcl_session_cleanup (wrk, session, session_handle,
1615 : 1 /* do_disconnect */ );
1616 : }
1617 :
1618 : int
1619 40 : vppcom_session_bind (uint32_t session_handle, vppcom_endpt_t * ep)
1620 : {
1621 40 : vcl_worker_t *wrk = vcl_worker_get_current ();
1622 40 : vcl_session_t *session = 0;
1623 :
1624 40 : if (!ep || !ep->ip)
1625 0 : return VPPCOM_EINVAL;
1626 :
1627 40 : session = vcl_session_get_w_handle (wrk, session_handle);
1628 40 : if (!session)
1629 0 : return VPPCOM_EBADFD;
1630 :
1631 40 : if (session->flags & VCL_SESSION_F_IS_VEP)
1632 : {
1633 0 : VDBG (0, "ERROR: cannot bind to epoll session %u!",
1634 : session->session_index);
1635 0 : return VPPCOM_EBADFD;
1636 : }
1637 :
1638 40 : session->transport.is_ip4 = ep->is_ip4;
1639 40 : if (ep->is_ip4)
1640 34 : clib_memcpy_fast (&session->transport.lcl_ip.ip4, ep->ip,
1641 : sizeof (ip4_address_t));
1642 : else
1643 6 : clib_memcpy_fast (&session->transport.lcl_ip.ip6, ep->ip,
1644 : sizeof (ip6_address_t));
1645 40 : session->transport.lcl_port = ep->port;
1646 :
1647 40 : VDBG (0,
1648 : "session %u handle %u: binding to local %s address %U port %u, "
1649 : "proto %s",
1650 : session->session_index, session_handle,
1651 : session->transport.is_ip4 ? "IPv4" : "IPv6", vcl_format_ip46_address,
1652 : &session->transport.lcl_ip,
1653 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1654 : clib_net_to_host_u16 (session->transport.lcl_port),
1655 : vppcom_proto_str (session->session_type));
1656 : vcl_evt (VCL_EVT_BIND, session);
1657 :
1658 40 : if (session->session_type == VPPCOM_PROTO_UDP)
1659 3 : vppcom_session_listen (session_handle, 10);
1660 :
1661 40 : return VPPCOM_OK;
1662 : }
1663 :
1664 : int
1665 40 : vppcom_session_listen (uint32_t listen_sh, uint32_t q_len)
1666 : {
1667 40 : vcl_worker_t *wrk = vcl_worker_get_current ();
1668 40 : vcl_session_t *listen_session = 0;
1669 : u64 listen_vpp_handle;
1670 : int rv;
1671 :
1672 40 : listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1673 40 : if (!listen_session || (listen_session->flags & VCL_SESSION_F_IS_VEP))
1674 0 : return VPPCOM_EBADFD;
1675 :
1676 40 : listen_vpp_handle = listen_session->vpp_handle;
1677 40 : if (listen_session->session_state == VCL_STATE_LISTEN)
1678 : {
1679 0 : VDBG (0, "session %u [0x%llx]: already in listen state!",
1680 : listen_sh, listen_vpp_handle);
1681 0 : return VPPCOM_OK;
1682 : }
1683 :
1684 40 : VDBG (0, "session %u: sending vpp listen request...", listen_sh);
1685 :
1686 : /*
1687 : * Send listen request to vpp and wait for reply
1688 : */
1689 40 : vcl_send_session_listen (wrk, listen_session);
1690 40 : rv = vppcom_wait_for_session_state_change (listen_session->session_index,
1691 : VCL_STATE_LISTEN,
1692 40 : vcm->cfg.session_timeout);
1693 :
1694 40 : if (PREDICT_FALSE (rv))
1695 : {
1696 0 : listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1697 0 : VDBG (0, "session %u [0x%llx]: listen failed! returning %d (%s)",
1698 : listen_sh, listen_session->vpp_handle, rv,
1699 : vppcom_retval_str (rv));
1700 0 : return rv;
1701 : }
1702 :
1703 40 : return VPPCOM_OK;
1704 : }
1705 :
1706 : int
1707 2 : vppcom_unformat_proto (uint8_t * proto, char *proto_str)
1708 : {
1709 2 : if (!strcmp (proto_str, "TCP"))
1710 0 : *proto = VPPCOM_PROTO_TCP;
1711 2 : else if (!strcmp (proto_str, "tcp"))
1712 0 : *proto = VPPCOM_PROTO_TCP;
1713 2 : else if (!strcmp (proto_str, "UDP"))
1714 0 : *proto = VPPCOM_PROTO_UDP;
1715 2 : else if (!strcmp (proto_str, "udp"))
1716 0 : *proto = VPPCOM_PROTO_UDP;
1717 2 : else if (!strcmp (proto_str, "TLS"))
1718 0 : *proto = VPPCOM_PROTO_TLS;
1719 2 : else if (!strcmp (proto_str, "tls"))
1720 0 : *proto = VPPCOM_PROTO_TLS;
1721 2 : else if (!strcmp (proto_str, "QUIC"))
1722 0 : *proto = VPPCOM_PROTO_QUIC;
1723 2 : else if (!strcmp (proto_str, "quic"))
1724 0 : *proto = VPPCOM_PROTO_QUIC;
1725 2 : else if (!strcmp (proto_str, "DTLS"))
1726 0 : *proto = VPPCOM_PROTO_DTLS;
1727 2 : else if (!strcmp (proto_str, "dtls"))
1728 2 : *proto = VPPCOM_PROTO_DTLS;
1729 0 : else if (!strcmp (proto_str, "SRTP"))
1730 0 : *proto = VPPCOM_PROTO_SRTP;
1731 0 : else if (!strcmp (proto_str, "srtp"))
1732 0 : *proto = VPPCOM_PROTO_SRTP;
1733 : else
1734 0 : return 1;
1735 2 : return 0;
1736 : }
1737 :
1738 : int
1739 37 : vppcom_session_accept (uint32_t ls_handle, vppcom_endpt_t *ep, uint32_t flags)
1740 : {
1741 37 : u32 client_session_index = ~0, ls_index, accept_flags = 0;
1742 37 : vcl_worker_t *wrk = vcl_worker_get_current ();
1743 : session_accepted_msg_t accepted_msg;
1744 37 : vcl_session_t *ls, *client_session = 0;
1745 : vcl_session_msg_t *evt;
1746 : u8 is_nonblocking;
1747 :
1748 37 : again:
1749 :
1750 37 : ls = vcl_session_get_w_handle (wrk, ls_handle);
1751 37 : if (!ls)
1752 0 : return VPPCOM_EBADFD;
1753 :
1754 37 : if ((ls->session_state != VCL_STATE_LISTEN) &&
1755 0 : (ls->session_state != VCL_STATE_LISTEN_NO_MQ) &&
1756 0 : (!vcl_session_is_connectable_listener (wrk, ls)))
1757 : {
1758 0 : VDBG (0, "ERROR: session [0x%llx]: not in listen state! state (%s)",
1759 : ls->vpp_handle, vcl_session_state_str (ls->session_state));
1760 0 : return VPPCOM_EBADFD;
1761 : }
1762 :
1763 37 : ls_index = ls->session_index;
1764 :
1765 37 : if (clib_fifo_elts (ls->accept_evts_fifo))
1766 : {
1767 37 : clib_fifo_sub2 (ls->accept_evts_fifo, evt);
1768 37 : accept_flags = evt->flags;
1769 37 : accepted_msg = evt->accepted_msg;
1770 37 : goto handle;
1771 : }
1772 :
1773 0 : is_nonblocking = vcl_session_has_attr (ls, VCL_SESS_ATTR_NONBLOCK);
1774 : while (1)
1775 : {
1776 0 : if (svm_msg_q_is_empty (wrk->app_event_queue) && is_nonblocking)
1777 0 : return VPPCOM_EAGAIN;
1778 :
1779 0 : svm_msg_q_wait (wrk->app_event_queue, SVM_MQ_WAIT_EMPTY);
1780 0 : vcl_worker_flush_mq_events (wrk);
1781 0 : goto again;
1782 : }
1783 :
1784 37 : handle:
1785 :
1786 : client_session_index =
1787 37 : vcl_session_accepted_handler (wrk, &accepted_msg, ls_index);
1788 37 : if (client_session_index == VCL_INVALID_SESSION_INDEX)
1789 0 : return VPPCOM_ECONNABORTED;
1790 :
1791 37 : ls = vcl_session_get (wrk, ls_index);
1792 37 : client_session = vcl_session_get (wrk, client_session_index);
1793 :
1794 37 : if (flags & O_NONBLOCK)
1795 0 : vcl_session_set_attr (client_session, VCL_SESS_ATTR_NONBLOCK);
1796 :
1797 37 : VDBG (1,
1798 : "listener %u [0x%llx]: Got a connect request! session %u [0x%llx],"
1799 : " flags %d, is_nonblocking %u",
1800 : ls->session_index, ls->vpp_handle, client_session_index,
1801 : client_session->vpp_handle, flags,
1802 : vcl_session_has_attr (client_session, VCL_SESS_ATTR_NONBLOCK));
1803 :
1804 37 : if (ep)
1805 : {
1806 37 : ep->is_ip4 = client_session->transport.is_ip4;
1807 37 : ep->port = client_session->transport.rmt_port;
1808 37 : if (client_session->transport.is_ip4)
1809 33 : clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip4,
1810 : sizeof (ip4_address_t));
1811 : else
1812 4 : clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip6,
1813 : sizeof (ip6_address_t));
1814 : }
1815 :
1816 37 : VDBG (0,
1817 : "listener %u [0x%llx] accepted %u [0x%llx] peer: %U:%u "
1818 : "local: %U:%u",
1819 : ls_handle, ls->vpp_handle, client_session_index,
1820 : client_session->vpp_handle, vcl_format_ip46_address,
1821 : &client_session->transport.rmt_ip,
1822 : client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1823 : clib_net_to_host_u16 (client_session->transport.rmt_port),
1824 : vcl_format_ip46_address, &client_session->transport.lcl_ip,
1825 : client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1826 : clib_net_to_host_u16 (client_session->transport.lcl_port));
1827 : vcl_evt (VCL_EVT_ACCEPT, client_session, ls, client_session_index);
1828 :
1829 : /*
1830 : * Session might have been closed already
1831 : */
1832 37 : if (accept_flags)
1833 : {
1834 0 : if (accept_flags & VCL_ACCEPTED_F_CLOSED)
1835 0 : client_session->session_state = VCL_STATE_VPP_CLOSING;
1836 0 : else if (accept_flags & VCL_ACCEPTED_F_RESET)
1837 0 : client_session->session_state = VCL_STATE_DISCONNECT;
1838 : }
1839 37 : return vcl_session_handle (client_session);
1840 : }
1841 :
1842 : int
1843 47 : vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep)
1844 : {
1845 47 : vcl_worker_t *wrk = vcl_worker_get_current ();
1846 47 : vcl_session_t *session = 0;
1847 : u32 session_index;
1848 : int rv;
1849 :
1850 47 : session = vcl_session_get_w_handle (wrk, session_handle);
1851 47 : if (!session)
1852 0 : return VPPCOM_EBADFD;
1853 47 : session_index = session->session_index;
1854 :
1855 47 : if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
1856 : {
1857 0 : VWRN ("cannot connect epoll session %u!", session->session_index);
1858 0 : return VPPCOM_EBADFD;
1859 : }
1860 :
1861 47 : if (PREDICT_FALSE (vcl_session_is_ready (session)))
1862 : {
1863 0 : VDBG (0,
1864 : "session %u [0x%llx]: already connected to %U:%d proto %s,"
1865 : " state (%s)",
1866 : session->session_index, session->vpp_handle,
1867 : vcl_format_ip46_address, &session->transport.rmt_ip,
1868 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1869 : clib_net_to_host_u16 (session->transport.rmt_port),
1870 : vppcom_proto_str (session->session_type),
1871 : vcl_session_state_str (session->session_state));
1872 0 : return VPPCOM_OK;
1873 : }
1874 :
1875 : /* Attempt to connect a connectionless listener */
1876 47 : if (PREDICT_FALSE (session->session_state == VCL_STATE_LISTEN))
1877 : {
1878 2 : if (session->session_type != VPPCOM_PROTO_UDP)
1879 0 : return VPPCOM_EINVAL;
1880 2 : vcl_send_session_unlisten (wrk, session);
1881 2 : session->session_state = VCL_STATE_CLOSED;
1882 : }
1883 :
1884 47 : session->transport.is_ip4 = server_ep->is_ip4;
1885 47 : vcl_ip_copy_from_ep (&session->transport.rmt_ip, server_ep);
1886 47 : session->transport.rmt_port = server_ep->port;
1887 47 : session->parent_handle = VCL_INVALID_SESSION_HANDLE;
1888 47 : session->flags |= VCL_SESSION_F_CONNECTED;
1889 :
1890 47 : VDBG (0, "session %u: connecting to peer %U:%d proto %s",
1891 : session->session_index, vcl_format_ip46_address,
1892 : &session->transport.rmt_ip,
1893 : session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1894 : clib_net_to_host_u16 (session->transport.rmt_port),
1895 : vppcom_proto_str (session->session_type));
1896 :
1897 47 : vcl_send_session_connect (wrk, session);
1898 :
1899 47 : if (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK))
1900 : {
1901 : /* State set to STATE_UPDATED to ensure the session is not assumed
1902 : * to be ready and to also allow the app to close it prior to vpp's
1903 : * connected reply. */
1904 0 : session->session_state = VCL_STATE_UPDATED;
1905 0 : return VPPCOM_EINPROGRESS;
1906 : }
1907 :
1908 : /*
1909 : * Wait for reply from vpp if blocking
1910 : */
1911 47 : rv = vppcom_wait_for_session_state_change (session_index, VCL_STATE_READY,
1912 47 : vcm->cfg.session_timeout);
1913 :
1914 47 : session = vcl_session_get (wrk, session_index);
1915 47 : VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1916 : session->vpp_handle, rv ? "failed" : "succeeded");
1917 :
1918 47 : return rv;
1919 : }
1920 :
1921 : int
1922 0 : vppcom_session_stream_connect (uint32_t session_handle,
1923 : uint32_t parent_session_handle)
1924 : {
1925 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
1926 : vcl_session_t *session, *parent_session;
1927 : u32 session_index, parent_session_index;
1928 : int rv;
1929 :
1930 0 : session = vcl_session_get_w_handle (wrk, session_handle);
1931 0 : if (!session)
1932 0 : return VPPCOM_EBADFD;
1933 0 : parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1934 0 : if (!parent_session)
1935 0 : return VPPCOM_EBADFD;
1936 :
1937 0 : session_index = session->session_index;
1938 0 : parent_session_index = parent_session->session_index;
1939 0 : if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
1940 : {
1941 0 : VDBG (0, "ERROR: cannot connect epoll session %u!",
1942 : session->session_index);
1943 0 : return VPPCOM_EBADFD;
1944 : }
1945 :
1946 0 : if (PREDICT_FALSE (vcl_session_is_ready (session)))
1947 : {
1948 0 : VDBG (0,
1949 : "session handle %u [0x%llx]: session already "
1950 : "connected to session %u [0x%llx] proto %s, state 0x%x (%s)",
1951 : session_handle, session->vpp_handle, parent_session_handle,
1952 : parent_session->vpp_handle,
1953 : vppcom_proto_str (session->session_type), session->session_state,
1954 : vcl_session_state_str (session->session_state));
1955 0 : return VPPCOM_OK;
1956 : }
1957 :
1958 : /* Connect to quic session specifics */
1959 0 : session->transport.is_ip4 = parent_session->transport.is_ip4;
1960 0 : session->transport.rmt_ip.ip4.as_u32 = (uint32_t) 1;
1961 0 : session->transport.rmt_port = 0;
1962 0 : session->parent_handle = parent_session->vpp_handle;
1963 :
1964 0 : VDBG (0, "session handle %u: connecting to session %u [0x%llx]",
1965 : session_handle, parent_session_handle, parent_session->vpp_handle);
1966 :
1967 : /*
1968 : * Send connect request and wait for reply from vpp
1969 : */
1970 0 : vcl_send_session_connect (wrk, session);
1971 0 : rv = vppcom_wait_for_session_state_change (session_index, VCL_STATE_READY,
1972 0 : vcm->cfg.session_timeout);
1973 :
1974 0 : session->listener_index = parent_session_index;
1975 0 : parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1976 0 : if (parent_session)
1977 0 : parent_session->n_accepted_sessions++;
1978 :
1979 0 : session = vcl_session_get (wrk, session_index);
1980 0 : VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1981 : session->vpp_handle, rv ? "failed" : "succeeded");
1982 :
1983 0 : return rv;
1984 : }
1985 :
1986 : static inline int
1987 719687 : vppcom_session_read_internal (uint32_t session_handle, void *buf, int n,
1988 : u8 peek)
1989 : {
1990 719687 : vcl_worker_t *wrk = vcl_worker_get_current ();
1991 719687 : int rv, n_read = 0, is_nonblocking;
1992 719687 : vcl_session_t *s = 0;
1993 : svm_fifo_t *rx_fifo;
1994 : session_event_t *e;
1995 : svm_msg_q_t *mq;
1996 : u8 is_ct;
1997 :
1998 719687 : if (PREDICT_FALSE (!buf))
1999 0 : return VPPCOM_EFAULT;
2000 :
2001 719687 : s = vcl_session_get_w_handle (wrk, session_handle);
2002 719687 : if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2003 0 : return VPPCOM_EBADFD;
2004 :
2005 719687 : if (PREDICT_FALSE (!vcl_session_is_open (s)))
2006 : {
2007 0 : VDBG (0, "session %u[0x%llx] is not open! state 0x%x (%s)",
2008 : s->session_index, s->vpp_handle, s->session_state,
2009 : vcl_session_state_str (s->session_state));
2010 0 : return vcl_session_closed_error (s);
2011 : }
2012 :
2013 719687 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_RD_SHUTDOWN))
2014 : {
2015 : /* Vpp would ack the incoming data and enqueue it for reading.
2016 : * So even if SHUT_RD is set, we can still read() the data if
2017 : * the session is ready.
2018 : */
2019 0 : if (!vcl_session_read_ready (s))
2020 : {
2021 0 : return 0;
2022 : }
2023 : }
2024 :
2025 719687 : is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2026 719687 : is_ct = vcl_session_is_ct (s);
2027 719687 : mq = wrk->app_event_queue;
2028 719687 : rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
2029 719687 : s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
2030 :
2031 719687 : if (svm_fifo_is_empty_cons (rx_fifo))
2032 : {
2033 261251 : if (is_nonblocking)
2034 : {
2035 261179 : if (vcl_session_is_closing (s))
2036 0 : return vcl_session_closing_error (s);
2037 261179 : if (is_ct)
2038 0 : svm_fifo_unset_event (s->rx_fifo);
2039 261179 : svm_fifo_unset_event (rx_fifo);
2040 261179 : return VPPCOM_EWOULDBLOCK;
2041 : }
2042 167 : while (svm_fifo_is_empty_cons (rx_fifo))
2043 : {
2044 95 : if (vcl_session_is_closing (s))
2045 0 : return vcl_session_closing_error (s);
2046 :
2047 95 : if (is_ct)
2048 22 : svm_fifo_unset_event (s->rx_fifo);
2049 95 : svm_fifo_unset_event (rx_fifo);
2050 :
2051 95 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2052 95 : vcl_worker_flush_mq_events (wrk);
2053 : }
2054 : }
2055 :
2056 458508 : read_again:
2057 :
2058 766779 : if (s->is_dgram)
2059 1380 : rv = app_recv_dgram_raw (rx_fifo, buf, n, &s->transport, 0, peek);
2060 : else
2061 765399 : rv = app_recv_stream_raw (rx_fifo, buf, n, 0, peek);
2062 :
2063 766779 : ASSERT (rv >= 0);
2064 :
2065 766779 : if (peek)
2066 0 : return rv;
2067 :
2068 766779 : n_read += rv;
2069 :
2070 766779 : if (svm_fifo_is_empty_cons (rx_fifo))
2071 : {
2072 401818 : if (is_ct)
2073 552 : svm_fifo_unset_event (s->rx_fifo);
2074 401818 : svm_fifo_unset_event (rx_fifo);
2075 401818 : if (!svm_fifo_is_empty_cons (rx_fifo)
2076 15426 : && svm_fifo_set_event (rx_fifo) && is_nonblocking)
2077 : {
2078 15398 : vec_add2 (wrk->unhandled_evts_vector, e, 1);
2079 15398 : e->event_type = SESSION_IO_EVT_RX;
2080 15398 : e->session_index = s->session_index;
2081 : }
2082 : }
2083 364961 : else if (PREDICT_FALSE (rv < n && !s->is_dgram))
2084 : {
2085 : /* More data enqueued while reading. Try to drain it
2086 : * or fill the buffer. Avoid doing that for dgrams */
2087 308271 : buf += rv;
2088 308271 : n -= rv;
2089 308271 : goto read_again;
2090 : }
2091 :
2092 458508 : if (PREDICT_FALSE (svm_fifo_needs_deq_ntf (rx_fifo, n_read)))
2093 : {
2094 13 : svm_fifo_clear_deq_ntf (rx_fifo);
2095 13 : app_send_io_evt_to_vpp (s->vpp_evt_q,
2096 13 : s->rx_fifo->shr->master_session_index,
2097 : SESSION_IO_EVT_RX, SVM_Q_WAIT);
2098 : }
2099 :
2100 458508 : VDBG (2, "session %u[0x%llx]: read %d bytes from (%p)", s->session_index,
2101 : s->vpp_handle, n_read, rx_fifo);
2102 :
2103 458508 : return n_read;
2104 : }
2105 :
2106 : int
2107 719687 : vppcom_session_read (uint32_t session_handle, void *buf, size_t n)
2108 : {
2109 719687 : return (vppcom_session_read_internal (session_handle, buf, n, 0));
2110 : }
2111 :
2112 : static int
2113 0 : vppcom_session_peek (uint32_t session_handle, void *buf, int n)
2114 : {
2115 0 : return (vppcom_session_read_internal (session_handle, buf, n, 1));
2116 : }
2117 :
2118 : int
2119 0 : vppcom_session_read_segments (uint32_t session_handle,
2120 : vppcom_data_segment_t * ds, uint32_t n_segments,
2121 : uint32_t max_bytes)
2122 : {
2123 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
2124 0 : int n_read = 0, is_nonblocking;
2125 0 : vcl_session_t *s = 0;
2126 : svm_fifo_t *rx_fifo;
2127 : svm_msg_q_t *mq;
2128 : u8 is_ct;
2129 :
2130 0 : s = vcl_session_get_w_handle (wrk, session_handle);
2131 0 : if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2132 0 : return VPPCOM_EBADFD;
2133 :
2134 0 : if (PREDICT_FALSE (!vcl_session_is_open (s)))
2135 0 : return vcl_session_closed_error (s);
2136 :
2137 0 : is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2138 0 : is_ct = vcl_session_is_ct (s);
2139 0 : mq = wrk->app_event_queue;
2140 0 : rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
2141 0 : s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
2142 :
2143 0 : if (svm_fifo_is_empty_cons (rx_fifo))
2144 : {
2145 0 : if (is_nonblocking)
2146 : {
2147 0 : if (is_ct)
2148 0 : svm_fifo_unset_event (s->rx_fifo);
2149 0 : svm_fifo_unset_event (rx_fifo);
2150 0 : return VPPCOM_EWOULDBLOCK;
2151 : }
2152 0 : while (svm_fifo_is_empty_cons (rx_fifo))
2153 : {
2154 0 : if (vcl_session_is_closing (s))
2155 0 : return vcl_session_closing_error (s);
2156 :
2157 0 : if (is_ct)
2158 0 : svm_fifo_unset_event (s->rx_fifo);
2159 0 : svm_fifo_unset_event (rx_fifo);
2160 :
2161 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2162 0 : vcl_worker_flush_mq_events (wrk);
2163 : }
2164 : }
2165 :
2166 0 : n_read = svm_fifo_segments (rx_fifo, s->rx_bytes_pending,
2167 : (svm_fifo_seg_t *) ds, &n_segments, max_bytes);
2168 0 : if (n_read < 0)
2169 0 : return VPPCOM_EAGAIN;
2170 :
2171 0 : if (svm_fifo_max_dequeue_cons (rx_fifo) == n_read)
2172 : {
2173 0 : if (is_ct)
2174 0 : svm_fifo_unset_event (s->rx_fifo);
2175 0 : svm_fifo_unset_event (rx_fifo);
2176 0 : if (svm_fifo_max_dequeue_cons (rx_fifo) != n_read
2177 0 : && svm_fifo_set_event (rx_fifo)
2178 0 : && vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
2179 : {
2180 : session_event_t *e;
2181 0 : vec_add2 (wrk->unhandled_evts_vector, e, 1);
2182 0 : e->event_type = SESSION_IO_EVT_RX;
2183 0 : e->session_index = s->session_index;
2184 : }
2185 : }
2186 :
2187 0 : s->rx_bytes_pending += n_read;
2188 0 : return n_read;
2189 : }
2190 :
2191 : void
2192 0 : vppcom_session_free_segments (uint32_t session_handle, uint32_t n_bytes)
2193 : {
2194 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
2195 : vcl_session_t *s;
2196 : u8 is_ct;
2197 :
2198 0 : s = vcl_session_get_w_handle (wrk, session_handle);
2199 0 : if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2200 0 : return;
2201 :
2202 0 : is_ct = vcl_session_is_ct (s);
2203 0 : svm_fifo_dequeue_drop (is_ct ? s->ct_rx_fifo : s->rx_fifo, n_bytes);
2204 :
2205 0 : ASSERT (s->rx_bytes_pending >= n_bytes);
2206 0 : s->rx_bytes_pending -= n_bytes;
2207 : }
2208 :
2209 : always_inline u8
2210 846364 : vcl_fifo_is_writeable (svm_fifo_t * f, u32 len, u8 is_dgram)
2211 : {
2212 846364 : u32 max_enq = svm_fifo_max_enqueue_prod (f);
2213 846364 : if (is_dgram)
2214 1380 : return max_enq >= (sizeof (session_dgram_hdr_t) + len);
2215 : else
2216 844984 : return max_enq > 0;
2217 : }
2218 :
2219 : always_inline int
2220 846364 : vppcom_session_write_inline (vcl_worker_t *wrk, vcl_session_t *s, void *buf,
2221 : size_t n, u8 is_flush, u8 is_dgram)
2222 : {
2223 : int n_write, is_nonblocking;
2224 : session_evt_type_t et;
2225 : svm_fifo_t *tx_fifo;
2226 : svm_msg_q_t *mq;
2227 : u8 is_ct;
2228 :
2229 : /* Accept zero length writes but just return */
2230 846364 : if (PREDICT_FALSE (!n))
2231 0 : return VPPCOM_OK;
2232 :
2233 846364 : if (PREDICT_FALSE (!buf))
2234 0 : return VPPCOM_EFAULT;
2235 :
2236 846364 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2237 : {
2238 0 : VDBG (0, "ERROR: session %u [0x%llx]: cannot write to an epoll"
2239 : " session!", s->session_index, s->vpp_handle);
2240 0 : return VPPCOM_EBADFD;
2241 : }
2242 :
2243 846364 : if (PREDICT_FALSE (!vcl_session_is_open (s)))
2244 : {
2245 0 : VDBG (1, "session %u [0x%llx]: is not open! state 0x%x (%s)",
2246 : s->session_index, s->vpp_handle, s->session_state,
2247 : vcl_session_state_str (s->session_state));
2248 0 : return vcl_session_closed_error (s);;
2249 : }
2250 :
2251 846364 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_WR_SHUTDOWN))
2252 : {
2253 0 : VDBG (1, "session %u [0x%llx]: is shutdown! state 0x%x (%s)",
2254 : s->session_index, s->vpp_handle, s->session_state,
2255 : vcl_session_state_str (s->session_state));
2256 0 : return VPPCOM_EPIPE;
2257 : }
2258 :
2259 846364 : is_ct = vcl_session_is_ct (s);
2260 846364 : tx_fifo = is_ct ? s->ct_tx_fifo : s->tx_fifo;
2261 846364 : is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2262 :
2263 846364 : mq = wrk->app_event_queue;
2264 846364 : if (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2265 : {
2266 682715 : if (is_nonblocking)
2267 : {
2268 682715 : return VPPCOM_EWOULDBLOCK;
2269 : }
2270 0 : while (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2271 : {
2272 0 : svm_fifo_add_want_deq_ntf (tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
2273 0 : if (vcl_session_is_closing (s))
2274 0 : return vcl_session_closing_error (s);
2275 :
2276 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2277 0 : vcl_worker_flush_mq_events (wrk);
2278 : }
2279 : }
2280 :
2281 163649 : et = SESSION_IO_EVT_TX;
2282 163649 : if (is_flush && !is_ct)
2283 27667 : et = SESSION_IO_EVT_TX_FLUSH;
2284 :
2285 163649 : if (is_dgram)
2286 : {
2287 1380 : et = vcl_session_dgram_tx_evt (s, et);
2288 : n_write =
2289 1380 : app_send_dgram_raw_gso (tx_fifo, &s->transport, s->vpp_evt_q, buf, n,
2290 1380 : s->gso_size, et, 0 /* do_evt */, SVM_Q_WAIT);
2291 : }
2292 : else
2293 : {
2294 162269 : n_write = app_send_stream_raw (tx_fifo, s->vpp_evt_q, buf, n, et,
2295 : 0 /* do_evt */, SVM_Q_WAIT);
2296 : }
2297 :
2298 163649 : if (svm_fifo_set_event (s->tx_fifo))
2299 125261 : app_send_io_evt_to_vpp (
2300 125261 : s->vpp_evt_q, s->tx_fifo->shr->master_session_index, et, SVM_Q_WAIT);
2301 :
2302 : /* The underlying fifo segment can run out of memory */
2303 163649 : if (PREDICT_FALSE (n_write < 0))
2304 0 : return VPPCOM_EAGAIN;
2305 :
2306 163649 : VDBG (2, "session %u [0x%llx]: wrote %d bytes", s->session_index,
2307 : s->vpp_handle, n_write);
2308 :
2309 163649 : return n_write;
2310 : }
2311 :
2312 : int
2313 17235 : vppcom_session_write (uint32_t session_handle, void *buf, size_t n)
2314 : {
2315 17235 : vcl_worker_t *wrk = vcl_worker_get_current ();
2316 : vcl_session_t *s;
2317 :
2318 17235 : s = vcl_session_get_w_handle (wrk, session_handle);
2319 17235 : if (PREDICT_FALSE (!s))
2320 0 : return VPPCOM_EBADFD;
2321 :
2322 17235 : return vppcom_session_write_inline (wrk, s, buf, n, 0 /* is_flush */,
2323 17235 : s->is_dgram ? 1 : 0);
2324 : }
2325 :
2326 : int
2327 829129 : vppcom_session_write_msg (uint32_t session_handle, void *buf, size_t n)
2328 : {
2329 829129 : vcl_worker_t *wrk = vcl_worker_get_current ();
2330 : vcl_session_t *s;
2331 :
2332 829129 : s = vcl_session_get_w_handle (wrk, session_handle);
2333 829129 : if (PREDICT_FALSE (!s))
2334 0 : return VPPCOM_EBADFD;
2335 :
2336 829129 : return vppcom_session_write_inline (wrk, s, buf, n, 1 /* is_flush */,
2337 829129 : s->is_dgram ? 1 : 0);
2338 : }
2339 :
2340 : #define vcl_fifo_rx_evt_valid_or_break(_s) \
2341 : if (PREDICT_FALSE (!_s->rx_fifo)) \
2342 : break; \
2343 : if (PREDICT_FALSE (svm_fifo_is_empty (_s->rx_fifo))) \
2344 : { \
2345 : if (!vcl_session_is_ct (_s)) \
2346 : { \
2347 : svm_fifo_unset_event (_s->rx_fifo); \
2348 : if (svm_fifo_is_empty (_s->rx_fifo)) \
2349 : break; \
2350 : } \
2351 : else if (svm_fifo_is_empty (_s->ct_rx_fifo)) \
2352 : { \
2353 : svm_fifo_unset_event (_s->rx_fifo); /* rx evts on actual fifo*/ \
2354 : if (svm_fifo_is_empty (_s->ct_rx_fifo)) \
2355 : break; \
2356 : } \
2357 : } \
2358 :
2359 : static void
2360 121560 : vcl_select_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
2361 : unsigned long n_bits, unsigned long *read_map,
2362 : unsigned long *write_map,
2363 : unsigned long *except_map, u32 * bits_set)
2364 : {
2365 : session_disconnected_msg_t *disconnected_msg;
2366 : session_connected_msg_t *connected_msg;
2367 : vcl_session_t *s;
2368 : u32 sid;
2369 :
2370 121560 : switch (e->event_type)
2371 : {
2372 99273 : case SESSION_IO_EVT_RX:
2373 99273 : sid = e->session_index;
2374 99273 : s = vcl_session_get (wrk, sid);
2375 99273 : if (!s || !vcl_session_is_open (s))
2376 : break;
2377 99273 : vcl_fifo_rx_evt_valid_or_break (s);
2378 39589 : if (sid < n_bits && read_map)
2379 : {
2380 39589 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2381 39589 : *bits_set += 1;
2382 : }
2383 39589 : break;
2384 22269 : case SESSION_IO_EVT_TX:
2385 22269 : sid = e->session_index;
2386 22269 : s = vcl_session_get (wrk, sid);
2387 22269 : if (!s || !vcl_session_is_open (s))
2388 : break;
2389 22269 : if (sid < n_bits && write_map)
2390 : {
2391 22269 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2392 22269 : *bits_set += 1;
2393 : }
2394 22269 : break;
2395 9 : case SESSION_CTRL_EVT_ACCEPTED:
2396 9 : if (!e->postponed)
2397 9 : s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
2398 : else
2399 0 : s = vcl_session_get (wrk, e->session_index);
2400 9 : if (!s)
2401 0 : break;
2402 9 : sid = s->session_index;
2403 9 : if (sid < n_bits && read_map)
2404 : {
2405 9 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2406 9 : *bits_set += 1;
2407 : }
2408 9 : break;
2409 0 : case SESSION_CTRL_EVT_CONNECTED:
2410 0 : if (!e->postponed)
2411 : {
2412 0 : connected_msg = (session_connected_msg_t *) e->data;
2413 0 : sid = vcl_session_connected_handler (wrk, connected_msg);
2414 : }
2415 : else
2416 0 : sid = e->session_index;
2417 0 : if (sid == VCL_INVALID_SESSION_INDEX)
2418 0 : break;
2419 0 : if (!(sid < n_bits && write_map))
2420 : break;
2421 0 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2422 0 : *bits_set += 1;
2423 0 : s = vcl_session_get (wrk, sid);
2424 : /* We didn't have a fifo when the event was added */
2425 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2426 0 : break;
2427 0 : case SESSION_CTRL_EVT_DISCONNECTED:
2428 0 : disconnected_msg = (session_disconnected_msg_t *) e->data;
2429 0 : s = vcl_session_disconnected_handler (wrk, disconnected_msg);
2430 0 : if (!s)
2431 0 : break;
2432 0 : sid = s->session_index;
2433 0 : if (sid < n_bits && except_map)
2434 : {
2435 0 : clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2436 0 : *bits_set += 1;
2437 : }
2438 0 : break;
2439 0 : case SESSION_CTRL_EVT_RESET:
2440 0 : sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
2441 0 : if (sid < n_bits && except_map)
2442 : {
2443 0 : clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2444 0 : *bits_set += 1;
2445 : }
2446 0 : break;
2447 1 : case SESSION_CTRL_EVT_UNLISTEN_REPLY:
2448 1 : vcl_session_unlisten_reply_handler (wrk, e->data);
2449 1 : break;
2450 0 : case SESSION_CTRL_EVT_MIGRATED:
2451 0 : vcl_session_migrated_handler (wrk, e->data);
2452 0 : break;
2453 2 : case SESSION_CTRL_EVT_CLEANUP:
2454 2 : vcl_session_cleanup_handler (wrk, e->data);
2455 2 : break;
2456 0 : case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
2457 0 : vcl_session_worker_update_reply_handler (wrk, e->data);
2458 0 : break;
2459 0 : case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
2460 0 : vcl_session_req_worker_update_handler (wrk, e->data);
2461 0 : break;
2462 6 : case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
2463 6 : vcl_session_app_add_segment_handler (wrk, e->data);
2464 6 : break;
2465 0 : case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
2466 0 : vcl_session_app_del_segment_handler (wrk, e->data);
2467 0 : break;
2468 0 : case SESSION_CTRL_EVT_APP_WRK_RPC:
2469 0 : vcl_worker_rpc_handler (wrk, e->data);
2470 0 : break;
2471 0 : default:
2472 0 : clib_warning ("unhandled: %u", e->event_type);
2473 0 : break;
2474 : }
2475 0 : }
2476 :
2477 : static int
2478 26849800 : vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
2479 : unsigned long n_bits, unsigned long *read_map,
2480 : unsigned long *write_map, unsigned long *except_map,
2481 : double time_to_wait, u32 * bits_set)
2482 : {
2483 : svm_msg_q_msg_t *msg;
2484 : session_event_t *e;
2485 : u32 i;
2486 :
2487 26849800 : if (svm_msg_q_is_empty (mq))
2488 : {
2489 26744400 : if (*bits_set)
2490 443940 : return 0;
2491 :
2492 26300400 : if (!time_to_wait)
2493 26300400 : return 0;
2494 0 : else if (time_to_wait < 0)
2495 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
2496 : else
2497 : {
2498 0 : if (svm_msg_q_timedwait (mq, time_to_wait))
2499 0 : return 0;
2500 : }
2501 : }
2502 105461 : vcl_mq_dequeue_batch (wrk, mq, ~0);
2503 :
2504 211564 : for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
2505 : {
2506 106103 : msg = vec_elt_at_index (wrk->mq_msg_vector, i);
2507 106103 : e = svm_msg_q_msg_data (mq, msg);
2508 106103 : vcl_select_handle_mq_event (wrk, e, n_bits, read_map, write_map,
2509 : except_map, bits_set);
2510 106103 : svm_msg_q_free_msg (mq, msg);
2511 : }
2512 105461 : vec_reset_length (wrk->mq_msg_vector);
2513 105461 : vcl_handle_pending_wrk_updates (wrk);
2514 105461 : return *bits_set;
2515 : }
2516 :
2517 : static int
2518 26849800 : vppcom_select_condvar (vcl_worker_t * wrk, int n_bits,
2519 : vcl_si_set * read_map, vcl_si_set * write_map,
2520 : vcl_si_set * except_map, double time_to_wait,
2521 : u32 * bits_set)
2522 : {
2523 26849800 : double wait = 0, start = 0;
2524 :
2525 26849800 : if (!*bits_set)
2526 : {
2527 26380600 : wait = time_to_wait;
2528 26380600 : start = clib_time_now (&wrk->clib_time);
2529 : }
2530 :
2531 : do
2532 : {
2533 26849800 : vcl_select_handle_mq (wrk, wrk->app_event_queue, n_bits, read_map,
2534 : write_map, except_map, wait, bits_set);
2535 26849800 : if (*bits_set)
2536 492316 : return *bits_set;
2537 26357500 : if (wait == -1)
2538 0 : continue;
2539 :
2540 26357500 : wait = wait - (clib_time_now (&wrk->clib_time) - start);
2541 : }
2542 26357500 : while (wait > 0);
2543 :
2544 26357500 : return 0;
2545 : }
2546 :
2547 : static int
2548 0 : vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits,
2549 : vcl_si_set * read_map, vcl_si_set * write_map,
2550 : vcl_si_set * except_map, double time_to_wait,
2551 : u32 * bits_set)
2552 : {
2553 : vcl_mq_evt_conn_t *mqc;
2554 : int __clib_unused n_read;
2555 : int n_mq_evts, i;
2556 : u64 buf;
2557 :
2558 0 : if (PREDICT_FALSE (wrk->api_client_handle == ~0))
2559 : {
2560 0 : vcl_api_retry_attach (wrk);
2561 0 : return 0;
2562 : }
2563 :
2564 0 : vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
2565 0 : n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
2566 0 : vec_len (wrk->mq_events), time_to_wait);
2567 0 : for (i = 0; i < n_mq_evts; i++)
2568 : {
2569 0 : if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0))
2570 : {
2571 0 : vcl_api_handle_disconnect (wrk);
2572 0 : continue;
2573 : }
2574 :
2575 0 : mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
2576 0 : n_read = read (mqc->mq_fd, &buf, sizeof (buf));
2577 0 : vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map,
2578 : except_map, 0, bits_set);
2579 : }
2580 :
2581 0 : return (n_mq_evts > 0 ? (int) *bits_set : 0);
2582 : }
2583 :
2584 : int
2585 26849800 : vppcom_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
2586 : vcl_si_set * except_map, double time_to_wait)
2587 : {
2588 26849800 : u32 sid, minbits = clib_max (n_bits, BITS (uword)), bits_set = 0;
2589 26849800 : vcl_worker_t *wrk = vcl_worker_get_current ();
2590 26849800 : vcl_session_t *s = 0;
2591 : int i;
2592 :
2593 26849800 : if (n_bits && read_map)
2594 : {
2595 26848800 : clib_bitmap_validate (wrk->rd_bitmap, minbits);
2596 26848800 : clib_memcpy_fast (wrk->rd_bitmap, read_map,
2597 26848800 : vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2598 26848800 : memset (read_map, 0, vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2599 : }
2600 26849800 : if (n_bits && write_map)
2601 : {
2602 26849800 : clib_bitmap_validate (wrk->wr_bitmap, minbits);
2603 26849800 : clib_memcpy_fast (wrk->wr_bitmap, write_map,
2604 26849800 : vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2605 26849800 : memset (write_map, 0, vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2606 : }
2607 26849800 : if (n_bits && except_map)
2608 : {
2609 0 : clib_bitmap_validate (wrk->ex_bitmap, minbits);
2610 0 : clib_memcpy_fast (wrk->ex_bitmap, except_map,
2611 0 : vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2612 0 : memset (except_map, 0, vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2613 : }
2614 :
2615 26849800 : if (!n_bits)
2616 0 : return 0;
2617 :
2618 26849800 : if (!write_map)
2619 6 : goto check_rd;
2620 :
2621 34477000 : clib_bitmap_foreach (sid, wrk->wr_bitmap)
2622 : {
2623 7627220 : if (!(s = vcl_session_get (wrk, sid)))
2624 : {
2625 0 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2626 0 : bits_set++;
2627 0 : continue;
2628 : }
2629 :
2630 7627220 : if (vcl_session_write_ready (s))
2631 : {
2632 113756 : clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2633 113756 : bits_set++;
2634 : }
2635 : else
2636 : {
2637 7513460 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF);
2638 : }
2639 : }
2640 :
2641 26849800 : check_rd:
2642 26849800 : if (!read_map)
2643 1000 : goto check_mq;
2644 :
2645 77187900 : clib_bitmap_foreach (sid, wrk->rd_bitmap)
2646 : {
2647 50339000 : if (!(s = vcl_session_get (wrk, sid)))
2648 : {
2649 0 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2650 0 : bits_set++;
2651 0 : continue;
2652 : }
2653 :
2654 50339000 : if (vcl_session_read_ready (s))
2655 : {
2656 410891 : clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2657 410891 : bits_set++;
2658 : }
2659 : }
2660 :
2661 26848800 : check_mq:
2662 :
2663 26865300 : for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
2664 : {
2665 15457 : vcl_select_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i], n_bits,
2666 : read_map, write_map, except_map, &bits_set);
2667 : }
2668 26849800 : vec_reset_length (wrk->unhandled_evts_vector);
2669 :
2670 26849800 : if (vcm->cfg.use_mq_eventfd)
2671 0 : vppcom_select_eventfd (wrk, n_bits, read_map, write_map, except_map,
2672 : time_to_wait, &bits_set);
2673 : else
2674 26849800 : vppcom_select_condvar (wrk, n_bits, read_map, write_map, except_map,
2675 : time_to_wait, &bits_set);
2676 :
2677 26849800 : return (bits_set);
2678 : }
2679 :
2680 : static inline void
2681 94 : vep_verify_epoll_chain (vcl_worker_t * wrk, u32 vep_handle)
2682 : {
2683 : vppcom_epoll_t *vep;
2684 94 : u32 sh = vep_handle;
2685 : vcl_session_t *s;
2686 :
2687 94 : if (VPPCOM_DEBUG <= 3)
2688 94 : return;
2689 :
2690 0 : s = vcl_session_get_w_handle (wrk, vep_handle);
2691 0 : if (PREDICT_FALSE (!s))
2692 : {
2693 0 : VDBG (0, "ERROR: Invalid vep_sh (%u)!", vep_handle);
2694 0 : goto done;
2695 : }
2696 0 : if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP)))
2697 : {
2698 0 : VDBG (0, "ERROR: vep_sh (%u) is not a vep!", vep_handle);
2699 0 : goto done;
2700 : }
2701 0 : vep = &s->vep;
2702 0 : VDBG (0, "vep_sh (%u): Dumping epoll chain\n"
2703 : "{\n"
2704 : " is_vep = %u\n"
2705 : " is_vep_session = %u\n"
2706 : " next_sh = 0x%x (%u)\n"
2707 : "}\n", vep_handle, s->flags & VCL_SESSION_F_IS_VEP,
2708 : s->flags & VCL_SESSION_F_IS_VEP_SESSION, vep->next_sh, vep->next_sh);
2709 :
2710 0 : for (sh = vep->next_sh; sh != ~0; sh = vep->next_sh)
2711 : {
2712 0 : s = vcl_session_get_w_handle (wrk, sh);
2713 0 : if (PREDICT_FALSE (!s))
2714 : {
2715 0 : VDBG (0, "ERROR: Invalid sh (%u)!", sh);
2716 0 : goto done;
2717 : }
2718 0 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2719 : {
2720 0 : VDBG (0, "ERROR: sh (%u) is a vep!", vep_handle);
2721 : }
2722 0 : else if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2723 : {
2724 0 : VDBG (0, "ERROR: sh (%u) is not a vep session handle!", sh);
2725 0 : goto done;
2726 : }
2727 0 : vep = &s->vep;
2728 0 : if (PREDICT_FALSE (vep->vep_sh != vep_handle))
2729 0 : VDBG (0, "ERROR: session (%u) vep_sh (%u) != vep_sh (%u)!",
2730 : sh, s->vep.vep_sh, vep_handle);
2731 0 : if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
2732 : {
2733 0 : VDBG (0, "vep_sh[%u]: sh 0x%x (%u)\n"
2734 : "{\n"
2735 : " next_sh = 0x%x (%u)\n"
2736 : " prev_sh = 0x%x (%u)\n"
2737 : " vep_sh = 0x%x (%u)\n"
2738 : " ev.events = 0x%x\n"
2739 : " ev.data.u64 = 0x%llx\n"
2740 : " et_mask = 0x%x\n"
2741 : "}\n",
2742 : vep_handle, sh, sh, vep->next_sh, vep->next_sh, vep->prev_sh,
2743 : vep->prev_sh, vep->vep_sh, vep->vep_sh, vep->ev.events,
2744 : vep->ev.data.u64, vep->et_mask);
2745 : }
2746 : }
2747 :
2748 0 : done:
2749 0 : VDBG (0, "vep_sh (%u): Dump complete!\n", vep_handle);
2750 : }
2751 :
2752 : int
2753 13 : vppcom_epoll_create (void)
2754 : {
2755 13 : vcl_worker_t *wrk = vcl_worker_get_current ();
2756 : vcl_session_t *vep_session;
2757 :
2758 13 : vep_session = vcl_session_alloc (wrk);
2759 :
2760 13 : vep_session->flags |= VCL_SESSION_F_IS_VEP;
2761 13 : vep_session->vep.vep_sh = ~0;
2762 13 : vep_session->vep.next_sh = ~0;
2763 13 : vep_session->vep.prev_sh = ~0;
2764 13 : vep_session->vpp_handle = ~0;
2765 :
2766 : vcl_evt (VCL_EVT_EPOLL_CREATE, vep_session, vep_session->session_index);
2767 13 : VDBG (0, "Created vep_idx %u", vep_session->session_index);
2768 :
2769 13 : return vcl_session_handle (vep_session);
2770 : }
2771 :
2772 : static void
2773 0 : vcl_epoll_ctl_add_unhandled_event (vcl_worker_t *wrk, vcl_session_t *s,
2774 : u8 is_epollet, session_evt_type_t evt)
2775 : {
2776 0 : if (!is_epollet)
2777 : {
2778 0 : if (s->vep.lt_next == VCL_INVALID_SESSION_INDEX)
2779 0 : vcl_epoll_lt_add (wrk, s);
2780 0 : return;
2781 : }
2782 :
2783 0 : session_event_t e = { 0 };
2784 0 : e.session_index = s->session_index;
2785 0 : e.event_type = evt;
2786 0 : if (evt == SESSION_IO_EVT_RX)
2787 0 : s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
2788 0 : vec_add1 (wrk->unhandled_evts_vector, e);
2789 : }
2790 :
2791 : int
2792 94 : vppcom_epoll_ctl (uint32_t vep_handle, int op, uint32_t session_handle,
2793 : struct epoll_event *event)
2794 : {
2795 94 : vcl_worker_t *wrk = vcl_worker_get_current ();
2796 94 : int rv = VPPCOM_OK, add_evt = 0;
2797 : vcl_session_t *vep_session;
2798 : vcl_session_t *s;
2799 :
2800 94 : if (vep_handle == session_handle)
2801 : {
2802 0 : VDBG (0, "vep_sh == session handle (%u)!", vep_handle);
2803 0 : return VPPCOM_EINVAL;
2804 : }
2805 :
2806 94 : vep_session = vcl_session_get_w_handle (wrk, vep_handle);
2807 94 : if (PREDICT_FALSE (!vep_session))
2808 : {
2809 0 : VDBG (0, "Invalid vep_sh (%u)!", vep_handle);
2810 0 : return VPPCOM_EBADFD;
2811 : }
2812 94 : if (PREDICT_FALSE (!(vep_session->flags & VCL_SESSION_F_IS_VEP)))
2813 : {
2814 0 : VDBG (0, "vep_sh (%u) is not a vep!", vep_handle);
2815 0 : return VPPCOM_EINVAL;
2816 : }
2817 :
2818 94 : ASSERT (vep_session->vep.vep_sh == ~0);
2819 94 : ASSERT (vep_session->vep.prev_sh == ~0);
2820 :
2821 94 : s = vcl_session_get_w_handle (wrk, session_handle);
2822 94 : if (PREDICT_FALSE (!s))
2823 : {
2824 0 : VDBG (0, "Invalid session_handle (%u)!", session_handle);
2825 0 : return VPPCOM_EBADFD;
2826 : }
2827 94 : if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2828 : {
2829 0 : VDBG (0, "session_handle (%u) is a vep!", vep_handle);
2830 0 : return VPPCOM_EINVAL;
2831 : }
2832 :
2833 94 : switch (op)
2834 : {
2835 53 : case EPOLL_CTL_ADD:
2836 53 : if (PREDICT_FALSE (!event))
2837 : {
2838 0 : VDBG (0, "EPOLL_CTL_ADD: NULL pointer to epoll_event structure!");
2839 0 : return VPPCOM_EINVAL;
2840 : }
2841 53 : if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
2842 : {
2843 0 : VDBG (0, "EPOLL_CTL_ADD: %u already epolled!", s->session_index);
2844 0 : rv = VPPCOM_EEXIST;
2845 0 : goto done;
2846 : }
2847 53 : if (vep_session->vep.next_sh != ~0)
2848 : {
2849 : vcl_session_t *next_session;
2850 40 : next_session = vcl_session_get_w_handle (wrk,
2851 : vep_session->vep.next_sh);
2852 40 : if (PREDICT_FALSE (!next_session))
2853 : {
2854 0 : VDBG (0, "EPOLL_CTL_ADD: Invalid vep.next_sh (%u) on "
2855 : "vep_idx (%u)!", vep_session->vep.next_sh, vep_handle);
2856 0 : return VPPCOM_EBADFD;
2857 : }
2858 40 : ASSERT (next_session->vep.prev_sh == vep_handle);
2859 40 : next_session->vep.prev_sh = session_handle;
2860 : }
2861 53 : s->vep.next_sh = vep_session->vep.next_sh;
2862 53 : s->vep.prev_sh = vep_handle;
2863 53 : s->vep.vep_sh = vep_handle;
2864 53 : s->vep.et_mask = VEP_DEFAULT_ET_MASK;
2865 53 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
2866 53 : s->vep.ev = *event;
2867 53 : s->vep.ev.events |= EPOLLHUP | EPOLLERR;
2868 53 : s->flags &= ~VCL_SESSION_F_IS_VEP;
2869 53 : s->flags |= VCL_SESSION_F_IS_VEP_SESSION;
2870 53 : vep_session->vep.next_sh = session_handle;
2871 :
2872 53 : if ((event->events & EPOLLOUT))
2873 : {
2874 0 : int write_ready = vcl_session_write_ready (s);
2875 :
2876 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2877 0 : if (write_ready > 0)
2878 : {
2879 : /* Generate EPOLLOUT if tx fifo not full */
2880 0 : vcl_epoll_ctl_add_unhandled_event (
2881 : wrk, s, event->events & EPOLLET, SESSION_IO_EVT_TX);
2882 0 : add_evt = 1;
2883 : }
2884 : else
2885 : {
2886 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF);
2887 : }
2888 : }
2889 : /* Generate EPOLLIN if rx fifo has data */
2890 53 : if ((event->events & EPOLLIN) && (vcl_session_read_ready (s) > 0))
2891 : {
2892 0 : vcl_epoll_ctl_add_unhandled_event (wrk, s, event->events & EPOLLET,
2893 : SESSION_IO_EVT_RX);
2894 0 : add_evt = 1;
2895 : }
2896 53 : if (!add_evt && vcl_session_is_closing (s))
2897 : {
2898 0 : session_event_t e = { 0 };
2899 0 : if (s->session_state == VCL_STATE_VPP_CLOSING)
2900 0 : e.event_type = SESSION_CTRL_EVT_DISCONNECTED;
2901 : else
2902 0 : e.event_type = SESSION_CTRL_EVT_RESET;
2903 0 : e.session_index = s->session_index;
2904 0 : e.postponed = 1;
2905 0 : vec_add1 (wrk->unhandled_evts_vector, e);
2906 : }
2907 53 : VDBG (1, "EPOLL_CTL_ADD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2908 : vep_handle, session_handle, event->events, event->data.u64);
2909 : vcl_evt (VCL_EVT_EPOLL_CTLADD, s, event->events, event->data.u64);
2910 53 : break;
2911 :
2912 0 : case EPOLL_CTL_MOD:
2913 0 : if (PREDICT_FALSE (!event))
2914 : {
2915 0 : VDBG (0, "EPOLL_CTL_MOD: NULL pointer to epoll_event structure!");
2916 0 : rv = VPPCOM_EINVAL;
2917 0 : goto done;
2918 : }
2919 0 : else if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2920 : {
2921 0 : VDBG (0, "sh %u EPOLL_CTL_MOD: not a vep session!", session_handle);
2922 0 : rv = VPPCOM_ENOENT;
2923 0 : goto done;
2924 : }
2925 0 : else if (PREDICT_FALSE (s->vep.vep_sh != vep_handle))
2926 : {
2927 0 : VDBG (0, "EPOLL_CTL_MOD: sh %u vep_sh (%u) != vep_sh (%u)!",
2928 : session_handle, s->vep.vep_sh, vep_handle);
2929 0 : rv = VPPCOM_EINVAL;
2930 0 : goto done;
2931 : }
2932 :
2933 : /* Generate EPOLLOUT if session write ready and event was not on */
2934 0 : if ((event->events & EPOLLOUT) && !(s->vep.ev.events & EPOLLOUT))
2935 0 : {
2936 : /* Fifo size load acq synchronized with update store rel */
2937 0 : int write_ready = vcl_session_write_ready (s);
2938 :
2939 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2940 0 : if (write_ready > 0)
2941 0 : vcl_epoll_ctl_add_unhandled_event (wrk, s, event->events & EPOLLET,
2942 : SESSION_IO_EVT_TX);
2943 : else
2944 : /* Request deq ntf in case dequeue happened while updating flag */
2945 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF);
2946 : }
2947 0 : else if (!(event->events & EPOLLOUT))
2948 0 : vcl_session_del_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2949 :
2950 : /* Generate EPOLLIN if session read ready and event was not on */
2951 0 : if ((event->events & EPOLLIN) && !(s->vep.ev.events & EPOLLIN) &&
2952 0 : (vcl_session_read_ready (s) > 0))
2953 : {
2954 0 : vcl_epoll_ctl_add_unhandled_event (wrk, s, event->events & EPOLLET,
2955 : SESSION_IO_EVT_RX);
2956 : }
2957 0 : s->vep.et_mask = VEP_DEFAULT_ET_MASK;
2958 0 : s->vep.ev = *event;
2959 0 : s->vep.ev.events |= EPOLLHUP | EPOLLERR;
2960 :
2961 0 : VDBG (1, "EPOLL_CTL_MOD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2962 : vep_handle, session_handle, event->events, event->data.u64);
2963 0 : break;
2964 :
2965 41 : case EPOLL_CTL_DEL:
2966 41 : if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2967 : {
2968 0 : VDBG (0, "EPOLL_CTL_DEL: %u not a vep session!", session_handle);
2969 0 : rv = VPPCOM_ENOENT;
2970 0 : goto done;
2971 : }
2972 41 : else if (PREDICT_FALSE (s->vep.vep_sh != vep_handle))
2973 : {
2974 0 : VDBG (0, "EPOLL_CTL_DEL: sh %u vep_sh (%u) != vep_sh (%u)!",
2975 : session_handle, s->vep.vep_sh, vep_handle);
2976 0 : rv = VPPCOM_EINVAL;
2977 0 : goto done;
2978 : }
2979 :
2980 41 : if (s->vep.prev_sh == vep_handle)
2981 34 : vep_session->vep.next_sh = s->vep.next_sh;
2982 : else
2983 : {
2984 : vcl_session_t *prev_session;
2985 7 : prev_session = vcl_session_get_w_handle (wrk, s->vep.prev_sh);
2986 7 : if (PREDICT_FALSE (!prev_session))
2987 : {
2988 0 : VDBG (0, "EPOLL_CTL_DEL: Invalid prev_sh (%u) on sh (%u)!",
2989 : s->vep.prev_sh, session_handle);
2990 0 : return VPPCOM_EBADFD;
2991 : }
2992 7 : ASSERT (prev_session->vep.next_sh == session_handle);
2993 7 : prev_session->vep.next_sh = s->vep.next_sh;
2994 : }
2995 41 : if (s->vep.next_sh != ~0)
2996 : {
2997 : vcl_session_t *next_session;
2998 40 : next_session = vcl_session_get_w_handle (wrk, s->vep.next_sh);
2999 40 : if (PREDICT_FALSE (!next_session))
3000 : {
3001 0 : VDBG (0, "EPOLL_CTL_DEL: Invalid next_sh (%u) on sh (%u)!",
3002 : s->vep.next_sh, session_handle);
3003 0 : return VPPCOM_EBADFD;
3004 : }
3005 40 : ASSERT (next_session->vep.prev_sh == session_handle);
3006 40 : next_session->vep.prev_sh = s->vep.prev_sh;
3007 : }
3008 :
3009 41 : if (s->vep.lt_next != VCL_INVALID_SESSION_INDEX)
3010 0 : vcl_epoll_lt_del (wrk, s);
3011 :
3012 41 : memset (&s->vep, 0, sizeof (s->vep));
3013 41 : s->vep.next_sh = ~0;
3014 41 : s->vep.prev_sh = ~0;
3015 41 : s->vep.vep_sh = ~0;
3016 41 : s->vep.lt_next = VCL_INVALID_SESSION_INDEX;
3017 41 : s->flags &= ~VCL_SESSION_F_IS_VEP_SESSION;
3018 :
3019 41 : if (vcl_session_is_open (s))
3020 27 : vcl_session_del_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
3021 :
3022 41 : VDBG (1, "EPOLL_CTL_DEL: vep_idx %u, sh %u!", vep_handle,
3023 : session_handle);
3024 : vcl_evt (VCL_EVT_EPOLL_CTLDEL, s, vep_sh);
3025 41 : break;
3026 :
3027 0 : default:
3028 0 : VDBG (0, "Invalid operation (%d)!", op);
3029 0 : rv = VPPCOM_EINVAL;
3030 : }
3031 :
3032 94 : vep_verify_epoll_chain (wrk, vep_handle);
3033 :
3034 94 : done:
3035 94 : return rv;
3036 : }
3037 :
3038 : always_inline u8
3039 5017 : vcl_ep_session_needs_evt (vcl_session_t *s, u32 evt)
3040 : {
3041 : /* No event if not epolled / events reset on hup or level-trigger on */
3042 10034 : return ((s->vep.ev.events & evt) &&
3043 5017 : s->vep.lt_next == VCL_INVALID_SESSION_INDEX);
3044 : }
3045 :
3046 : static inline void
3047 5838 : vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
3048 : struct epoll_event *events, u32 * num_ev)
3049 : {
3050 : session_disconnected_msg_t *disconnected_msg;
3051 : session_connected_msg_t *connected_msg;
3052 5838 : u32 sid = ~0, session_events;
3053 5838 : u64 session_evt_data = ~0;
3054 : vcl_session_t *s;
3055 5838 : u8 add_event = 0;
3056 :
3057 5838 : switch (e->event_type)
3058 : {
3059 5771 : case SESSION_IO_EVT_RX:
3060 5771 : sid = e->session_index;
3061 5771 : s = vcl_session_get (wrk, sid);
3062 5771 : if (vcl_session_is_closed (s))
3063 1 : break;
3064 5770 : vcl_fifo_rx_evt_valid_or_break (s);
3065 4983 : if (!vcl_ep_session_needs_evt (s, EPOLLIN) ||
3066 4983 : (s->flags & VCL_SESSION_F_HAS_RX_EVT))
3067 : break;
3068 4975 : session_events = s->vep.ev.events;
3069 4975 : add_event = 1;
3070 4975 : events[*num_ev].events = EPOLLIN;
3071 4975 : session_evt_data = s->vep.ev.data.u64;
3072 4975 : s->flags |= VCL_SESSION_F_HAS_RX_EVT;
3073 4975 : break;
3074 0 : case SESSION_IO_EVT_TX:
3075 0 : sid = e->session_index;
3076 0 : s = vcl_session_get (wrk, sid);
3077 0 : if (!s || !vcl_session_is_open (s))
3078 : break;
3079 0 : svm_fifo_reset_has_deq_ntf (vcl_session_is_ct (s) ? s->ct_tx_fifo :
3080 : s->tx_fifo);
3081 0 : if (!vcl_ep_session_needs_evt (s, EPOLLOUT))
3082 0 : break;
3083 0 : session_events = s->vep.ev.events;
3084 0 : add_event = 1;
3085 0 : events[*num_ev].events = EPOLLOUT;
3086 0 : session_evt_data = s->vep.ev.data.u64;
3087 0 : break;
3088 28 : case SESSION_CTRL_EVT_ACCEPTED:
3089 28 : if (!e->postponed)
3090 28 : s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
3091 : else
3092 0 : s = vcl_session_get (wrk, e->session_index);
3093 28 : if (!s || !vcl_ep_session_needs_evt (s, EPOLLIN))
3094 : break;
3095 28 : sid = s->session_index;
3096 28 : session_events = s->vep.ev.events;
3097 28 : add_event = 1;
3098 28 : events[*num_ev].events = EPOLLIN;
3099 28 : session_evt_data = s->vep.ev.data.u64;
3100 28 : break;
3101 0 : case SESSION_CTRL_EVT_CONNECTED:
3102 0 : if (!e->postponed)
3103 : {
3104 0 : connected_msg = (session_connected_msg_t *) e->data;
3105 0 : sid = vcl_session_connected_handler (wrk, connected_msg);
3106 : }
3107 : else
3108 0 : sid = e->session_index;
3109 0 : s = vcl_session_get (wrk, sid);
3110 0 : if (vcl_session_is_closed (s) || !vcl_ep_session_needs_evt (s, EPOLLOUT))
3111 : break;
3112 : /* We didn't have a fifo when the event was added */
3113 0 : vcl_session_add_want_deq_ntf (s, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
3114 0 : add_event = 1;
3115 0 : session_events = s->vep.ev.events;
3116 : /* Generate EPOLLOUT because there's no connected event */
3117 0 : events[*num_ev].events = EPOLLOUT;
3118 0 : session_evt_data = s->vep.ev.data.u64;
3119 0 : if (s->session_state == VCL_STATE_DETACHED)
3120 : {
3121 0 : events[*num_ev].events |= EPOLLHUP;
3122 0 : s->vep.ev.events = 0;
3123 : }
3124 0 : break;
3125 10 : case SESSION_CTRL_EVT_DISCONNECTED:
3126 10 : if (!e->postponed)
3127 : {
3128 10 : disconnected_msg = (session_disconnected_msg_t *) e->data;
3129 10 : s = vcl_session_disconnected_handler (wrk, disconnected_msg);
3130 : }
3131 : else
3132 : {
3133 0 : s = vcl_session_get (wrk, e->session_index);
3134 0 : s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
3135 : }
3136 10 : if (vcl_session_is_closed (s) || !vcl_ep_session_needs_evt (s, EPOLLHUP))
3137 : {
3138 5 : if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
3139 0 : vcl_session_free (wrk, s);
3140 5 : break;
3141 : }
3142 5 : sid = s->session_index;
3143 5 : session_events = s->vep.ev.events;
3144 5 : add_event = 1;
3145 5 : if (EPOLLRDHUP & session_events)
3146 : {
3147 : /* If app can distinguish between RDHUP and HUP,
3148 : * we make finer control */
3149 0 : events[*num_ev].events = EPOLLRDHUP;
3150 0 : if (s->flags & VCL_SESSION_F_WR_SHUTDOWN)
3151 : {
3152 0 : events[*num_ev].events |= EPOLLHUP;
3153 : }
3154 : }
3155 : else
3156 : {
3157 5 : events[*num_ev].events = EPOLLHUP;
3158 : }
3159 5 : session_evt_data = s->vep.ev.data.u64;
3160 5 : s->vep.ev.events = 0;
3161 5 : break;
3162 0 : case SESSION_CTRL_EVT_BOUND:
3163 0 : vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
3164 0 : break;
3165 2 : case SESSION_CTRL_EVT_RESET:
3166 2 : if (!e->postponed)
3167 : {
3168 : sid =
3169 2 : vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
3170 2 : s = vcl_session_get (wrk, sid);
3171 : }
3172 : else
3173 : {
3174 0 : sid = e->session_index;
3175 0 : s = vcl_session_get (wrk, sid);
3176 0 : s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
3177 : }
3178 2 : if (vcl_session_is_closed (s) || !vcl_ep_session_needs_evt (s, EPOLLHUP))
3179 : {
3180 1 : if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
3181 0 : vcl_session_free (wrk, s);
3182 1 : break;
3183 : }
3184 1 : session_events = s->vep.ev.events;
3185 1 : add_event = 1;
3186 1 : events[*num_ev].events = EPOLLERR | EPOLLHUP;
3187 1 : if ((EPOLLRDHUP & session_events) &&
3188 0 : (s->flags & VCL_SESSION_F_RD_SHUTDOWN))
3189 : {
3190 0 : events[*num_ev].events |= EPOLLRDHUP;
3191 : }
3192 1 : if ((EPOLLIN & session_events) && (s->flags & VCL_SESSION_F_RD_SHUTDOWN))
3193 : {
3194 1 : events[*num_ev].events |= EPOLLIN;
3195 : }
3196 1 : session_evt_data = s->vep.ev.data.u64;
3197 1 : s->vep.ev.events = 0;
3198 1 : break;
3199 0 : case SESSION_CTRL_EVT_UNLISTEN_REPLY:
3200 0 : vcl_session_unlisten_reply_handler (wrk, e->data);
3201 0 : break;
3202 0 : case SESSION_CTRL_EVT_MIGRATED:
3203 0 : vcl_session_migrated_handler (wrk, e->data);
3204 0 : break;
3205 5 : case SESSION_CTRL_EVT_CLEANUP:
3206 5 : vcl_session_cleanup_handler (wrk, e->data);
3207 5 : break;
3208 0 : case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
3209 0 : vcl_session_req_worker_update_handler (wrk, e->data);
3210 0 : break;
3211 0 : case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
3212 0 : vcl_session_worker_update_reply_handler (wrk, e->data);
3213 0 : break;
3214 20 : case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
3215 20 : vcl_session_app_add_segment_handler (wrk, e->data);
3216 20 : break;
3217 2 : case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
3218 2 : vcl_session_app_del_segment_handler (wrk, e->data);
3219 2 : break;
3220 0 : case SESSION_CTRL_EVT_APP_WRK_RPC:
3221 0 : vcl_worker_rpc_handler (wrk, e->data);
3222 0 : break;
3223 0 : default:
3224 0 : VDBG (0, "unhandled: %u", e->event_type);
3225 0 : break;
3226 : }
3227 :
3228 5838 : if (add_event)
3229 : {
3230 5009 : ASSERT (s->flags & VCL_SESSION_F_IS_VEP_SESSION);
3231 5009 : events[*num_ev].data.u64 = session_evt_data;
3232 5009 : if (EPOLLONESHOT & session_events)
3233 : {
3234 0 : s = vcl_session_get (wrk, sid);
3235 0 : if (!(events[*num_ev].events & EPOLLHUP))
3236 0 : s->vep.ev.events = EPOLLHUP | EPOLLERR;
3237 : }
3238 5009 : else if (!(EPOLLET & session_events))
3239 : {
3240 0 : s = vcl_session_get (wrk, sid);
3241 0 : if (s->vep.lt_next == VCL_INVALID_SESSION_INDEX)
3242 0 : vcl_epoll_lt_add (wrk, s);
3243 : }
3244 5009 : *num_ev += 1;
3245 : }
3246 5838 : }
3247 :
3248 : static int
3249 122158000 : vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
3250 : struct epoll_event *events, u32 maxevents,
3251 : double wait_for_time, u32 * num_ev)
3252 : {
3253 : svm_msg_q_msg_t *msg;
3254 : session_event_t *e;
3255 : int i;
3256 :
3257 122158000 : if (vec_len (wrk->mq_msg_vector) && svm_msg_q_is_empty (mq))
3258 0 : goto handle_dequeued;
3259 :
3260 122158000 : if (svm_msg_q_is_empty (mq))
3261 : {
3262 122153000 : if (!wait_for_time)
3263 122153000 : return 0;
3264 0 : else if (wait_for_time < 0)
3265 0 : svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY);
3266 : else
3267 : {
3268 0 : if (svm_msg_q_timedwait (mq, wait_for_time / 1e3))
3269 0 : return 0;
3270 : }
3271 : }
3272 4860 : ASSERT (maxevents > *num_ev);
3273 4860 : vcl_mq_dequeue_batch (wrk, mq, ~0);
3274 :
3275 4860 : handle_dequeued:
3276 10698 : for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
3277 : {
3278 5838 : msg = vec_elt_at_index (wrk->mq_msg_vector, i);
3279 5838 : e = svm_msg_q_msg_data (mq, msg);
3280 5838 : if (*num_ev < maxevents)
3281 5838 : vcl_epoll_wait_handle_mq_event (wrk, e, events, num_ev);
3282 : else
3283 0 : vcl_handle_mq_event (wrk, e);
3284 5838 : svm_msg_q_free_msg (mq, msg);
3285 : }
3286 4860 : vec_reset_length (wrk->mq_msg_vector);
3287 4860 : vcl_handle_pending_wrk_updates (wrk);
3288 4860 : return *num_ev;
3289 : }
3290 :
3291 : static int
3292 122158000 : vppcom_epoll_wait_condvar (vcl_worker_t *wrk, struct epoll_event *events,
3293 : int maxevents, u32 n_evts, double timeout_ms)
3294 : {
3295 122158000 : double end = -1;
3296 :
3297 122158000 : if (!n_evts)
3298 : {
3299 122158000 : if (timeout_ms > 0)
3300 0 : end = clib_time_now (&wrk->clib_time) + (timeout_ms / 1e3);
3301 : }
3302 :
3303 : do
3304 : {
3305 122158000 : vcl_epoll_wait_handle_mq (wrk, wrk->app_event_queue, events, maxevents,
3306 : timeout_ms, &n_evts);
3307 122158000 : if (n_evts || !timeout_ms)
3308 122158000 : return n_evts;
3309 : }
3310 0 : while (end == -1 || clib_time_now (&wrk->clib_time) < end);
3311 :
3312 0 : return 0;
3313 : }
3314 :
3315 : static int
3316 0 : vppcom_epoll_wait_eventfd (vcl_worker_t *wrk, struct epoll_event *events,
3317 : int maxevents, u32 n_evts, double timeout_ms)
3318 : {
3319 : int __clib_unused n_read;
3320 : vcl_mq_evt_conn_t *mqc;
3321 : int n_mq_evts, i;
3322 0 : double end = -1;
3323 : u64 buf;
3324 :
3325 0 : if (PREDICT_FALSE (wrk->api_client_handle == ~0))
3326 : {
3327 0 : vcl_api_retry_attach (wrk);
3328 0 : return n_evts;
3329 : }
3330 :
3331 0 : vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
3332 0 : if (!n_evts)
3333 : {
3334 0 : if (timeout_ms > 0)
3335 0 : end = clib_time_now (&wrk->clib_time) + (timeout_ms / 1e3);
3336 : }
3337 :
3338 : do
3339 : {
3340 0 : n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
3341 0 : vec_len (wrk->mq_events), timeout_ms);
3342 0 : if (n_mq_evts < 0)
3343 : {
3344 0 : VDBG (0, "epoll_wait error %u", errno);
3345 0 : return n_evts;
3346 : }
3347 :
3348 0 : for (i = 0; i < n_mq_evts; i++)
3349 : {
3350 0 : if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0))
3351 : {
3352 : /* api socket was closed */
3353 0 : vcl_api_handle_disconnect (wrk);
3354 0 : continue;
3355 : }
3356 :
3357 0 : mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
3358 0 : n_read = read (mqc->mq_fd, &buf, sizeof (buf));
3359 0 : vcl_epoll_wait_handle_mq (wrk, mqc->mq, events, maxevents, 0,
3360 : &n_evts);
3361 : }
3362 :
3363 0 : if (n_evts || !timeout_ms)
3364 0 : return n_evts;
3365 : }
3366 0 : while (end == -1 || clib_time_now (&wrk->clib_time) < end);
3367 :
3368 0 : return 0;
3369 : }
3370 :
3371 : static void
3372 0 : vcl_epoll_wait_handle_lt (vcl_worker_t *wrk, struct epoll_event *events,
3373 : int maxevents, u32 *n_evts)
3374 : {
3375 0 : u32 add_event = 0, evt_flags = 0, next, *to_remove = 0, *si;
3376 : vcl_session_t *s;
3377 : u64 evt_data;
3378 : int rv;
3379 :
3380 0 : ASSERT (wrk->ep_lt_current != VCL_INVALID_SESSION_INDEX);
3381 0 : if (*n_evts >= maxevents)
3382 0 : return;
3383 :
3384 0 : next = wrk->ep_lt_current;
3385 : do
3386 : {
3387 0 : s = vcl_session_get (wrk, next);
3388 0 : next = s->vep.lt_next;
3389 :
3390 0 : if (s->vep.ev.events == 0)
3391 : {
3392 0 : vec_add1 (to_remove, s->session_index);
3393 0 : continue;
3394 : }
3395 0 : if ((s->vep.ev.events & EPOLLIN) && (rv = vcl_session_read_ready (s)))
3396 : {
3397 0 : add_event = 1;
3398 0 : evt_flags |= rv > 0 ? EPOLLIN : EPOLLHUP | EPOLLRDHUP;
3399 0 : evt_data = s->vep.ev.data.u64;
3400 : }
3401 0 : if ((s->vep.ev.events & EPOLLOUT) && (rv = vcl_session_write_ready (s)))
3402 : {
3403 0 : add_event = 1;
3404 0 : evt_flags |= rv > 0 ? EPOLLOUT : EPOLLHUP | EPOLLRDHUP;
3405 0 : evt_data = s->vep.ev.data.u64;
3406 : }
3407 0 : if (!add_event && s->session_state > VCL_STATE_READY)
3408 : {
3409 0 : add_event = 1;
3410 0 : evt_flags |= EPOLLHUP | EPOLLRDHUP;
3411 0 : evt_data = s->vep.ev.data.u64;
3412 : }
3413 0 : if (add_event)
3414 : {
3415 0 : events[*n_evts].events = evt_flags;
3416 0 : events[*n_evts].data.u64 = evt_data;
3417 0 : if (EPOLLONESHOT & s->vep.ev.events)
3418 0 : s->vep.ev.events = EPOLLHUP | EPOLLERR;
3419 0 : if (evt_flags & EPOLLHUP)
3420 0 : s->vep.ev.events = 0;
3421 0 : *n_evts += 1;
3422 0 : add_event = 0;
3423 0 : evt_flags = 0;
3424 0 : if (*n_evts == maxevents)
3425 : {
3426 0 : wrk->ep_lt_current = next;
3427 0 : break;
3428 : }
3429 : }
3430 : else
3431 : {
3432 0 : vec_add1 (to_remove, s->session_index);
3433 : }
3434 : }
3435 0 : while (next != wrk->ep_lt_current);
3436 :
3437 0 : vec_foreach (si, to_remove)
3438 : {
3439 0 : s = vcl_session_get (wrk, *si);
3440 0 : vcl_epoll_lt_del (wrk, s);
3441 : }
3442 0 : vec_free (to_remove);
3443 : }
3444 :
3445 : int
3446 122158000 : vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
3447 : int maxevents, double wait_for_time)
3448 : {
3449 122158000 : vcl_worker_t *wrk = vcl_worker_get_current ();
3450 : vcl_session_t *vep_session;
3451 122158000 : u32 n_evts = 0;
3452 : int i;
3453 :
3454 122158000 : if (PREDICT_FALSE (maxevents <= 0))
3455 : {
3456 0 : VDBG (0, "ERROR: Invalid maxevents (%d)!", maxevents);
3457 0 : return VPPCOM_EINVAL;
3458 : }
3459 :
3460 122158000 : vep_session = vcl_session_get_w_handle (wrk, vep_handle);
3461 122158000 : if (!vep_session)
3462 0 : return VPPCOM_EBADFD;
3463 :
3464 122158000 : if (PREDICT_FALSE (!(vep_session->flags & VCL_SESSION_F_IS_VEP)))
3465 : {
3466 0 : VDBG (0, "ERROR: vep_idx (%u) is not a vep!", vep_handle);
3467 0 : return VPPCOM_EINVAL;
3468 : }
3469 :
3470 122158000 : if (vec_len (wrk->unhandled_evts_vector))
3471 : {
3472 0 : for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
3473 : {
3474 0 : vcl_epoll_wait_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i],
3475 : events, &n_evts);
3476 0 : if (n_evts == maxevents)
3477 : {
3478 0 : vec_delete (wrk->unhandled_evts_vector, i + 1, 0);
3479 0 : return n_evts;
3480 : }
3481 : }
3482 0 : vec_reset_length (wrk->unhandled_evts_vector);
3483 : }
3484 :
3485 122158000 : if (PREDICT_FALSE (wrk->ep_lt_current != VCL_INVALID_SESSION_INDEX))
3486 0 : vcl_epoll_wait_handle_lt (wrk, events, maxevents, &n_evts);
3487 :
3488 : /* Request to only drain unhandled */
3489 122158000 : if ((int) wait_for_time == -2)
3490 0 : return n_evts;
3491 :
3492 :
3493 122158000 : if (vcm->cfg.use_mq_eventfd)
3494 0 : n_evts = vppcom_epoll_wait_eventfd (wrk, events, maxevents, n_evts,
3495 : wait_for_time);
3496 : else
3497 122158000 : n_evts = vppcom_epoll_wait_condvar (wrk, events, maxevents, n_evts,
3498 : wait_for_time);
3499 :
3500 122158000 : return n_evts;
3501 : }
3502 :
3503 : int
3504 6567 : vppcom_session_attr (uint32_t session_handle, uint32_t op,
3505 : void *buffer, uint32_t * buflen)
3506 : {
3507 6567 : vcl_worker_t *wrk = vcl_worker_get_current ();
3508 6567 : u32 *flags = buffer;
3509 6567 : vppcom_endpt_t *ep = buffer;
3510 : transport_endpt_attr_t tea;
3511 : vcl_session_t *session;
3512 6567 : int rv = VPPCOM_OK;
3513 :
3514 6567 : session = vcl_session_get_w_handle (wrk, session_handle);
3515 6567 : if (!session)
3516 0 : return VPPCOM_EBADFD;
3517 :
3518 6567 : switch (op)
3519 : {
3520 6128 : case VPPCOM_ATTR_GET_NREAD:
3521 6128 : rv = vcl_session_read_ready (session);
3522 6128 : VDBG (2, "VPPCOM_ATTR_GET_NREAD: sh %u, nread = %d", session_handle,
3523 : rv);
3524 6128 : break;
3525 :
3526 0 : case VPPCOM_ATTR_GET_NWRITE:
3527 0 : rv = vcl_session_write_ready (session);
3528 0 : VDBG (2, "VPPCOM_ATTR_GET_NWRITE: sh %u, nwrite = %d", session_handle,
3529 : rv);
3530 0 : break;
3531 :
3532 14 : case VPPCOM_ATTR_GET_FLAGS:
3533 14 : if (PREDICT_TRUE (buffer && buflen && (*buflen >= sizeof (*flags))))
3534 : {
3535 14 : *flags =
3536 14 : O_RDWR |
3537 14 : (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK) ?
3538 : O_NONBLOCK : 0);
3539 14 : *buflen = sizeof (*flags);
3540 14 : VDBG (2, "VPPCOM_ATTR_GET_FLAGS: sh %u, flags = 0x%08x, "
3541 : "is_nonblocking = %u", session_handle, *flags,
3542 : vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK));
3543 : }
3544 : else
3545 0 : rv = VPPCOM_EINVAL;
3546 14 : break;
3547 :
3548 37 : case VPPCOM_ATTR_SET_FLAGS:
3549 37 : if (PREDICT_TRUE (buffer && buflen && (*buflen == sizeof (*flags))))
3550 : {
3551 37 : if (*flags & O_NONBLOCK)
3552 33 : vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
3553 : else
3554 4 : vcl_session_clear_attr (session, VCL_SESS_ATTR_NONBLOCK);
3555 :
3556 37 : VDBG (2, "VPPCOM_ATTR_SET_FLAGS: sh %u, flags = 0x%08x,"
3557 : " is_nonblocking = %u", session_handle, *flags,
3558 : vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK));
3559 : }
3560 : else
3561 0 : rv = VPPCOM_EINVAL;
3562 37 : break;
3563 :
3564 17 : case VPPCOM_ATTR_GET_PEER_ADDR:
3565 17 : if (PREDICT_TRUE (buffer && buflen &&
3566 : (*buflen >= sizeof (*ep)) && ep->ip))
3567 : {
3568 17 : ep->is_ip4 = session->transport.is_ip4;
3569 17 : ep->port = session->transport.rmt_port;
3570 17 : if (session->transport.is_ip4)
3571 14 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
3572 : sizeof (ip4_address_t));
3573 : else
3574 3 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
3575 : sizeof (ip6_address_t));
3576 17 : *buflen = sizeof (*ep);
3577 17 : VDBG (1,
3578 : "VPPCOM_ATTR_GET_PEER_ADDR: sh %u, is_ip4 = %u, "
3579 : "addr = %U, port %u",
3580 : session_handle, ep->is_ip4, vcl_format_ip46_address,
3581 : &session->transport.rmt_ip,
3582 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3583 : clib_net_to_host_u16 (ep->port));
3584 : }
3585 : else
3586 0 : rv = VPPCOM_EINVAL;
3587 17 : break;
3588 :
3589 29 : case VPPCOM_ATTR_GET_LCL_ADDR:
3590 29 : if (PREDICT_TRUE (buffer && buflen &&
3591 : (*buflen >= sizeof (*ep)) && ep->ip))
3592 : {
3593 29 : ep->is_ip4 = session->transport.is_ip4;
3594 29 : ep->port = session->transport.lcl_port;
3595 29 : if (session->transport.is_ip4)
3596 24 : clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip4,
3597 : sizeof (ip4_address_t));
3598 : else
3599 5 : clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip6,
3600 : sizeof (ip6_address_t));
3601 29 : *buflen = sizeof (*ep);
3602 29 : VDBG (1,
3603 : "VPPCOM_ATTR_GET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3604 : " port %d",
3605 : session_handle, ep->is_ip4, vcl_format_ip46_address,
3606 : &session->transport.lcl_ip,
3607 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3608 : clib_net_to_host_u16 (ep->port));
3609 : }
3610 : else
3611 0 : rv = VPPCOM_EINVAL;
3612 29 : break;
3613 :
3614 0 : case VPPCOM_ATTR_SET_LCL_ADDR:
3615 0 : if (PREDICT_TRUE (buffer && buflen &&
3616 : (*buflen >= sizeof (*ep)) && ep->ip))
3617 : {
3618 0 : session->transport.is_ip4 = ep->is_ip4;
3619 0 : session->transport.lcl_port = ep->port;
3620 0 : vcl_ip_copy_from_ep (&session->transport.lcl_ip, ep);
3621 0 : *buflen = sizeof (*ep);
3622 0 : VDBG (1,
3623 : "VPPCOM_ATTR_SET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3624 : " port %d",
3625 : session_handle, ep->is_ip4, vcl_format_ip46_address,
3626 : &session->transport.lcl_ip,
3627 : ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3628 : clib_net_to_host_u16 (ep->port));
3629 : }
3630 : else
3631 0 : rv = VPPCOM_EINVAL;
3632 0 : break;
3633 :
3634 287 : case VPPCOM_ATTR_GET_LIBC_EPFD:
3635 287 : rv = session->libc_epfd;
3636 287 : VDBG (2, "VPPCOM_ATTR_GET_LIBC_EPFD: libc_epfd %d", rv);
3637 287 : break;
3638 :
3639 1 : case VPPCOM_ATTR_SET_LIBC_EPFD:
3640 1 : if (PREDICT_TRUE (buffer && buflen &&
3641 : (*buflen == sizeof (session->libc_epfd))))
3642 : {
3643 1 : session->libc_epfd = *(int *) buffer;
3644 1 : *buflen = sizeof (session->libc_epfd);
3645 :
3646 1 : VDBG (2, "VPPCOM_ATTR_SET_LIBC_EPFD: libc_epfd %d, buflen %d",
3647 : session->libc_epfd, *buflen);
3648 : }
3649 : else
3650 0 : rv = VPPCOM_EINVAL;
3651 1 : break;
3652 :
3653 0 : case VPPCOM_ATTR_GET_PROTOCOL:
3654 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3655 : {
3656 0 : *(int *) buffer = session->session_type;
3657 0 : *buflen = sizeof (int);
3658 :
3659 0 : VDBG (2, "VPPCOM_ATTR_GET_PROTOCOL: %d (%s), buflen %d",
3660 : *(int *) buffer, *(int *) buffer ? "UDP" : "TCP", *buflen);
3661 : }
3662 : else
3663 0 : rv = VPPCOM_EINVAL;
3664 0 : break;
3665 :
3666 0 : case VPPCOM_ATTR_GET_LISTEN:
3667 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3668 : {
3669 0 : *(int *) buffer = vcl_session_has_attr (session,
3670 : VCL_SESS_ATTR_LISTEN);
3671 0 : *buflen = sizeof (int);
3672 :
3673 0 : VDBG (2, "VPPCOM_ATTR_GET_LISTEN: %d, buflen %d", *(int *) buffer,
3674 : *buflen);
3675 : }
3676 : else
3677 0 : rv = VPPCOM_EINVAL;
3678 0 : break;
3679 :
3680 0 : case VPPCOM_ATTR_GET_ERROR:
3681 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3682 : {
3683 0 : *(int *) buffer = 0;
3684 0 : *buflen = sizeof (int);
3685 :
3686 0 : VDBG (2, "VPPCOM_ATTR_GET_ERROR: %d, buflen %d, #VPP-TBD#",
3687 : *(int *) buffer, *buflen);
3688 : }
3689 : else
3690 0 : rv = VPPCOM_EINVAL;
3691 0 : break;
3692 :
3693 12 : case VPPCOM_ATTR_GET_TX_FIFO_LEN:
3694 12 : if (buffer && buflen && (*buflen >= sizeof (u32)))
3695 : {
3696 :
3697 : /* VPP-TBD */
3698 24 : *(size_t *) buffer = (session->sndbuf_size ? session->sndbuf_size :
3699 12 : session->tx_fifo ?
3700 4 : svm_fifo_size (session->tx_fifo) :
3701 8 : vcm->cfg.tx_fifo_size);
3702 12 : *buflen = sizeof (u32);
3703 :
3704 12 : VDBG (2, "VPPCOM_ATTR_GET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3705 : " #VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer,
3706 : *buflen);
3707 : }
3708 : else
3709 0 : rv = VPPCOM_EINVAL;
3710 12 : break;
3711 :
3712 0 : case VPPCOM_ATTR_SET_DSCP:
3713 0 : if (buffer && buflen && (*buflen >= sizeof (u8)))
3714 : {
3715 0 : session->dscp = *(u8 *) buffer;
3716 :
3717 0 : VDBG (2, "VPPCOM_ATTR_SET_DSCP: %u (0x%x), buflen %d,",
3718 : *(u8 *) buffer, *(u8 *) buffer, *buflen);
3719 : }
3720 : else
3721 0 : rv = VPPCOM_EINVAL;
3722 0 : break;
3723 :
3724 0 : case VPPCOM_ATTR_SET_TX_FIFO_LEN:
3725 0 : if (buffer && buflen && (*buflen == sizeof (u32)))
3726 : {
3727 : /* VPP-TBD */
3728 0 : session->sndbuf_size = *(u32 *) buffer;
3729 0 : VDBG (2, "VPPCOM_ATTR_SET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3730 : " #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3731 : *buflen);
3732 : }
3733 : else
3734 0 : rv = VPPCOM_EINVAL;
3735 0 : break;
3736 :
3737 12 : case VPPCOM_ATTR_GET_RX_FIFO_LEN:
3738 12 : if (buffer && buflen && (*buflen >= sizeof (u32)))
3739 : {
3740 :
3741 : /* VPP-TBD */
3742 24 : *(size_t *) buffer = (session->rcvbuf_size ? session->rcvbuf_size :
3743 12 : session->rx_fifo ?
3744 4 : svm_fifo_size (session->rx_fifo) :
3745 8 : vcm->cfg.rx_fifo_size);
3746 12 : *buflen = sizeof (u32);
3747 :
3748 12 : VDBG (2, "VPPCOM_ATTR_GET_RX_FIFO_LEN: %u (0x%x), buflen %d, "
3749 : "#VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer, *buflen);
3750 : }
3751 : else
3752 0 : rv = VPPCOM_EINVAL;
3753 12 : break;
3754 :
3755 0 : case VPPCOM_ATTR_SET_RX_FIFO_LEN:
3756 0 : if (buffer && buflen && (*buflen == sizeof (u32)))
3757 : {
3758 : /* VPP-TBD */
3759 0 : session->rcvbuf_size = *(u32 *) buffer;
3760 0 : VDBG (2, "VPPCOM_ATTR_SET_RX_FIFO_LEN: %u (0x%x), buflen %d,"
3761 : " #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3762 : *buflen);
3763 : }
3764 : else
3765 0 : rv = VPPCOM_EINVAL;
3766 0 : break;
3767 :
3768 0 : case VPPCOM_ATTR_GET_REUSEADDR:
3769 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3770 : {
3771 : /* VPP-TBD */
3772 0 : *(int *) buffer = vcl_session_has_attr (session,
3773 : VCL_SESS_ATTR_REUSEADDR);
3774 0 : *buflen = sizeof (int);
3775 :
3776 0 : VDBG (2, "VPPCOM_ATTR_GET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3777 : *(int *) buffer, *buflen);
3778 : }
3779 : else
3780 0 : rv = VPPCOM_EINVAL;
3781 0 : break;
3782 :
3783 15 : case VPPCOM_ATTR_SET_REUSEADDR:
3784 30 : if (buffer && buflen && (*buflen == sizeof (int)) &&
3785 15 : !vcl_session_has_attr (session, VCL_SESS_ATTR_LISTEN))
3786 : {
3787 : /* VPP-TBD */
3788 15 : if (*(int *) buffer)
3789 15 : vcl_session_set_attr (session, VCL_SESS_ATTR_REUSEADDR);
3790 : else
3791 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_REUSEADDR);
3792 :
3793 15 : VDBG (2, "VPPCOM_ATTR_SET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3794 : vcl_session_has_attr (session, VCL_SESS_ATTR_REUSEADDR),
3795 : *buflen);
3796 : }
3797 : else
3798 0 : rv = VPPCOM_EINVAL;
3799 15 : break;
3800 :
3801 0 : case VPPCOM_ATTR_GET_REUSEPORT:
3802 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3803 : {
3804 : /* VPP-TBD */
3805 0 : *(int *) buffer = vcl_session_has_attr (session,
3806 : VCL_SESS_ATTR_REUSEPORT);
3807 0 : *buflen = sizeof (int);
3808 :
3809 0 : VDBG (2, "VPPCOM_ATTR_GET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3810 : *(int *) buffer, *buflen);
3811 : }
3812 : else
3813 0 : rv = VPPCOM_EINVAL;
3814 0 : break;
3815 :
3816 0 : case VPPCOM_ATTR_SET_REUSEPORT:
3817 0 : if (buffer && buflen && (*buflen == sizeof (int)) &&
3818 0 : !vcl_session_has_attr (session, VCL_SESS_ATTR_LISTEN))
3819 : {
3820 : /* VPP-TBD */
3821 0 : if (*(int *) buffer)
3822 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_REUSEPORT);
3823 : else
3824 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_REUSEPORT);
3825 :
3826 0 : VDBG (2, "VPPCOM_ATTR_SET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3827 : vcl_session_has_attr (session, VCL_SESS_ATTR_REUSEPORT),
3828 : *buflen);
3829 : }
3830 : else
3831 0 : rv = VPPCOM_EINVAL;
3832 0 : break;
3833 :
3834 0 : case VPPCOM_ATTR_GET_BROADCAST:
3835 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3836 : {
3837 : /* VPP-TBD */
3838 0 : *(int *) buffer = vcl_session_has_attr (session,
3839 : VCL_SESS_ATTR_BROADCAST);
3840 0 : *buflen = sizeof (int);
3841 :
3842 0 : VDBG (2, "VPPCOM_ATTR_GET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3843 : *(int *) buffer, *buflen);
3844 : }
3845 : else
3846 0 : rv = VPPCOM_EINVAL;
3847 0 : break;
3848 :
3849 0 : case VPPCOM_ATTR_SET_BROADCAST:
3850 0 : if (buffer && buflen && (*buflen == sizeof (int)))
3851 : {
3852 : /* VPP-TBD */
3853 0 : if (*(int *) buffer)
3854 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_BROADCAST);
3855 : else
3856 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_BROADCAST);
3857 :
3858 0 : VDBG (2, "VPPCOM_ATTR_SET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3859 : vcl_session_has_attr (session, VCL_SESS_ATTR_BROADCAST),
3860 : *buflen);
3861 : }
3862 : else
3863 0 : rv = VPPCOM_EINVAL;
3864 0 : break;
3865 :
3866 0 : case VPPCOM_ATTR_GET_V6ONLY:
3867 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3868 : {
3869 : /* VPP-TBD */
3870 0 : *(int *) buffer = vcl_session_has_attr (session,
3871 : VCL_SESS_ATTR_V6ONLY);
3872 0 : *buflen = sizeof (int);
3873 :
3874 0 : VDBG (2, "VPPCOM_ATTR_GET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3875 : *(int *) buffer, *buflen);
3876 : }
3877 : else
3878 0 : rv = VPPCOM_EINVAL;
3879 0 : break;
3880 :
3881 2 : case VPPCOM_ATTR_SET_V6ONLY:
3882 2 : if (buffer && buflen && (*buflen == sizeof (int)))
3883 : {
3884 : /* VPP-TBD */
3885 2 : if (*(int *) buffer)
3886 2 : vcl_session_set_attr (session, VCL_SESS_ATTR_V6ONLY);
3887 : else
3888 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_V6ONLY);
3889 :
3890 2 : VDBG (2, "VPPCOM_ATTR_SET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3891 : vcl_session_has_attr (session, VCL_SESS_ATTR_V6ONLY),
3892 : *buflen);
3893 : }
3894 : else
3895 0 : rv = VPPCOM_EINVAL;
3896 2 : break;
3897 :
3898 0 : case VPPCOM_ATTR_GET_KEEPALIVE:
3899 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3900 : {
3901 : /* VPP-TBD */
3902 0 : *(int *) buffer = vcl_session_has_attr (session,
3903 : VCL_SESS_ATTR_KEEPALIVE);
3904 0 : *buflen = sizeof (int);
3905 :
3906 0 : VDBG (2, "VPPCOM_ATTR_GET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3907 : *(int *) buffer, *buflen);
3908 : }
3909 : else
3910 0 : rv = VPPCOM_EINVAL;
3911 0 : break;
3912 :
3913 0 : case VPPCOM_ATTR_SET_KEEPALIVE:
3914 0 : if (buffer && buflen && (*buflen == sizeof (int)))
3915 : {
3916 : /* VPP-TBD */
3917 0 : if (*(int *) buffer)
3918 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_KEEPALIVE);
3919 : else
3920 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_KEEPALIVE);
3921 :
3922 0 : VDBG (2, "VPPCOM_ATTR_SET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3923 : vcl_session_has_attr (session, VCL_SESS_ATTR_KEEPALIVE),
3924 : *buflen);
3925 : }
3926 : else
3927 0 : rv = VPPCOM_EINVAL;
3928 0 : break;
3929 :
3930 0 : case VPPCOM_ATTR_GET_TCP_NODELAY:
3931 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3932 : {
3933 : /* VPP-TBD */
3934 0 : *(int *) buffer = vcl_session_has_attr (session,
3935 : VCL_SESS_ATTR_TCP_NODELAY);
3936 0 : *buflen = sizeof (int);
3937 :
3938 0 : VDBG (2, "VPPCOM_ATTR_GET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
3939 : *(int *) buffer, *buflen);
3940 : }
3941 : else
3942 0 : rv = VPPCOM_EINVAL;
3943 0 : break;
3944 :
3945 0 : case VPPCOM_ATTR_SET_TCP_NODELAY:
3946 0 : if (buffer && buflen && (*buflen == sizeof (int)))
3947 : {
3948 : /* VPP-TBD */
3949 0 : if (*(int *) buffer)
3950 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_NODELAY);
3951 : else
3952 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_NODELAY);
3953 :
3954 0 : VDBG (2, "VPPCOM_ATTR_SET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
3955 : vcl_session_has_attr (session, VCL_SESS_ATTR_TCP_NODELAY),
3956 : *buflen);
3957 : }
3958 : else
3959 0 : rv = VPPCOM_EINVAL;
3960 0 : break;
3961 :
3962 0 : case VPPCOM_ATTR_GET_TCP_KEEPIDLE:
3963 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3964 : {
3965 : /* VPP-TBD */
3966 0 : *(int *) buffer = vcl_session_has_attr (session,
3967 : VCL_SESS_ATTR_TCP_KEEPIDLE);
3968 0 : *buflen = sizeof (int);
3969 :
3970 0 : VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
3971 : *(int *) buffer, *buflen);
3972 : }
3973 : else
3974 0 : rv = VPPCOM_EINVAL;
3975 0 : break;
3976 :
3977 0 : case VPPCOM_ATTR_SET_TCP_KEEPIDLE:
3978 0 : if (buffer && buflen && (*buflen == sizeof (int)))
3979 : {
3980 : /* VPP-TBD */
3981 0 : if (*(int *) buffer)
3982 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_KEEPIDLE);
3983 : else
3984 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_KEEPIDLE);
3985 :
3986 0 : VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
3987 : vcl_session_has_attr (session,
3988 : VCL_SESS_ATTR_TCP_KEEPIDLE), *buflen);
3989 : }
3990 : else
3991 0 : rv = VPPCOM_EINVAL;
3992 0 : break;
3993 :
3994 0 : case VPPCOM_ATTR_GET_TCP_KEEPINTVL:
3995 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
3996 : {
3997 : /* VPP-TBD */
3998 0 : *(int *) buffer = vcl_session_has_attr (session,
3999 : VCL_SESS_ATTR_TCP_KEEPINTVL);
4000 0 : *buflen = sizeof (int);
4001 :
4002 0 : VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
4003 : *(int *) buffer, *buflen);
4004 : }
4005 : else
4006 0 : rv = VPPCOM_EINVAL;
4007 0 : break;
4008 :
4009 0 : case VPPCOM_ATTR_SET_TCP_KEEPINTVL:
4010 0 : if (buffer && buflen && (*buflen == sizeof (int)))
4011 : {
4012 : /* VPP-TBD */
4013 0 : if (*(int *) buffer)
4014 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_KEEPINTVL);
4015 : else
4016 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_KEEPINTVL);
4017 :
4018 0 : VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
4019 : vcl_session_has_attr (session,
4020 : VCL_SESS_ATTR_TCP_KEEPINTVL), *buflen);
4021 : }
4022 : else
4023 0 : rv = VPPCOM_EINVAL;
4024 0 : break;
4025 :
4026 5 : case VPPCOM_ATTR_GET_TCP_USER_MSS:
4027 5 : if (!(buffer && buflen && (*buflen >= sizeof (u32))))
4028 : {
4029 0 : rv = VPPCOM_EINVAL;
4030 0 : break;
4031 : }
4032 :
4033 5 : tea.type = TRANSPORT_ENDPT_ATTR_MSS;
4034 5 : tea.mss = *(u32 *) buffer;
4035 5 : if (vcl_session_transport_attr (wrk, session, 1 /* is_get */, &tea))
4036 2 : rv = VPPCOM_ENOPROTOOPT;
4037 :
4038 5 : if (!rv)
4039 : {
4040 3 : *(u32 *) buffer = tea.mss;
4041 3 : *buflen = sizeof (int);
4042 : }
4043 :
4044 5 : VDBG (2, "VPPCOM_ATTR_GET_TCP_USER_MSS: %d, buflen %d", *(int *) buffer,
4045 : *buflen);
4046 5 : break;
4047 2 : case VPPCOM_ATTR_SET_TCP_USER_MSS:
4048 2 : if (!(buffer && buflen && (*buflen == sizeof (u32))))
4049 : {
4050 0 : rv = VPPCOM_EINVAL;
4051 0 : break;
4052 : }
4053 :
4054 2 : tea.type = TRANSPORT_ENDPT_ATTR_MSS;
4055 2 : tea.mss = *(u32 *) buffer;
4056 2 : if (vcl_session_transport_attr (wrk, session, 0 /* is_get */, &tea))
4057 0 : rv = VPPCOM_ENOPROTOOPT;
4058 :
4059 2 : VDBG (2, "VPPCOM_ATTR_SET_TCP_USER_MSS: %u, buflen %d", tea.mss,
4060 : *buflen);
4061 2 : break;
4062 :
4063 0 : case VPPCOM_ATTR_SET_CONNECTED:
4064 0 : session->flags |= VCL_SESSION_F_CONNECTED;
4065 0 : break;
4066 :
4067 6 : case VPPCOM_ATTR_SET_CKPAIR:
4068 12 : if (!(buffer && buflen && (*buflen == sizeof (int))) ||
4069 6 : !vcl_session_has_crypto (session))
4070 : {
4071 0 : rv = VPPCOM_EINVAL;
4072 0 : break;
4073 : }
4074 6 : if (!session->ext_config)
4075 : {
4076 6 : vcl_session_alloc_ext_cfg (session, TRANSPORT_ENDPT_EXT_CFG_CRYPTO,
4077 : sizeof (transport_endpt_ext_cfg_t));
4078 : }
4079 0 : else if (session->ext_config->type != TRANSPORT_ENDPT_EXT_CFG_CRYPTO)
4080 : {
4081 0 : rv = VPPCOM_EINVAL;
4082 0 : break;
4083 : }
4084 :
4085 6 : session->ext_config->crypto.ckpair_index = *(uint32_t *) buffer;
4086 6 : break;
4087 :
4088 0 : case VPPCOM_ATTR_SET_VRF:
4089 0 : if (!(buffer && buflen && (*buflen == sizeof (u32))))
4090 : {
4091 0 : rv = VPPCOM_EINVAL;
4092 0 : break;
4093 : }
4094 0 : session->vrf = *(u32 *) buffer;
4095 0 : break;
4096 :
4097 0 : case VPPCOM_ATTR_GET_VRF:
4098 0 : if (!(buffer && buflen && (*buflen >= sizeof (u32))))
4099 : {
4100 0 : rv = VPPCOM_EINVAL;
4101 0 : break;
4102 : }
4103 0 : *(u32 *) buffer = session->vrf;
4104 0 : *buflen = sizeof (u32);
4105 0 : break;
4106 :
4107 0 : case VPPCOM_ATTR_GET_DOMAIN:
4108 0 : if (!(buffer && buflen && (*buflen >= sizeof (int))))
4109 : {
4110 0 : rv = VPPCOM_EINVAL;
4111 0 : break;
4112 : }
4113 :
4114 0 : if (session->transport.is_ip4)
4115 0 : *(int *) buffer = AF_INET;
4116 : else
4117 0 : *(int *) buffer = AF_INET6;
4118 0 : *buflen = sizeof (int);
4119 :
4120 0 : VDBG (2, "VPPCOM_ATTR_GET_DOMAIN: %d, buflen %u", *(int *) buffer,
4121 : *buflen);
4122 0 : break;
4123 :
4124 0 : case VPPCOM_ATTR_SET_ENDPT_EXT_CFG:
4125 0 : if (!(buffer && buflen && (*buflen > 0)))
4126 : {
4127 0 : rv = VPPCOM_EINVAL;
4128 0 : break;
4129 : }
4130 0 : if (session->ext_config)
4131 : {
4132 0 : rv = VPPCOM_EINVAL;
4133 0 : break;
4134 : }
4135 0 : vcl_session_alloc_ext_cfg (session, TRANSPORT_ENDPT_EXT_CFG_NONE,
4136 0 : *buflen + sizeof (u32));
4137 0 : clib_memcpy (session->ext_config->data, buffer, *buflen);
4138 0 : session->ext_config->len = *buflen;
4139 0 : break;
4140 0 : case VPPCOM_ATTR_SET_IP_PKTINFO:
4141 0 : if (buffer && buflen && (*buflen == sizeof (int)) &&
4142 0 : !vcl_session_has_attr (session, VCL_SESS_ATTR_IP_PKTINFO))
4143 : {
4144 0 : if (*(int *) buffer)
4145 0 : vcl_session_set_attr (session, VCL_SESS_ATTR_IP_PKTINFO);
4146 : else
4147 0 : vcl_session_clear_attr (session, VCL_SESS_ATTR_IP_PKTINFO);
4148 :
4149 0 : VDBG (2, "VCL_SESS_ATTR_IP_PKTINFO: %d, buflen %d",
4150 : vcl_session_has_attr (session, VCL_SESS_ATTR_IP_PKTINFO),
4151 : *buflen);
4152 : }
4153 : else
4154 0 : rv = VPPCOM_EINVAL;
4155 0 : break;
4156 :
4157 0 : case VPPCOM_ATTR_GET_IP_PKTINFO:
4158 0 : if (buffer && buflen && (*buflen >= sizeof (int)))
4159 : {
4160 0 : *(int *) buffer =
4161 0 : vcl_session_has_attr (session, VCL_SESS_ATTR_IP_PKTINFO);
4162 0 : *buflen = sizeof (int);
4163 :
4164 0 : VDBG (2, "VCL_SESS_ATTR_IP_PKTINFO: %d, buflen %d", *(int *) buffer,
4165 : *buflen);
4166 : }
4167 : else
4168 0 : rv = VPPCOM_EINVAL;
4169 0 : break;
4170 :
4171 0 : default:
4172 0 : rv = VPPCOM_EINVAL;
4173 0 : break;
4174 : }
4175 :
4176 6567 : return rv;
4177 : }
4178 :
4179 : int
4180 4 : vppcom_session_recvfrom (uint32_t session_handle, void *buffer,
4181 : uint32_t buflen, int flags, vppcom_endpt_t * ep)
4182 : {
4183 4 : vcl_worker_t *wrk = vcl_worker_get_current ();
4184 : vcl_session_t *session;
4185 4 : int rv = VPPCOM_OK;
4186 :
4187 4 : if (flags == 0)
4188 4 : rv = vppcom_session_read (session_handle, buffer, buflen);
4189 0 : else if (flags & MSG_PEEK)
4190 0 : rv = vppcom_session_peek (session_handle, buffer, buflen);
4191 : else
4192 : {
4193 0 : VDBG (0, "Unsupport flags for recvfrom %d", flags);
4194 0 : return VPPCOM_EAFNOSUPPORT;
4195 : }
4196 :
4197 4 : if (ep && rv > 0)
4198 : {
4199 2 : session = vcl_session_get_w_handle (wrk, session_handle);
4200 2 : if (session->transport.is_ip4)
4201 2 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
4202 : sizeof (ip4_address_t));
4203 : else
4204 0 : clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
4205 : sizeof (ip6_address_t));
4206 2 : ep->is_ip4 = session->transport.is_ip4;
4207 2 : ep->port = session->transport.rmt_port;
4208 : }
4209 :
4210 4 : return rv;
4211 : }
4212 :
4213 : static void
4214 0 : vcl_handle_ep_app_tlvs (vcl_session_t *s, vppcom_endpt_t *ep)
4215 : {
4216 0 : vppcom_endpt_tlv_t *tlv = ep->app_tlvs;
4217 :
4218 : do
4219 : {
4220 0 : switch (tlv->data_type)
4221 : {
4222 0 : case VCL_UDP_SEGMENT:
4223 0 : s->gso_size = *(u16 *) tlv->data;
4224 0 : break;
4225 0 : case VCL_IP_PKTINFO:
4226 0 : clib_memcpy_fast (&s->transport.lcl_ip, (ip4_address_t *) tlv->data,
4227 : sizeof (ip4_address_t));
4228 0 : break;
4229 0 : default:
4230 0 : VDBG (0, "Ignorning unsupported app tlv %u", tlv->data_type);
4231 0 : break;
4232 : }
4233 0 : tlv = VCL_EP_NEXT_APP_TLV (ep, tlv);
4234 : }
4235 0 : while (tlv);
4236 0 : }
4237 :
4238 : int
4239 0 : vppcom_session_sendto (uint32_t session_handle, void *buffer,
4240 : uint32_t buflen, int flags, vppcom_endpt_t * ep)
4241 : {
4242 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4243 : vcl_session_t *s;
4244 :
4245 0 : s = vcl_session_get_w_handle (wrk, session_handle);
4246 0 : if (PREDICT_FALSE (!s))
4247 0 : return VPPCOM_EBADFD;
4248 :
4249 0 : if (ep)
4250 : {
4251 0 : if (!vcl_session_is_cl (s))
4252 0 : return VPPCOM_EINVAL;
4253 :
4254 0 : s->transport.is_ip4 = ep->is_ip4;
4255 0 : s->transport.rmt_port = ep->port;
4256 0 : vcl_ip_copy_from_ep (&s->transport.rmt_ip, ep);
4257 :
4258 0 : if (ep->app_tlvs)
4259 0 : vcl_handle_ep_app_tlvs (s, ep);
4260 :
4261 : /* Session not connected/bound in vpp. Create it by 'connecting' it */
4262 0 : if (PREDICT_FALSE (s->session_state == VCL_STATE_CLOSED))
4263 : {
4264 0 : u32 session_index = s->session_index;
4265 0 : f64 timeout = vcm->cfg.session_timeout;
4266 : int rv;
4267 :
4268 0 : vcl_send_session_connect (wrk, s);
4269 0 : rv = vppcom_wait_for_session_state_change (session_index,
4270 : VCL_STATE_READY,
4271 : timeout);
4272 0 : if (rv < 0)
4273 0 : return rv;
4274 0 : s = vcl_session_get (wrk, session_index);
4275 : }
4276 : }
4277 :
4278 0 : if (flags)
4279 : {
4280 : // TBD check the flags and do the right thing
4281 0 : VDBG (2, "handling flags 0x%u (%d) not implemented yet.", flags, flags);
4282 : }
4283 :
4284 0 : return (vppcom_session_write_inline (wrk, s, buffer, buflen, 1,
4285 0 : s->is_dgram ? 1 : 0));
4286 : }
4287 :
4288 : int
4289 0 : vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
4290 : {
4291 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4292 0 : f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
4293 0 : u32 i, keep_trying = 1;
4294 : svm_msg_q_msg_t msg;
4295 : session_event_t *e;
4296 0 : int rv, num_ev = 0;
4297 :
4298 0 : VDBG (3, "vp %p, nsids %u, wait_for_time %f", vp, n_sids, wait_for_time);
4299 :
4300 0 : if (!vp)
4301 0 : return VPPCOM_EFAULT;
4302 :
4303 : do
4304 : {
4305 : vcl_session_t *session;
4306 :
4307 : /* Dequeue all events and drop all unhandled io events */
4308 0 : while (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0) == 0)
4309 : {
4310 0 : e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
4311 0 : vcl_handle_mq_event (wrk, e);
4312 0 : svm_msg_q_free_msg (wrk->app_event_queue, &msg);
4313 : }
4314 0 : vec_reset_length (wrk->unhandled_evts_vector);
4315 :
4316 0 : for (i = 0; i < n_sids; i++)
4317 : {
4318 0 : session = vcl_session_get (wrk, vp[i].sh);
4319 0 : if (!session)
4320 : {
4321 0 : vp[i].revents = POLLHUP;
4322 0 : num_ev++;
4323 0 : continue;
4324 : }
4325 :
4326 0 : vp[i].revents = 0;
4327 :
4328 0 : if (POLLIN & vp[i].events)
4329 : {
4330 0 : rv = vcl_session_read_ready (session);
4331 0 : if (rv > 0)
4332 : {
4333 0 : vp[i].revents |= POLLIN;
4334 0 : num_ev++;
4335 : }
4336 0 : else if (rv < 0)
4337 : {
4338 0 : switch (rv)
4339 : {
4340 0 : case VPPCOM_ECONNRESET:
4341 0 : vp[i].revents = POLLHUP;
4342 0 : break;
4343 :
4344 0 : default:
4345 0 : vp[i].revents = POLLERR;
4346 0 : break;
4347 : }
4348 0 : num_ev++;
4349 : }
4350 : }
4351 :
4352 0 : if (POLLOUT & vp[i].events)
4353 : {
4354 0 : rv = vcl_session_write_ready (session);
4355 0 : if (rv > 0)
4356 : {
4357 0 : vp[i].revents |= POLLOUT;
4358 0 : num_ev++;
4359 : }
4360 0 : else if (rv < 0)
4361 : {
4362 0 : switch (rv)
4363 : {
4364 0 : case VPPCOM_ECONNRESET:
4365 0 : vp[i].revents = POLLHUP;
4366 0 : break;
4367 :
4368 0 : default:
4369 0 : vp[i].revents = POLLERR;
4370 0 : break;
4371 : }
4372 0 : num_ev++;
4373 : }
4374 : }
4375 :
4376 : if (0) // Note "done:" label used by VCL_SESSION_LOCK_AND_GET()
4377 : {
4378 : vp[i].revents = POLLNVAL;
4379 : num_ev++;
4380 : }
4381 : }
4382 0 : if (wait_for_time != -1)
4383 0 : keep_trying = (clib_time_now (&wrk->clib_time) <= timeout) ? 1 : 0;
4384 : }
4385 0 : while ((num_ev == 0) && keep_trying);
4386 :
4387 0 : return num_ev;
4388 : }
4389 :
4390 : int
4391 0 : vppcom_mq_epoll_fd (void)
4392 : {
4393 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4394 0 : return wrk->mqs_epfd;
4395 : }
4396 :
4397 : int
4398 1585120 : vppcom_session_index (vcl_session_handle_t session_handle)
4399 : {
4400 1585120 : return session_handle & 0xFFFFFF;
4401 : }
4402 :
4403 : int
4404 47 : vppcom_session_worker (vcl_session_handle_t session_handle)
4405 : {
4406 47 : return session_handle >> 24;
4407 : }
4408 :
4409 : int
4410 0 : vppcom_worker_register (void)
4411 : {
4412 0 : if (!vcl_worker_alloc_and_init ())
4413 0 : return VPPCOM_EEXIST;
4414 :
4415 0 : if (vcl_worker_register_with_vpp ())
4416 0 : return VPPCOM_EEXIST;
4417 :
4418 0 : return VPPCOM_OK;
4419 : }
4420 :
4421 : void
4422 0 : vppcom_worker_unregister (void)
4423 : {
4424 0 : vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
4425 0 : vcl_set_worker_index (~0);
4426 0 : }
4427 :
4428 : void
4429 0 : vppcom_worker_index_set (int index)
4430 : {
4431 0 : vcl_set_worker_index (index);
4432 0 : }
4433 :
4434 : int
4435 442235 : vppcom_worker_index (void)
4436 : {
4437 442235 : return vcl_get_worker_index ();
4438 : }
4439 :
4440 : int
4441 0 : vppcom_worker_mqs_epfd (void)
4442 : {
4443 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4444 0 : if (!vcm->cfg.use_mq_eventfd)
4445 0 : return -1;
4446 0 : return wrk->mqs_epfd;
4447 : }
4448 :
4449 : int
4450 4725 : vppcom_session_is_connectable_listener (uint32_t session_handle)
4451 : {
4452 : vcl_session_t *session;
4453 4725 : vcl_worker_t *wrk = vcl_worker_get_current ();
4454 4725 : session = vcl_session_get_w_handle (wrk, session_handle);
4455 4725 : if (!session)
4456 0 : return VPPCOM_EBADFD;
4457 4725 : return vcl_session_is_connectable_listener (wrk, session);
4458 : }
4459 :
4460 : int
4461 0 : vppcom_session_listener (uint32_t session_handle)
4462 : {
4463 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4464 : vcl_session_t *listen_session, *session;
4465 0 : session = vcl_session_get_w_handle (wrk, session_handle);
4466 0 : if (!session)
4467 0 : return VPPCOM_EBADFD;
4468 0 : if (session->listener_index == VCL_INVALID_SESSION_INDEX)
4469 0 : return VPPCOM_EBADFD;
4470 0 : listen_session = vcl_session_get_w_handle (wrk, session->listener_index);
4471 0 : if (!listen_session)
4472 0 : return VPPCOM_EBADFD;
4473 0 : return vcl_session_handle (listen_session);
4474 : }
4475 :
4476 : int
4477 0 : vppcom_session_n_accepted (uint32_t session_handle)
4478 : {
4479 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4480 0 : vcl_session_t *session = vcl_session_get_w_handle (wrk, session_handle);
4481 0 : if (!session)
4482 0 : return VPPCOM_EBADFD;
4483 0 : return session->n_accepted_sessions;
4484 : }
4485 :
4486 : const char *
4487 122 : vppcom_proto_str (vppcom_proto_t proto)
4488 : {
4489 : char const *proto_str;
4490 :
4491 122 : switch (proto)
4492 : {
4493 104 : case VPPCOM_PROTO_TCP:
4494 104 : proto_str = "TCP";
4495 104 : break;
4496 12 : case VPPCOM_PROTO_UDP:
4497 12 : proto_str = "UDP";
4498 12 : break;
4499 4 : case VPPCOM_PROTO_TLS:
4500 4 : proto_str = "TLS";
4501 4 : break;
4502 0 : case VPPCOM_PROTO_QUIC:
4503 0 : proto_str = "QUIC";
4504 0 : break;
4505 2 : case VPPCOM_PROTO_DTLS:
4506 2 : proto_str = "DTLS";
4507 2 : break;
4508 0 : case VPPCOM_PROTO_SRTP:
4509 0 : proto_str = "SRTP";
4510 0 : break;
4511 0 : default:
4512 0 : proto_str = "UNKNOWN";
4513 0 : break;
4514 : }
4515 122 : return proto_str;
4516 : }
4517 :
4518 : const char *
4519 0 : vppcom_retval_str (int retval)
4520 : {
4521 : char const *st;
4522 :
4523 0 : switch (retval)
4524 : {
4525 0 : case VPPCOM_OK:
4526 0 : st = "VPPCOM_OK";
4527 0 : break;
4528 :
4529 0 : case VPPCOM_EAGAIN:
4530 0 : st = "VPPCOM_EAGAIN";
4531 0 : break;
4532 :
4533 0 : case VPPCOM_EFAULT:
4534 0 : st = "VPPCOM_EFAULT";
4535 0 : break;
4536 :
4537 0 : case VPPCOM_ENOMEM:
4538 0 : st = "VPPCOM_ENOMEM";
4539 0 : break;
4540 :
4541 0 : case VPPCOM_EINVAL:
4542 0 : st = "VPPCOM_EINVAL";
4543 0 : break;
4544 :
4545 0 : case VPPCOM_EBADFD:
4546 0 : st = "VPPCOM_EBADFD";
4547 0 : break;
4548 :
4549 0 : case VPPCOM_EAFNOSUPPORT:
4550 0 : st = "VPPCOM_EAFNOSUPPORT";
4551 0 : break;
4552 :
4553 0 : case VPPCOM_ECONNABORTED:
4554 0 : st = "VPPCOM_ECONNABORTED";
4555 0 : break;
4556 :
4557 0 : case VPPCOM_ECONNRESET:
4558 0 : st = "VPPCOM_ECONNRESET";
4559 0 : break;
4560 :
4561 0 : case VPPCOM_ENOTCONN:
4562 0 : st = "VPPCOM_ENOTCONN";
4563 0 : break;
4564 :
4565 0 : case VPPCOM_ECONNREFUSED:
4566 0 : st = "VPPCOM_ECONNREFUSED";
4567 0 : break;
4568 :
4569 0 : case VPPCOM_ETIMEDOUT:
4570 0 : st = "VPPCOM_ETIMEDOUT";
4571 0 : break;
4572 :
4573 0 : case VPPCOM_EADDRINUSE:
4574 0 : st = "VPPCOM_EADDRINUSE";
4575 0 : break;
4576 :
4577 0 : default:
4578 0 : st = "UNKNOWN_STATE";
4579 0 : break;
4580 : }
4581 :
4582 0 : return st;
4583 : }
4584 :
4585 : int
4586 6 : vppcom_add_cert_key_pair (vppcom_cert_key_pair_t *ckpair)
4587 : {
4588 6 : if (vcm->cfg.vpp_app_socket_api)
4589 4 : return vcl_sapi_add_cert_key_pair (ckpair);
4590 : else
4591 2 : return vcl_bapi_add_cert_key_pair (ckpair);
4592 : }
4593 :
4594 : int
4595 0 : vppcom_del_cert_key_pair (uint32_t ckpair_index)
4596 : {
4597 0 : if (vcm->cfg.vpp_app_socket_api)
4598 0 : return vcl_sapi_del_cert_key_pair (ckpair_index);
4599 : else
4600 0 : return vcl_bapi_del_cert_key_pair (ckpair_index);
4601 : }
4602 :
4603 : int
4604 0 : vppcom_session_get_error (uint32_t session_handle)
4605 : {
4606 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4607 0 : vcl_session_t *session = 0;
4608 :
4609 0 : session = vcl_session_get_w_handle (wrk, session_handle);
4610 0 : if (!session)
4611 0 : return VPPCOM_EBADFD;
4612 :
4613 0 : if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
4614 : {
4615 0 : VWRN ("epoll session %u! will not have connect", session->session_index);
4616 0 : return VPPCOM_EBADFD;
4617 : }
4618 :
4619 0 : if (session->vpp_error == SESSION_E_PORTINUSE)
4620 0 : return VPPCOM_EADDRINUSE;
4621 0 : else if (session->vpp_error == SESSION_E_REFUSED)
4622 0 : return VPPCOM_ECONNREFUSED;
4623 0 : else if (session->vpp_error != SESSION_E_NONE)
4624 0 : return VPPCOM_EFAULT;
4625 : else
4626 0 : return VPPCOM_OK;
4627 : }
4628 :
4629 : int
4630 0 : vppcom_worker_is_detached (void)
4631 : {
4632 0 : vcl_worker_t *wrk = vcl_worker_get_current ();
4633 :
4634 0 : if (!vcm->cfg.use_mq_eventfd)
4635 0 : return VPPCOM_ENOTSUP;
4636 :
4637 0 : return wrk->api_client_handle == ~0;
4638 : }
4639 :
4640 : /*
4641 : * fd.io coding-style-patch-verification: ON
4642 : *
4643 : * Local Variables:
4644 : * eval: (c-set-style "gnu")
4645 : * End:
4646 : */
|
