Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2016-2020 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #include <vnet/udp/udp.h>
      17             : #include <vnet/session/session.h>
      18             : #include <vnet/dpo/load_balance.h>
      19             : #include <vnet/ip/ip4_inlines.h>
      20             : #include <vnet/ip/ip6_inlines.h>
      21             : #include <vppinfra/sparse_vec.h>
      22             : 
      23             : udp_main_t udp_main;
      24             : 
      25             : static void
      26          34 : udp_connection_register_port (u16 lcl_port, u8 is_ip4)
      27             : {
      28          34 :   udp_main_t *um = &udp_main;
      29             :   u16 *n;
      30             : 
      31             :   /* Setup udp protocol -> next index sparse vector mapping. Do not setup
      32             :    * udp_dst_port_info_t as that is used to distinguish between external
      33             :    * and transport consumed ports */
      34             : 
      35          34 :   if (is_ip4)
      36          34 :     n = sparse_vec_validate (um->next_by_dst_port4, lcl_port);
      37             :   else
      38           0 :     n = sparse_vec_validate (um->next_by_dst_port6, lcl_port);
      39             : 
      40          34 :   n[0] = um->local_to_input_edge[is_ip4];
      41             : 
      42          34 :   __atomic_add_fetch (&um->transport_ports_refcnt[is_ip4][lcl_port], 1,
      43             :                       __ATOMIC_RELAXED);
      44          34 : }
      45             : 
      46             : static void
      47          29 : udp_connection_unregister_port (u16 lcl_port, u8 is_ip4)
      48             : {
      49          29 :   udp_main_t *um = &udp_main;
      50             :   u16 *n;
      51             : 
      52             :   /* Needed because listeners are not tracked as local endpoints */
      53          29 :   if (__atomic_sub_fetch (&um->transport_ports_refcnt[is_ip4][lcl_port], 1,
      54             :                           __ATOMIC_RELAXED))
      55           3 :     return;
      56             : 
      57          26 :   if (is_ip4)
      58          26 :     n = sparse_vec_validate (um->next_by_dst_port4, lcl_port);
      59             :   else
      60           0 :     n = sparse_vec_validate (um->next_by_dst_port6, lcl_port);
      61             : 
      62          26 :   n[0] = UDP_NO_NODE_SET;
      63             : }
      64             : 
      65             : udp_connection_t *
      66          44 : udp_connection_alloc (u32 thread_index)
      67             : {
      68          44 :   udp_worker_t *wrk = udp_worker_get (thread_index);
      69             :   udp_connection_t *uc;
      70             : 
      71          44 :   pool_get_aligned_safe (wrk->connections, uc, CLIB_CACHE_LINE_BYTES);
      72             : 
      73          44 :   clib_memset (uc, 0, sizeof (*uc));
      74          44 :   uc->c_c_index = uc - wrk->connections;
      75          44 :   uc->c_thread_index = thread_index;
      76          44 :   uc->c_proto = TRANSPORT_PROTO_UDP;
      77          44 :   return uc;
      78             : }
      79             : 
      80             : void
      81          43 : udp_connection_free (udp_connection_t * uc)
      82             : {
      83          43 :   udp_worker_t *wrk = udp_worker_get (uc->c_thread_index);
      84             : 
      85          43 :   clib_spinlock_free (&uc->rx_lock);
      86             :   if (CLIB_DEBUG)
      87          43 :     clib_memset (uc, 0xFA, sizeof (*uc));
      88          43 :   pool_put (wrk->connections, uc);
      89          43 : }
      90             : 
      91             : static void
      92          43 : udp_connection_cleanup (udp_connection_t * uc)
      93             : {
      94             :   /* Unregister port from udp_local only if refcount went to zero */
      95          43 :   if (!transport_release_local_endpoint (TRANSPORT_PROTO_UDP, &uc->c_lcl_ip,
      96          43 :                                          uc->c_lcl_port))
      97          23 :     udp_connection_unregister_port (uc->c_lcl_port, uc->c_is_ip4);
      98          43 :   udp_connection_free (uc);
      99          43 : }
     100             : 
     101             : void
     102          43 : udp_connection_delete (udp_connection_t * uc)
     103             : {
     104          43 :   session_transport_delete_notify (&uc->connection);
     105          43 :   udp_connection_cleanup (uc);
     106          43 : }
     107             : 
     108             : static void
     109          33 : udp_handle_cleanups (void *args)
     110             : {
     111          33 :   u32 thread_index = (u32) pointer_to_uword (args);
     112             :   udp_connection_t *uc;
     113             :   udp_worker_t *wrk;
     114             :   u32 *uc_index;
     115             : 
     116          33 :   wrk = udp_worker_get (thread_index);
     117          76 :   vec_foreach (uc_index, wrk->pending_cleanups)
     118             :     {
     119          43 :       uc = udp_connection_get (*uc_index, thread_index);
     120          43 :       udp_connection_delete (uc);
     121             :     }
     122          33 :   vec_reset_length (wrk->pending_cleanups);
     123          33 : }
     124             : 
     125             : static void
     126          43 : udp_connection_program_cleanup (udp_connection_t *uc)
     127             : {
     128          43 :   uword thread_index = uc->c_thread_index;
     129             :   udp_worker_t *wrk;
     130             : 
     131          43 :   wrk = udp_worker_get (uc->c_thread_index);
     132          43 :   vec_add1 (wrk->pending_cleanups, uc->c_c_index);
     133             : 
     134          43 :   if (vec_len (wrk->pending_cleanups) == 1)
     135          33 :     session_send_rpc_evt_to_thread_force (
     136             :       thread_index, udp_handle_cleanups,
     137             :       uword_to_pointer (thread_index, void *));
     138          43 : }
     139             : 
     140             : static u8
     141          34 : udp_connection_port_used_extern (u16 lcl_port, u8 is_ip4)
     142             : {
     143          34 :   udp_main_t *um = vnet_get_udp_main ();
     144             :   udp_dst_port_info_t *pi;
     145             : 
     146          34 :   pi = udp_get_dst_port_info (um, lcl_port, is_ip4);
     147          34 :   return (pi && udp_is_valid_dst_port (lcl_port, is_ip4));
     148             : }
     149             : 
     150             : static u16
     151          34 : udp_default_mtu (udp_main_t * um, u8 is_ip4)
     152             : {
     153          34 :   u16 ip_hlen = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
     154          34 :   return (um->default_mtu - sizeof (udp_header_t) - ip_hlen);
     155             : }
     156             : 
     157             : static u32
     158          10 : udp_session_bind (u32 session_index, transport_endpoint_cfg_t *lcl)
     159             : {
     160          10 :   udp_main_t *um = vnet_get_udp_main ();
     161             :   transport_endpoint_cfg_t *lcl_ext;
     162             :   udp_connection_t *listener;
     163             :   void *iface_ip;
     164             : 
     165          10 :   if (udp_connection_port_used_extern (clib_net_to_host_u16 (lcl->port),
     166          10 :                                        lcl->is_ip4))
     167             :     {
     168           0 :       clib_warning ("port already used");
     169           0 :       return SESSION_E_PORTINUSE;
     170             :     }
     171             : 
     172          10 :   pool_get (um->listener_pool, listener);
     173          10 :   clib_memset (listener, 0, sizeof (udp_connection_t));
     174             : 
     175          10 :   listener->c_lcl_port = lcl->port;
     176          10 :   listener->c_c_index = listener - um->listener_pool;
     177             : 
     178             :   /* If we are provided a sw_if_index, bind using one of its ips */
     179          10 :   if (ip_is_zero (&lcl->ip, 1) && lcl->sw_if_index != ENDPOINT_INVALID_INDEX)
     180             :     {
     181           4 :       if ((iface_ip = ip_interface_get_first_ip (lcl->sw_if_index,
     182           4 :                                                  lcl->is_ip4)))
     183           4 :         ip_set (&lcl->ip, iface_ip, lcl->is_ip4);
     184             :     }
     185          10 :   ip_copy (&listener->c_lcl_ip, &lcl->ip, lcl->is_ip4);
     186          10 :   listener->c_is_ip4 = lcl->is_ip4;
     187          10 :   listener->c_proto = TRANSPORT_PROTO_UDP;
     188          10 :   listener->c_s_index = session_index;
     189          10 :   listener->c_fib_index = lcl->fib_index;
     190           0 :   listener->mss =
     191          10 :     lcl->mss ? lcl->mss : udp_default_mtu (um, listener->c_is_ip4);
     192          10 :   listener->flags |= UDP_CONN_F_OWNS_PORT | UDP_CONN_F_LISTEN;
     193          10 :   lcl_ext = (transport_endpoint_cfg_t *) lcl;
     194          10 :   if (lcl_ext->transport_flags & TRANSPORT_CFG_F_CONNECTED)
     195           7 :     listener->flags |= UDP_CONN_F_CONNECTED;
     196             :   else
     197           3 :     listener->c_flags |= TRANSPORT_CONNECTION_F_CLESS;
     198          10 :   clib_spinlock_init (&listener->rx_lock);
     199          10 :   if (!um->csum_offload)
     200           0 :     listener->cfg_flags |= UDP_CFG_F_NO_CSUM_OFFLOAD;
     201             : 
     202          10 :   udp_connection_register_port (listener->c_lcl_port, lcl->is_ip4);
     203          10 :   return listener->c_c_index;
     204             : }
     205             : 
     206             : static u32
     207           6 : udp_session_unbind (u32 listener_index)
     208             : {
     209           6 :   udp_main_t *um = &udp_main;
     210             :   udp_connection_t *listener;
     211             : 
     212           6 :   listener = udp_listener_get (listener_index);
     213           6 :   udp_connection_unregister_port (listener->c_lcl_port, listener->c_is_ip4);
     214           6 :   clib_spinlock_free (&listener->rx_lock);
     215           6 :   pool_put (um->listener_pool, listener);
     216           6 :   return 0;
     217             : }
     218             : 
     219             : static transport_connection_t *
     220          55 : udp_session_get_listener (u32 listener_index)
     221             : {
     222             :   udp_connection_t *us;
     223             : 
     224          55 :   us = udp_listener_get (listener_index);
     225          55 :   return &us->connection;
     226             : }
     227             : 
     228             : always_inline u32
     229       58172 : udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b,
     230             :                      u8 is_cless)
     231             : {
     232       58172 :   b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
     233             :   /* reuse tcp medatada for now */
     234       58172 :   vnet_buffer (b)->tcp.connection_index = uc->c_c_index;
     235             : 
     236       58172 :   if (!is_cless)
     237             :     {
     238       58172 :       vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port,
     239       58172 :                             udp_csum_offload (uc));
     240             : 
     241       58172 :       if (uc->c_is_ip4)
     242       58172 :         vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4,
     243       58172 :                                      IP_PROTOCOL_UDP, udp_csum_offload (uc),
     244       58172 :                                      0 /* is_df */, uc->c_dscp);
     245             :       else
     246           0 :         vlib_buffer_push_ip6 (vm, b, &uc->c_lcl_ip6, &uc->c_rmt_ip6,
     247             :                               IP_PROTOCOL_UDP);
     248             : 
     249       58172 :       vnet_buffer (b)->tcp.flags = 0;
     250             :     }
     251             :   else
     252             :     {
     253           0 :       u8 *data = vlib_buffer_get_current (b);
     254             :       session_dgram_hdr_t hdr;
     255             : 
     256           0 :       hdr = *(session_dgram_hdr_t *) (data - sizeof (hdr));
     257             : 
     258             :       /* Local port assumed to be bound, not overwriting it */
     259           0 :       vlib_buffer_push_udp (b, uc->c_lcl_port, hdr.rmt_port,
     260           0 :                             udp_csum_offload (uc));
     261             : 
     262           0 :       if (uc->c_is_ip4)
     263           0 :         vlib_buffer_push_ip4_custom (vm, b, &hdr.lcl_ip.ip4, &hdr.rmt_ip.ip4,
     264           0 :                                      IP_PROTOCOL_UDP, udp_csum_offload (uc),
     265           0 :                                      0 /* is_df */, uc->c_dscp);
     266             :       else
     267           0 :         vlib_buffer_push_ip6 (vm, b, &hdr.lcl_ip.ip6, &hdr.rmt_ip.ip6,
     268             :                               IP_PROTOCOL_UDP);
     269             : 
     270             :       /* Not connected udp session. Mark buffer for custom handling in
     271             :        * udp_output */
     272           0 :       vnet_buffer (b)->tcp.flags |= UDP_CONN_F_LISTEN;
     273             :     }
     274             : 
     275       58172 :   return 0;
     276             : }
     277             : 
     278             : always_inline void
     279       16427 : udp_push_header_batch (udp_connection_t *uc, vlib_buffer_t **bs, u32 n_bufs,
     280             :                        u8 is_cless)
     281             : {
     282       16427 :   vlib_main_t *vm = vlib_get_main ();
     283             : 
     284       33150 :   while (n_bufs >= 4)
     285             :     {
     286       16723 :       vlib_prefetch_buffer_header (bs[2], STORE);
     287       16723 :       vlib_prefetch_buffer_header (bs[3], STORE);
     288             : 
     289       16723 :       udp_push_one_header (vm, uc, bs[0], is_cless);
     290       16723 :       udp_push_one_header (vm, uc, bs[1], is_cless);
     291             : 
     292       16723 :       n_bufs -= 2;
     293       16723 :       bs += 2;
     294             :     }
     295       41153 :   while (n_bufs)
     296             :     {
     297       24726 :       if (n_bufs > 1)
     298        8299 :         vlib_prefetch_buffer_header (bs[1], STORE);
     299             : 
     300       24726 :       udp_push_one_header (vm, uc, bs[0], is_cless);
     301             : 
     302       24726 :       n_bufs -= 1;
     303       24726 :       bs += 1;
     304             :     }
     305       16427 : }
     306             : 
     307             : static u32
     308       16427 : udp_push_header (transport_connection_t *tc, vlib_buffer_t **bs, u32 n_bufs)
     309             : {
     310             :   udp_connection_t *uc;
     311             : 
     312       16427 :   uc = udp_connection_from_transport (tc);
     313       16427 :   if (uc->flags & UDP_CONN_F_CONNECTED)
     314       16427 :     udp_push_header_batch (uc, bs, n_bufs, 0 /* is_cless */);
     315             :   else
     316           0 :     udp_push_header_batch (uc, bs, n_bufs, 1 /* is_cless */);
     317             : 
     318       16427 :   if (PREDICT_FALSE (uc->flags & UDP_CONN_F_CLOSING))
     319             :     {
     320           2 :       if (!transport_tx_fifo_has_dgram (&uc->connection))
     321           2 :         udp_connection_program_cleanup (uc);
     322             :     }
     323             : 
     324       16427 :   return 0;
     325             : }
     326             : 
     327             : static transport_connection_t *
     328      170990 : udp_session_get (u32 connection_index, u32 thread_index)
     329             : {
     330             :   udp_connection_t *uc;
     331      170990 :   uc = udp_connection_get (connection_index, thread_index);
     332      170990 :   if (uc)
     333      170990 :     return &uc->connection;
     334           0 :   return 0;
     335             : }
     336             : 
     337             : static void
     338          43 : udp_session_close (u32 connection_index, u32 thread_index)
     339             : {
     340             :   udp_connection_t *uc;
     341             : 
     342          43 :   uc = udp_connection_get (connection_index, thread_index);
     343          43 :   if (!uc || (uc->flags & UDP_CONN_F_MIGRATED))
     344           0 :     return;
     345             : 
     346          43 :   if (!transport_tx_fifo_has_dgram (&uc->connection))
     347          41 :     udp_connection_program_cleanup (uc);
     348             :   else
     349           2 :     uc->flags |= UDP_CONN_F_CLOSING;
     350             : }
     351             : 
     352             : static void
     353           0 : udp_session_cleanup (u32 connection_index, u32 thread_index)
     354             : {
     355             :   udp_connection_t *uc;
     356           0 :   uc = udp_connection_get (connection_index, thread_index);
     357           0 :   if (!uc)
     358           0 :     return;
     359           0 :   if (uc->flags & UDP_CONN_F_MIGRATED)
     360           0 :     udp_connection_free (uc);
     361             :   else
     362           0 :     udp_connection_cleanup (uc);
     363             : }
     364             : 
     365             : static int
     366       17228 : udp_session_send_params (transport_connection_t * tconn,
     367             :                          transport_send_params_t * sp)
     368             : {
     369             :   udp_connection_t *uc;
     370             : 
     371       17228 :   uc = udp_connection_from_transport (tconn);
     372             : 
     373             :   /* No constraint on TX window */
     374       17228 :   sp->snd_space = ~0;
     375             :   /* TODO figure out MTU of output interface */
     376       17228 :   sp->snd_mss = uc->mss;
     377       17228 :   sp->tx_offset = 0;
     378       17228 :   sp->flags = 0;
     379       17228 :   return 0;
     380             : }
     381             : 
     382             : static int
     383          24 : udp_open_connection (transport_endpoint_cfg_t * rmt)
     384             : {
     385          24 :   udp_main_t *um = &udp_main;
     386             :   ip46_address_t lcl_addr;
     387             :   udp_connection_t *uc;
     388             :   u32 thread_index;
     389             :   u16 lcl_port;
     390             :   int rv;
     391             : 
     392          24 :   rv = transport_alloc_local_endpoint (TRANSPORT_PROTO_UDP, rmt, &lcl_addr,
     393             :                                        &lcl_port);
     394          24 :   if (rv)
     395           0 :     return rv;
     396             : 
     397          24 :   if (udp_connection_port_used_extern (clib_net_to_host_u16 (lcl_port),
     398          24 :                                        rmt->is_ip4))
     399             :     {
     400             :       /* If specific source port was requested abort */
     401           0 :       if (rmt->peer.port)
     402             :         {
     403           0 :           transport_release_local_endpoint (TRANSPORT_PROTO_UDP, &lcl_addr,
     404             :                                             lcl_port);
     405           0 :           return SESSION_E_PORTINUSE;
     406             :         }
     407             : 
     408             :       /* Try to find a port that's not used */
     409           0 :       while (udp_connection_port_used_extern (clib_net_to_host_u16 (lcl_port),
     410           0 :                                               rmt->is_ip4))
     411             :         {
     412           0 :           transport_release_local_endpoint (TRANSPORT_PROTO_UDP, &lcl_addr,
     413             :                                             lcl_port);
     414           0 :           lcl_port =
     415           0 :             transport_alloc_local_port (TRANSPORT_PROTO_UDP, &lcl_addr, rmt);
     416           0 :           if (lcl_port < 1)
     417           0 :             return SESSION_E_PORTINUSE;
     418             :         }
     419             :     }
     420             : 
     421             :   /* We don't poll main thread if we have workers */
     422          24 :   thread_index = transport_cl_thread ();
     423             : 
     424          24 :   uc = udp_connection_alloc (thread_index);
     425          24 :   ip_copy (&uc->c_rmt_ip, &rmt->ip, rmt->is_ip4);
     426          24 :   ip_copy (&uc->c_lcl_ip, &lcl_addr, rmt->is_ip4);
     427          24 :   uc->c_rmt_port = rmt->port;
     428          24 :   uc->c_lcl_port = clib_host_to_net_u16 (lcl_port);
     429          24 :   uc->c_is_ip4 = rmt->is_ip4;
     430          24 :   uc->c_proto = TRANSPORT_PROTO_UDP;
     431          24 :   uc->c_fib_index = rmt->fib_index;
     432          24 :   uc->c_dscp = rmt->dscp;
     433          24 :   uc->mss = rmt->mss ? rmt->mss : udp_default_mtu (um, uc->c_is_ip4);
     434          24 :   if (rmt->peer.sw_if_index != ENDPOINT_INVALID_INDEX)
     435          24 :     uc->sw_if_index = rmt->peer.sw_if_index;
     436          24 :   uc->flags |= UDP_CONN_F_OWNS_PORT;
     437          24 :   if (rmt->transport_flags & TRANSPORT_CFG_F_CONNECTED)
     438             :     {
     439          24 :       uc->flags |= UDP_CONN_F_CONNECTED;
     440             :     }
     441             :   else
     442             :     {
     443           0 :       clib_spinlock_init (&uc->rx_lock);
     444           0 :       uc->c_flags |= TRANSPORT_CONNECTION_F_CLESS;
     445             :     }
     446          24 :   if (!um->csum_offload)
     447           0 :     uc->cfg_flags |= UDP_CFG_F_NO_CSUM_OFFLOAD;
     448          24 :   uc->next_node_index = rmt->next_node_index;
     449          24 :   uc->next_node_opaque = rmt->next_node_opaque;
     450             : 
     451          24 :   udp_connection_register_port (uc->c_lcl_port, rmt->is_ip4);
     452             : 
     453          24 :   return uc->c_c_index;
     454             : }
     455             : 
     456             : static transport_connection_t *
     457          24 : udp_session_get_half_open (u32 conn_index)
     458             : {
     459             :   udp_connection_t *uc;
     460             :   u32 thread_index;
     461             : 
     462             :   /* We don't poll main thread if we have workers */
     463          24 :   thread_index = transport_cl_thread ();
     464          24 :   uc = udp_connection_get (conn_index, thread_index);
     465          24 :   if (!uc)
     466           0 :     return 0;
     467          24 :   return &uc->connection;
     468             : }
     469             : 
     470             : static u8 *
     471           1 : format_udp_session (u8 * s, va_list * args)
     472             : {
     473           1 :   u32 uci = va_arg (*args, u32);
     474           1 :   u32 thread_index = va_arg (*args, u32);
     475           1 :   u32 verbose = va_arg (*args, u32);
     476             :   udp_connection_t *uc;
     477             : 
     478           1 :   uc = udp_connection_get (uci, thread_index);
     479           1 :   return format (s, "%U", format_udp_connection, uc, verbose);
     480             : }
     481             : 
     482             : static u8 *
     483           0 : format_udp_half_open_session (u8 * s, va_list * args)
     484             : {
     485           0 :   u32 __clib_unused tci = va_arg (*args, u32);
     486           0 :   u32 __clib_unused thread_index = va_arg (*args, u32);
     487           0 :   clib_warning ("BUG");
     488           0 :   return 0;
     489             : }
     490             : 
     491             : static u8 *
     492           1 : format_udp_listener_session (u8 * s, va_list * args)
     493             : {
     494           1 :   u32 tci = va_arg (*args, u32);
     495           1 :   u32 __clib_unused thread_index = va_arg (*args, u32);
     496           1 :   u32 verbose = va_arg (*args, u32);
     497           1 :   udp_connection_t *uc = udp_listener_get (tci);
     498           1 :   return format (s, "%U", format_udp_connection, uc, verbose);
     499             : }
     500             : 
     501             : static void
     502         114 : udp_realloc_ports_sv (u16 **ports_nh_svp)
     503             : {
     504             :   u16 port, port_no, *ports_nh_sv, *mc;
     505         114 :   u32 *ports = 0, *nh = 0, msum, i;
     506             :   sparse_vec_header_t *h;
     507             :   uword sv_index, *mb;
     508             : 
     509         114 :   ports_nh_sv = *ports_nh_svp;
     510             : 
     511     7470990 :   for (port = 1; port < 65535; port++)
     512             :     {
     513     7470880 :       port_no = clib_host_to_net_u16 (port);
     514             : 
     515     7470880 :       sv_index = sparse_vec_index (ports_nh_sv, port_no);
     516     7470880 :       if (sv_index != SPARSE_VEC_INVALID_INDEX)
     517             :         {
     518     1048540 :           vec_add1 (ports, port_no);
     519     1048540 :           vec_add1 (nh, ports_nh_sv[sv_index]);
     520             :         }
     521             :     }
     522             : 
     523         114 :   sparse_vec_free (ports_nh_sv);
     524             : 
     525         114 :   ports_nh_sv =
     526         114 :     sparse_vec_new (/* elt bytes */ sizeof (ports_nh_sv[0]),
     527             :                     /* bits in index */ BITS (((udp_header_t *) 0)->dst_port));
     528             : 
     529         114 :   vec_resize (ports_nh_sv, 65535);
     530             : 
     531     7470990 :   for (port = 1; port < 65535; port++)
     532     7470880 :     ports_nh_sv[port] = UDP_NO_NODE_SET;
     533             : 
     534     1048660 :   for (i = 0; i < vec_len (ports); i++)
     535     1048540 :     ports_nh_sv[ports[i]] = nh[i];
     536             : 
     537         114 :   h = sparse_vec_header (ports_nh_sv);
     538      116850 :   vec_foreach (mb, h->is_member_bitmap)
     539      116736 :     *mb = (uword) ~0;
     540             : 
     541         114 :   msum = 0;
     542      116850 :   vec_foreach (mc, h->member_counts)
     543             :     {
     544      116736 :       *mc = msum;
     545      116736 :       msum += msum == 0 ? 63 : 64;
     546             :     }
     547             : 
     548         114 :   vec_free (ports);
     549         114 :   vec_free (nh);
     550             : 
     551         114 :   *ports_nh_svp = ports_nh_sv;
     552         114 : }
     553             : 
     554             : static clib_error_t *
     555          57 : udp_enable_disable (vlib_main_t *vm, u8 is_en)
     556             : {
     557          57 :   udp_main_t *um = &udp_main;
     558             : 
     559             :   /* Not ideal. The sparse vector used to map ports to next nodes assumes
     560             :    * only a few ports are ever used. When udp transport is enabled this does
     561             :    * not hold and, to make matters worse, ports are consumed in a random
     562             :    * order.
     563             :    *
     564             :    * This can lead to a lot of slow updates to internal data structures
     565             :    * which in turn can slow udp connection allocations until all ports are
     566             :    * eventually consumed.
     567             :    *
     568             :    * Consequently, reallocate sparse vector, preallocate all ports and have
     569             :    * them point to UDP_NO_NODE_SET. We could consider switching the sparse
     570             :    * vector to a preallocated vector but that would increase memory
     571             :    * consumption for vpp deployments that do not rely on host stack.
     572             :    */
     573             : 
     574          57 :   udp_realloc_ports_sv (&um->next_by_dst_port4);
     575          57 :   udp_realloc_ports_sv (&um->next_by_dst_port6);
     576             : 
     577          57 :   vec_validate (um->transport_ports_refcnt[0], 65535);
     578          57 :   vec_validate (um->transport_ports_refcnt[1], 65535);
     579             : 
     580          57 :   return 0;
     581             : }
     582             : 
     583             : static const transport_proto_vft_t udp_proto = {
     584             :   .enable = udp_enable_disable,
     585             :   .start_listen = udp_session_bind,
     586             :   .connect = udp_open_connection,
     587             :   .stop_listen = udp_session_unbind,
     588             :   .push_header = udp_push_header,
     589             :   .get_connection = udp_session_get,
     590             :   .get_listener = udp_session_get_listener,
     591             :   .get_half_open = udp_session_get_half_open,
     592             :   .close = udp_session_close,
     593             :   .cleanup = udp_session_cleanup,
     594             :   .send_params = udp_session_send_params,
     595             :   .format_connection = format_udp_session,
     596             :   .format_half_open = format_udp_half_open_session,
     597             :   .format_listener = format_udp_listener_session,
     598             :   .transport_options = {
     599             :     .name = "udp",
     600             :     .short_name = "U",
     601             :     .tx_type = TRANSPORT_TX_DGRAM,
     602             :     .service_type = TRANSPORT_SERVICE_CL,
     603             :   },
     604             : };
     605             : 
     606             : static clib_error_t *
     607         559 : udp_init (vlib_main_t * vm)
     608             : {
     609         559 :   udp_main_t *um = vnet_get_udp_main ();
     610         559 :   ip_main_t *im = &ip_main;
     611         559 :   vlib_thread_main_t *tm = vlib_get_thread_main ();
     612             :   u32 num_threads;
     613             :   ip_protocol_info_t *pi;
     614             : 
     615             :   /*
     616             :    * Registrations
     617             :    */
     618             : 
     619             :   /* IP registration */
     620         559 :   pi = ip_get_protocol_info (im, IP_PROTOCOL_UDP);
     621         559 :   if (pi == 0)
     622           0 :     return clib_error_return (0, "UDP protocol info AWOL");
     623         559 :   pi->format_header = format_udp_header;
     624         559 :   pi->unformat_pg_edit = unformat_pg_udp_header;
     625             : 
     626             :   /* Register as transport with session layer */
     627         559 :   transport_register_protocol (TRANSPORT_PROTO_UDP, &udp_proto,
     628             :                                FIB_PROTOCOL_IP4, udp4_output_node.index);
     629         559 :   transport_register_protocol (TRANSPORT_PROTO_UDP, &udp_proto,
     630             :                                FIB_PROTOCOL_IP6, udp6_output_node.index);
     631             : 
     632             :   /*
     633             :    * Initialize data structures
     634             :    */
     635             : 
     636         559 :   num_threads = 1 /* main thread */  + tm->n_threads;
     637         559 :   vec_validate (um->wrk, num_threads - 1);
     638             : 
     639         559 :   um->local_to_input_edge[UDP_IP4] =
     640         559 :     vlib_node_add_next (vm, udp4_local_node.index, udp4_input_node.index);
     641         559 :   um->local_to_input_edge[UDP_IP6] =
     642         559 :     vlib_node_add_next (vm, udp6_local_node.index, udp6_input_node.index);
     643             : 
     644         559 :   um->default_mtu = 1500;
     645         559 :   um->csum_offload = 1;
     646         559 :   return 0;
     647             : }
     648             : 
     649             : /* *INDENT-OFF* */
     650       17919 : VLIB_INIT_FUNCTION (udp_init) =
     651             : {
     652             :   .runs_after = VLIB_INITS("ip_main_init", "ip4_lookup_init",
     653             :                            "ip6_lookup_init"),
     654             : };
     655             : /* *INDENT-ON* */
     656             : 
     657             : /*
     658             :  * fd.io coding-style-patch-verification: ON
     659             :  *
     660             :  * Local Variables:
     661             :  * eval: (c-set-style "gnu")
     662             :  * End:
     663             :  */