Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #include <vlibmemory/api.h>
      17             : #include <vlib/vlib.h>
      18             : 
      19             : #include <vppinfra/hash.h>
      20             : #include <vppinfra/error.h>
      21             : #include <vppinfra/elog.h>
      22             : 
      23             : #include <vnet/vnet.h>
      24             : #include <vnet/ip/ip.h>
      25             : #include <vnet/udp/udp.h>
      26             : #include <vnet/udp/udp_packet.h>
      27             : #include <vnet/session/session.h>
      28             : 
      29             : static vlib_error_desc_t udp_error_counters[] = {
      30             : #define udp_error(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s },
      31             : #include "udp_error.def"
      32             : #undef udp_error
      33             : };
      34             : 
      35             : typedef struct
      36             : {
      37             :   u32 connection;
      38             :   u32 disposition;
      39             :   u32 thread_index;
      40             : } udp_input_trace_t;
      41             : 
      42             : /* packet trace format function */
      43             : static u8 *
      44           0 : format_udp_input_trace (u8 * s, va_list * args)
      45             : {
      46           0 :   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
      47           0 :   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
      48           0 :   udp_input_trace_t *t = va_arg (*args, udp_input_trace_t *);
      49             : 
      50           0 :   s = format (s, "UDP_INPUT: connection %d, disposition %d, thread %d",
      51             :               t->connection, t->disposition, t->thread_index);
      52           0 :   return s;
      53             : }
      54             : 
      55             : #define foreach_udp_input_next                  \
      56             :   _ (DROP, "error-drop")
      57             : 
      58             : typedef enum
      59             : {
      60             : #define _(s, n) UDP_INPUT_NEXT_##s,
      61             :   foreach_udp_input_next
      62             : #undef _
      63             :     UDP_INPUT_N_NEXT,
      64             : } udp_input_next_t;
      65             : 
      66             : always_inline void
      67       10786 : udp_input_inc_counter (vlib_main_t * vm, u8 is_ip4, u8 evt, u8 val)
      68             : {
      69       10786 :   if (is_ip4)
      70       10786 :     vlib_node_increment_counter (vm, udp4_input_node.index, evt, val);
      71             :   else
      72           0 :     vlib_node_increment_counter (vm, udp6_input_node.index, evt, val);
      73       10786 : }
      74             : 
      75             : #define udp_store_err_counters(vm, is_ip4, cnts)                        \
      76             : {                                                                       \
      77             :   int i;                                                                \
      78             :   for (i = 0; i < UDP_N_ERROR; i++)                                  \
      79             :     if (cnts[i])                                                        \
      80             :       udp_input_inc_counter(vm, is_ip4, i, cnts[i]);                    \
      81             : }
      82             : 
      83             : #define udp_inc_err_counter(cnts, err, val)                             \
      84             : {                                                                       \
      85             :   cnts[err] += val;                                                     \
      86             : }
      87             : 
      88             : static void
      89           0 : udp_trace_buffer (vlib_main_t * vm, vlib_node_runtime_t * node,
      90             :                   vlib_buffer_t * b, session_t * s, u16 error0)
      91             : {
      92             :   udp_input_trace_t *t;
      93             : 
      94           0 :   if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_IS_TRACED)))
      95           0 :     return;
      96             : 
      97           0 :   t = vlib_add_trace (vm, node, b, sizeof (*t));
      98           0 :   t->connection = s ? s->connection_index : ~0;
      99           0 :   t->disposition = error0;
     100           0 :   t->thread_index = s ? s->thread_index : vm->thread_index;
     101             : }
     102             : 
     103             : static udp_connection_t *
     104          20 : udp_connection_accept (udp_connection_t * listener, session_dgram_hdr_t * hdr,
     105             :                        u32 thread_index)
     106             : {
     107             :   udp_connection_t *uc;
     108             : 
     109          20 :   uc = udp_connection_alloc (thread_index);
     110          20 :   ip_copy (&uc->c_lcl_ip, &hdr->lcl_ip, hdr->is_ip4);
     111          20 :   ip_copy (&uc->c_rmt_ip, &hdr->rmt_ip, hdr->is_ip4);
     112          20 :   uc->c_lcl_port = hdr->lcl_port;
     113          20 :   uc->c_rmt_port = hdr->rmt_port;
     114          20 :   uc->c_is_ip4 = hdr->is_ip4;
     115          20 :   uc->c_fib_index = listener->c_fib_index;
     116          20 :   uc->mss = listener->mss;
     117          20 :   uc->flags |= UDP_CONN_F_CONNECTED;
     118          20 :   uc->cfg_flags = listener->cfg_flags;
     119             : 
     120          20 :   if (session_dgram_accept (&uc->connection, listener->c_s_index,
     121             :                             listener->c_thread_index))
     122             :     {
     123           0 :       udp_connection_free (uc);
     124           0 :       return 0;
     125             :     }
     126          20 :   transport_share_local_endpoint (TRANSPORT_PROTO_UDP, &uc->c_lcl_ip,
     127          20 :                                   uc->c_lcl_port);
     128          20 :   return uc;
     129             : }
     130             : 
     131             : static void
     132       59064 : udp_connection_enqueue (udp_connection_t * uc0, session_t * s0,
     133             :                         session_dgram_hdr_t * hdr0, u32 thread_index,
     134             :                         vlib_buffer_t * b, u8 queue_event, u32 * error0)
     135             : {
     136             :   int wrote0;
     137             : 
     138       59064 :   if (!(uc0->flags & UDP_CONN_F_CONNECTED))
     139           2 :     clib_spinlock_lock (&uc0->rx_lock);
     140             : 
     141       59064 :   if (svm_fifo_max_enqueue_prod (s0->rx_fifo)
     142       59064 :       < hdr0->data_length + sizeof (session_dgram_hdr_t))
     143             :     {
     144           0 :       *error0 = UDP_ERROR_FIFO_FULL;
     145           0 :       goto unlock_rx_lock;
     146             :     }
     147             : 
     148             :   /* If session is owned by another thread and rx event needed,
     149             :    * enqueue event now while we still have the peeker lock */
     150       59064 :   if (s0->thread_index != thread_index)
     151             :     {
     152           0 :       wrote0 = session_enqueue_dgram_connection_cl (
     153             :         s0, hdr0, b, TRANSPORT_PROTO_UDP,
     154           0 :         /* queue event */ queue_event && !svm_fifo_has_event (s0->rx_fifo));
     155             :     }
     156             :   else
     157             :     {
     158       59064 :       wrote0 = session_enqueue_dgram_connection (s0, hdr0, b,
     159             :                                                  TRANSPORT_PROTO_UDP,
     160             :                                                  queue_event);
     161             :     }
     162             : 
     163             :   /* In some rare cases, session_enqueue_dgram_connection can fail because a
     164             :    * chunk cannot be allocated in the RX FIFO */
     165       59064 :   if (PREDICT_FALSE (wrote0 == 0))
     166           0 :     *error0 = UDP_ERROR_FIFO_NOMEM;
     167             : 
     168       59064 : unlock_rx_lock:
     169             : 
     170       59064 :   if (!(uc0->flags & UDP_CONN_F_CONNECTED))
     171           2 :     clib_spinlock_unlock (&uc0->rx_lock);
     172       59064 : }
     173             : 
     174             : always_inline session_t *
     175       59064 : udp_parse_and_lookup_buffer (vlib_buffer_t * b, session_dgram_hdr_t * hdr,
     176             :                              u8 is_ip4)
     177             : {
     178             :   udp_header_t *udp;
     179             :   u32 fib_index;
     180             :   session_t *s;
     181             : 
     182             :   /* udp_local hands us a pointer to the udp data */
     183       59064 :   udp = (udp_header_t *) (vlib_buffer_get_current (b) - sizeof (*udp));
     184       59064 :   fib_index = vnet_buffer (b)->ip.fib_index;
     185             : 
     186       59064 :   hdr->data_offset = 0;
     187       59064 :   hdr->lcl_port = udp->dst_port;
     188       59064 :   hdr->rmt_port = udp->src_port;
     189       59064 :   hdr->is_ip4 = is_ip4;
     190       59064 :   hdr->gso_size = 0;
     191             : 
     192       59064 :   if (is_ip4)
     193             :     {
     194             :       ip4_header_t *ip4;
     195             : 
     196             :       /* TODO: must fix once udp_local does ip options correctly */
     197       59064 :       ip4 = (ip4_header_t *) (((u8 *) udp) - sizeof (*ip4));
     198       59064 :       ip_set (&hdr->lcl_ip, &ip4->dst_address, 1);
     199       59064 :       ip_set (&hdr->rmt_ip, &ip4->src_address, 1);
     200       59064 :       hdr->data_length = clib_net_to_host_u16 (ip4->length);
     201       59064 :       hdr->data_length -= sizeof (ip4_header_t) + sizeof (udp_header_t);
     202       59064 :       s = session_lookup_safe4 (fib_index, &ip4->dst_address,
     203       59064 :                                 &ip4->src_address, udp->dst_port,
     204       59064 :                                 udp->src_port, TRANSPORT_PROTO_UDP);
     205             :     }
     206             :   else
     207             :     {
     208             :       ip6_header_t *ip60;
     209             : 
     210           0 :       ip60 = (ip6_header_t *) (((u8 *) udp) - sizeof (*ip60));
     211           0 :       ip_set (&hdr->lcl_ip, &ip60->dst_address, 0);
     212           0 :       ip_set (&hdr->rmt_ip, &ip60->src_address, 0);
     213           0 :       hdr->data_length = clib_net_to_host_u16 (ip60->payload_length);
     214           0 :       hdr->data_length -= sizeof (udp_header_t);
     215           0 :       s = session_lookup_safe6 (fib_index, &ip60->dst_address,
     216           0 :                                 &ip60->src_address, udp->dst_port,
     217           0 :                                 udp->src_port, TRANSPORT_PROTO_UDP);
     218             :     }
     219             : 
     220       59064 :   if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)))
     221       59064 :     b->current_length = hdr->data_length;
     222             :   else
     223           0 :     b->total_length_not_including_first_buffer = hdr->data_length
     224           0 :       - b->current_length;
     225             : 
     226       59064 :   return s;
     227             : }
     228             : 
     229             : always_inline uword
     230       10786 : udp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     231             :                     vlib_frame_t * frame, u8 is_ip4)
     232             : {
     233       10786 :   u32 thread_index = vm->thread_index, n_left_from, *from, *first_buffer;
     234             :   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
     235       10786 :   u16 err_counters[UDP_N_ERROR] = { 0 };
     236             : 
     237       10786 :   from = first_buffer = vlib_frame_vector_args (frame);
     238       10786 :   n_left_from = frame->n_vectors;
     239       10786 :   vlib_get_buffers (vm, from, bufs, n_left_from);
     240             : 
     241       10786 :   b = bufs;
     242             : 
     243       69850 :   while (n_left_from > 0)
     244             :     {
     245       59064 :       u32 error0 = UDP_ERROR_ENQUEUED;
     246             :       session_dgram_hdr_t hdr0;
     247             :       udp_connection_t *uc0;
     248             :       session_t *s0;
     249             : 
     250       59064 :       s0 = udp_parse_and_lookup_buffer (b[0], &hdr0, is_ip4);
     251       59064 :       if (PREDICT_FALSE (!s0))
     252             :         {
     253           0 :           error0 = UDP_ERROR_NO_LISTENER;
     254           0 :           goto done;
     255             :         }
     256             : 
     257       59064 :       if (s0->session_state == SESSION_STATE_OPENED)
     258             :         {
     259          23 :           u8 queue_event = 1;
     260          23 :           uc0 = udp_connection_from_transport (session_get_transport (s0));
     261          23 :           uc0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
     262          23 :           if (uc0->flags & UDP_CONN_F_CONNECTED)
     263             :             {
     264          23 :               if (s0->thread_index != thread_index)
     265             :                 {
     266             :                   /*
     267             :                    * Clone the transport. It will be cleaned up with the
     268             :                    * session once we notify the session layer.
     269             :                    */
     270           0 :                   uc0 = udp_connection_clone_safe (s0->connection_index,
     271           0 :                                                    s0->thread_index);
     272           0 :                   ASSERT (s0->session_index == uc0->c_s_index);
     273             : 
     274             :                   /*
     275             :                    * Ask session layer for a new session.
     276             :                    */
     277           0 :                   session_dgram_connect_notify (&uc0->connection,
     278           0 :                                                 s0->thread_index, &s0);
     279           0 :                   queue_event = 0;
     280             :                 }
     281             :               else
     282          23 :                 s0->session_state = SESSION_STATE_READY;
     283             :             }
     284          23 :           udp_connection_enqueue (uc0, s0, &hdr0, thread_index, b[0],
     285             :                                   queue_event, &error0);
     286             :         }
     287       59041 :       else if (s0->session_state == SESSION_STATE_READY ||
     288          22 :                s0->session_state == SESSION_STATE_ACCEPTING)
     289             :         {
     290       59019 :           uc0 = udp_connection_from_transport (session_get_transport (s0));
     291       59019 :           udp_connection_enqueue (uc0, s0, &hdr0, thread_index, b[0], 1,
     292             :                                   &error0);
     293             :         }
     294          22 :       else if (s0->session_state == SESSION_STATE_LISTENING)
     295             :         {
     296          22 :           uc0 = udp_connection_from_transport (session_get_transport (s0));
     297          22 :           if (uc0->flags & UDP_CONN_F_CONNECTED)
     298             :             {
     299          20 :               uc0 = udp_connection_accept (uc0, &hdr0, thread_index);
     300          20 :               if (!uc0)
     301             :                 {
     302           0 :                   error0 = UDP_ERROR_CREATE_SESSION;
     303           0 :                   goto done;
     304             :                 }
     305          20 :               s0 = session_get (uc0->c_s_index, uc0->c_thread_index);
     306          20 :               uc0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
     307          20 :               error0 = UDP_ERROR_ACCEPT;
     308             :             }
     309          22 :           udp_connection_enqueue (uc0, s0, &hdr0, thread_index, b[0], 1,
     310             :                                   &error0);
     311             :         }
     312             :       else
     313             :         {
     314           0 :           error0 = UDP_ERROR_NOT_READY;
     315             :         }
     316             : 
     317       59064 :     done:
     318       59064 :       if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
     319           0 :         udp_trace_buffer (vm, node, b[0], s0, error0);
     320             : 
     321       59064 :       b += 1;
     322       59064 :       n_left_from -= 1;
     323             : 
     324       59064 :       udp_inc_err_counter (err_counters, error0, 1);
     325             :     }
     326             : 
     327       10786 :   vlib_buffer_free (vm, first_buffer, frame->n_vectors);
     328       10786 :   session_main_flush_enqueue_events (TRANSPORT_PROTO_UDP, thread_index);
     329      151004 :   udp_store_err_counters (vm, is_ip4, err_counters);
     330       10786 :   return frame->n_vectors;
     331             : }
     332             : 
     333             : static uword
     334       10786 : udp4_input (vlib_main_t * vm, vlib_node_runtime_t * node,
     335             :             vlib_frame_t * frame)
     336             : {
     337       10786 :   return udp46_input_inline (vm, node, frame, 1);
     338             : }
     339             : 
     340             : /* *INDENT-OFF* */
     341      183788 : VLIB_REGISTER_NODE (udp4_input_node) =
     342             : {
     343             :   .function = udp4_input,
     344             :   .name = "udp4-input",
     345             :   .vector_size = sizeof (u32),
     346             :   .format_trace = format_udp_input_trace,
     347             :   .type = VLIB_NODE_TYPE_INTERNAL,
     348             :   .n_errors = UDP_N_ERROR,
     349             :   .error_counters = udp_error_counters,
     350             :   .n_next_nodes = UDP_INPUT_N_NEXT,
     351             :   .next_nodes = {
     352             : #define _(s, n) [UDP_INPUT_NEXT_##s] = n,
     353             :       foreach_udp_input_next
     354             : #undef _
     355             :   },
     356             : };
     357             : /* *INDENT-ON* */
     358             : 
     359             : static uword
     360           0 : udp6_input (vlib_main_t * vm, vlib_node_runtime_t * node,
     361             :             vlib_frame_t * frame)
     362             : {
     363           0 :   return udp46_input_inline (vm, node, frame, 0);
     364             : }
     365             : 
     366             : /* *INDENT-OFF* */
     367      183788 : VLIB_REGISTER_NODE (udp6_input_node) =
     368             : {
     369             :   .function = udp6_input,
     370             :   .name = "udp6-input",
     371             :   .vector_size = sizeof (u32),
     372             :   .format_trace = format_udp_input_trace,
     373             :   .type = VLIB_NODE_TYPE_INTERNAL,
     374             :   .n_errors = UDP_N_ERROR,
     375             :   .error_counters = udp_error_counters,
     376             :   .n_next_nodes = UDP_INPUT_N_NEXT,
     377             :   .next_nodes = {
     378             : #define _(s, n) [UDP_INPUT_NEXT_##s] = n,
     379             :       foreach_udp_input_next
     380             : #undef _
     381             :   },
     382             : };
     383             : /* *INDENT-ON* */
     384             : 
     385             : /*
     386             :  * fd.io coding-style-patch-verification: ON
     387             :  *
     388             :  * Local Variables:
     389             :  * eval: (c-set-style "gnu")
     390             :  * End:
     391             :  */