Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #include <vlibmemory/api.h>
      17             : #include <vlib/vlib.h>
      18             : 
      19             : #include <vppinfra/hash.h>
      20             : #include <vppinfra/error.h>
      21             : #include <vppinfra/elog.h>
      22             : 
      23             : #include <vnet/vnet.h>
      24             : #include <vnet/ip/ip.h>
      25             : #include <vnet/udp/udp.h>
      26             : #include <vnet/udp/udp_packet.h>
      27             : #include <vnet/session/session.h>
      28             : 
      29             : static vlib_error_desc_t udp_error_counters[] = {
      30             : #define udp_error(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s },
      31             : #include "udp_error.def"
      32             : #undef udp_error
      33             : };
      34             : 
      35             : typedef struct
      36             : {
      37             :   u32 connection;
      38             :   u32 disposition;
      39             :   u32 thread_index;
      40             : } udp_input_trace_t;
      41             : 
      42             : /* packet trace format function */
      43             : static u8 *
      44           0 : format_udp_input_trace (u8 * s, va_list * args)
      45             : {
      46           0 :   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
      47           0 :   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
      48           0 :   udp_input_trace_t *t = va_arg (*args, udp_input_trace_t *);
      49             : 
      50           0 :   s = format (s, "UDP_INPUT: connection %d, disposition %d, thread %d",
      51             :               t->connection, t->disposition, t->thread_index);
      52           0 :   return s;
      53             : }
      54             : 
      55             : #define foreach_udp_input_next                  \
      56             :   _ (DROP, "error-drop")
      57             : 
      58             : typedef enum
      59             : {
      60             : #define _(s, n) UDP_INPUT_NEXT_##s,
      61             :   foreach_udp_input_next
      62             : #undef _
      63             :     UDP_INPUT_N_NEXT,
      64             : } udp_input_next_t;
      65             : 
      66             : always_inline void
      67       10673 : udp_input_inc_counter (vlib_main_t * vm, u8 is_ip4, u8 evt, u8 val)
      68             : {
      69       10673 :   if (is_ip4)
      70       10673 :     vlib_node_increment_counter (vm, udp4_input_node.index, evt, val);
      71             :   else
      72           0 :     vlib_node_increment_counter (vm, udp6_input_node.index, evt, val);
      73       10673 : }
      74             : 
      75             : #define udp_store_err_counters(vm, is_ip4, cnts)                        \
      76             : {                                                                       \
      77             :   int i;                                                                \
      78             :   for (i = 0; i < UDP_N_ERROR; i++)                                  \
      79             :     if (cnts[i])                                                        \
      80             :       udp_input_inc_counter(vm, is_ip4, i, cnts[i]);                    \
      81             : }
      82             : 
      83             : #define udp_inc_err_counter(cnts, err, val)                             \
      84             : {                                                                       \
      85             :   cnts[err] += val;                                                     \
      86             : }
      87             : 
      88             : static void
      89           0 : udp_trace_buffer (vlib_main_t * vm, vlib_node_runtime_t * node,
      90             :                   vlib_buffer_t * b, session_t * s, u16 error0)
      91             : {
      92             :   udp_input_trace_t *t;
      93             : 
      94           0 :   if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_IS_TRACED)))
      95           0 :     return;
      96             : 
      97           0 :   t = vlib_add_trace (vm, node, b, sizeof (*t));
      98           0 :   t->connection = s ? s->connection_index : ~0;
      99           0 :   t->disposition = error0;
     100           0 :   t->thread_index = s ? s->thread_index : vm->thread_index;
     101             : }
     102             : 
     103             : static udp_connection_t *
     104          20 : udp_connection_accept (udp_connection_t * listener, session_dgram_hdr_t * hdr,
     105             :                        u32 thread_index)
     106             : {
     107             :   udp_connection_t *uc;
     108             : 
     109          20 :   uc = udp_connection_alloc (thread_index);
     110          20 :   ip_copy (&uc->c_lcl_ip, &hdr->lcl_ip, hdr->is_ip4);
     111          20 :   ip_copy (&uc->c_rmt_ip, &hdr->rmt_ip, hdr->is_ip4);
     112          20 :   uc->c_lcl_port = hdr->lcl_port;
     113          20 :   uc->c_rmt_port = hdr->rmt_port;
     114          20 :   uc->c_is_ip4 = hdr->is_ip4;
     115          20 :   uc->c_fib_index = listener->c_fib_index;
     116          20 :   uc->mss = listener->mss;
     117          20 :   uc->flags |= UDP_CONN_F_CONNECTED;
     118          20 :   uc->cfg_flags = listener->cfg_flags;
     119             : 
     120          20 :   if (session_dgram_accept (&uc->connection, listener->c_s_index,
     121             :                             listener->c_thread_index))
     122             :     {
     123           0 :       udp_connection_free (uc);
     124           0 :       return 0;
     125             :     }
     126          20 :   transport_share_local_endpoint (TRANSPORT_PROTO_UDP, &uc->c_lcl_ip,
     127          20 :                                   uc->c_lcl_port);
     128          20 :   return uc;
     129             : }
     130             : 
     131             : static void
     132       58171 : udp_connection_enqueue (udp_connection_t * uc0, session_t * s0,
     133             :                         session_dgram_hdr_t * hdr0, u32 thread_index,
     134             :                         vlib_buffer_t * b, u8 queue_event, u32 * error0)
     135             : {
     136             :   int wrote0;
     137             : 
     138       58171 :   if (!(uc0->flags & UDP_CONN_F_CONNECTED))
     139           2 :     clib_spinlock_lock (&uc0->rx_lock);
     140             : 
     141       58171 :   if (svm_fifo_max_enqueue_prod (s0->rx_fifo)
     142       58171 :       < hdr0->data_length + sizeof (session_dgram_hdr_t))
     143             :     {
     144           0 :       *error0 = UDP_ERROR_FIFO_FULL;
     145           0 :       goto unlock_rx_lock;
     146             :     }
     147             : 
     148             :   /* If session is owned by another thread and rx event needed,
     149             :    * enqueue event now while we still have the peeker lock */
     150       58171 :   if (s0->thread_index != thread_index)
     151             :     {
     152           0 :       wrote0 = session_enqueue_dgram_connection (s0, hdr0, b,
     153             :                                                  TRANSPORT_PROTO_UDP,
     154             :                                                  /* queue event */ 0);
     155           0 :       if (queue_event && !svm_fifo_has_event (s0->rx_fifo))
     156           0 :         session_enqueue_notify (s0);
     157             :     }
     158             :   else
     159             :     {
     160       58171 :       wrote0 = session_enqueue_dgram_connection (s0, hdr0, b,
     161             :                                                  TRANSPORT_PROTO_UDP,
     162             :                                                  queue_event);
     163             :     }
     164             : 
     165             :   /* In some rare cases, session_enqueue_dgram_connection can fail because a
     166             :    * chunk cannot be allocated in the RX FIFO */
     167       58171 :   if (PREDICT_FALSE (wrote0 == 0))
     168           0 :     *error0 = UDP_ERROR_FIFO_NOMEM;
     169             : 
     170       58171 : unlock_rx_lock:
     171             : 
     172       58171 :   if (!(uc0->flags & UDP_CONN_F_CONNECTED))
     173           2 :     clib_spinlock_unlock (&uc0->rx_lock);
     174       58171 : }
     175             : 
     176             : always_inline session_t *
     177       58171 : udp_parse_and_lookup_buffer (vlib_buffer_t * b, session_dgram_hdr_t * hdr,
     178             :                              u8 is_ip4)
     179             : {
     180             :   udp_header_t *udp;
     181             :   u32 fib_index;
     182             :   session_t *s;
     183             : 
     184             :   /* udp_local hands us a pointer to the udp data */
     185       58171 :   udp = (udp_header_t *) (vlib_buffer_get_current (b) - sizeof (*udp));
     186       58171 :   fib_index = vnet_buffer (b)->ip.fib_index;
     187             : 
     188       58171 :   hdr->data_offset = 0;
     189       58171 :   hdr->lcl_port = udp->dst_port;
     190       58171 :   hdr->rmt_port = udp->src_port;
     191       58171 :   hdr->is_ip4 = is_ip4;
     192       58171 :   hdr->gso_size = 0;
     193             : 
     194       58171 :   if (is_ip4)
     195             :     {
     196             :       ip4_header_t *ip4;
     197             : 
     198             :       /* TODO: must fix once udp_local does ip options correctly */
     199       58171 :       ip4 = (ip4_header_t *) (((u8 *) udp) - sizeof (*ip4));
     200       58171 :       ip_set (&hdr->lcl_ip, &ip4->dst_address, 1);
     201       58171 :       ip_set (&hdr->rmt_ip, &ip4->src_address, 1);
     202       58171 :       hdr->data_length = clib_net_to_host_u16 (ip4->length);
     203       58171 :       hdr->data_length -= sizeof (ip4_header_t) + sizeof (udp_header_t);
     204       58171 :       s = session_lookup_safe4 (fib_index, &ip4->dst_address,
     205       58171 :                                 &ip4->src_address, udp->dst_port,
     206       58171 :                                 udp->src_port, TRANSPORT_PROTO_UDP);
     207             :     }
     208             :   else
     209             :     {
     210             :       ip6_header_t *ip60;
     211             : 
     212           0 :       ip60 = (ip6_header_t *) (((u8 *) udp) - sizeof (*ip60));
     213           0 :       ip_set (&hdr->lcl_ip, &ip60->dst_address, 0);
     214           0 :       ip_set (&hdr->rmt_ip, &ip60->src_address, 0);
     215           0 :       hdr->data_length = clib_net_to_host_u16 (ip60->payload_length);
     216           0 :       hdr->data_length -= sizeof (udp_header_t);
     217           0 :       s = session_lookup_safe6 (fib_index, &ip60->dst_address,
     218           0 :                                 &ip60->src_address, udp->dst_port,
     219           0 :                                 udp->src_port, TRANSPORT_PROTO_UDP);
     220             :     }
     221             : 
     222       58171 :   if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)))
     223       58171 :     b->current_length = hdr->data_length;
     224             :   else
     225           0 :     b->total_length_not_including_first_buffer = hdr->data_length
     226           0 :       - b->current_length;
     227             : 
     228       58171 :   return s;
     229             : }
     230             : 
     231             : always_inline uword
     232       10673 : udp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     233             :                     vlib_frame_t * frame, u8 is_ip4)
     234             : {
     235             :   u32 n_left_from, *from, errors, *first_buffer;
     236             :   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
     237       10673 :   u16 err_counters[UDP_N_ERROR] = { 0 };
     238       10673 :   u32 thread_index = vm->thread_index;
     239             : 
     240       10673 :   from = first_buffer = vlib_frame_vector_args (frame);
     241       10673 :   n_left_from = frame->n_vectors;
     242       10673 :   vlib_get_buffers (vm, from, bufs, n_left_from);
     243             : 
     244       10673 :   b = bufs;
     245             : 
     246       68844 :   while (n_left_from > 0)
     247             :     {
     248       58171 :       u32 error0 = UDP_ERROR_ENQUEUED;
     249             :       session_dgram_hdr_t hdr0;
     250             :       udp_connection_t *uc0;
     251             :       session_t *s0;
     252             : 
     253       58171 :       s0 = udp_parse_and_lookup_buffer (b[0], &hdr0, is_ip4);
     254       58171 :       if (PREDICT_FALSE (!s0))
     255             :         {
     256           0 :           error0 = UDP_ERROR_NO_LISTENER;
     257           0 :           goto done;
     258             :         }
     259             : 
     260       58171 :       if (s0->session_state == SESSION_STATE_OPENED)
     261             :         {
     262          23 :           u8 queue_event = 1;
     263          23 :           uc0 = udp_connection_from_transport (session_get_transport (s0));
     264          23 :           uc0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
     265          23 :           if (uc0->flags & UDP_CONN_F_CONNECTED)
     266             :             {
     267          23 :               if (s0->thread_index != thread_index)
     268             :                 {
     269             :                   /*
     270             :                    * Clone the transport. It will be cleaned up with the
     271             :                    * session once we notify the session layer.
     272             :                    */
     273           0 :                   uc0 = udp_connection_clone_safe (s0->connection_index,
     274           0 :                                                    s0->thread_index);
     275           0 :                   ASSERT (s0->session_index == uc0->c_s_index);
     276             : 
     277             :                   /*
     278             :                    * Ask session layer for a new session.
     279             :                    */
     280           0 :                   session_dgram_connect_notify (&uc0->connection,
     281           0 :                                                 s0->thread_index, &s0);
     282           0 :                   queue_event = 0;
     283             :                 }
     284             :               else
     285          23 :                 s0->session_state = SESSION_STATE_READY;
     286             :             }
     287          23 :           udp_connection_enqueue (uc0, s0, &hdr0, thread_index, b[0],
     288             :                                   queue_event, &error0);
     289             :         }
     290       58148 :       else if (s0->session_state == SESSION_STATE_READY)
     291             :         {
     292       58126 :           uc0 = udp_connection_from_transport (session_get_transport (s0));
     293       58126 :           udp_connection_enqueue (uc0, s0, &hdr0, thread_index, b[0], 1,
     294             :                                   &error0);
     295             :         }
     296          22 :       else if (s0->session_state == SESSION_STATE_LISTENING)
     297             :         {
     298          22 :           uc0 = udp_connection_from_transport (session_get_transport (s0));
     299          22 :           if (uc0->flags & UDP_CONN_F_CONNECTED)
     300             :             {
     301          20 :               uc0 = udp_connection_accept (uc0, &hdr0, thread_index);
     302          20 :               if (!uc0)
     303             :                 {
     304           0 :                   error0 = UDP_ERROR_CREATE_SESSION;
     305           0 :                   goto done;
     306             :                 }
     307          20 :               s0 = session_get (uc0->c_s_index, uc0->c_thread_index);
     308          20 :               uc0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
     309          20 :               error0 = UDP_ERROR_ACCEPT;
     310             :             }
     311          22 :           udp_connection_enqueue (uc0, s0, &hdr0, thread_index, b[0], 1,
     312             :                                   &error0);
     313             :         }
     314             :       else
     315             :         {
     316           0 :           error0 = UDP_ERROR_NOT_READY;
     317             :         }
     318             : 
     319       58171 :     done:
     320       58171 :       if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
     321           0 :         udp_trace_buffer (vm, node, b[0], s0, error0);
     322             : 
     323       58171 :       b += 1;
     324       58171 :       n_left_from -= 1;
     325             : 
     326       58171 :       udp_inc_err_counter (err_counters, error0, 1);
     327             :     }
     328             : 
     329       10673 :   vlib_buffer_free (vm, first_buffer, frame->n_vectors);
     330       10673 :   errors = session_main_flush_enqueue_events (TRANSPORT_PROTO_UDP,
     331             :                                               thread_index);
     332       10673 :   err_counters[UDP_ERROR_MQ_FULL] = errors;
     333      149422 :   udp_store_err_counters (vm, is_ip4, err_counters);
     334       10673 :   return frame->n_vectors;
     335             : }
     336             : 
     337             : static uword
     338       10673 : udp4_input (vlib_main_t * vm, vlib_node_runtime_t * node,
     339             :             vlib_frame_t * frame)
     340             : {
     341       10673 :   return udp46_input_inline (vm, node, frame, 1);
     342             : }
     343             : 
     344             : /* *INDENT-OFF* */
     345      178120 : VLIB_REGISTER_NODE (udp4_input_node) =
     346             : {
     347             :   .function = udp4_input,
     348             :   .name = "udp4-input",
     349             :   .vector_size = sizeof (u32),
     350             :   .format_trace = format_udp_input_trace,
     351             :   .type = VLIB_NODE_TYPE_INTERNAL,
     352             :   .n_errors = UDP_N_ERROR,
     353             :   .error_counters = udp_error_counters,
     354             :   .n_next_nodes = UDP_INPUT_N_NEXT,
     355             :   .next_nodes = {
     356             : #define _(s, n) [UDP_INPUT_NEXT_##s] = n,
     357             :       foreach_udp_input_next
     358             : #undef _
     359             :   },
     360             : };
     361             : /* *INDENT-ON* */
     362             : 
     363             : static uword
     364           0 : udp6_input (vlib_main_t * vm, vlib_node_runtime_t * node,
     365             :             vlib_frame_t * frame)
     366             : {
     367           0 :   return udp46_input_inline (vm, node, frame, 0);
     368             : }
     369             : 
     370             : /* *INDENT-OFF* */
     371      178120 : VLIB_REGISTER_NODE (udp6_input_node) =
     372             : {
     373             :   .function = udp6_input,
     374             :   .name = "udp6-input",
     375             :   .vector_size = sizeof (u32),
     376             :   .format_trace = format_udp_input_trace,
     377             :   .type = VLIB_NODE_TYPE_INTERNAL,
     378             :   .n_errors = UDP_N_ERROR,
     379             :   .error_counters = udp_error_counters,
     380             :   .n_next_nodes = UDP_INPUT_N_NEXT,
     381             :   .next_nodes = {
     382             : #define _(s, n) [UDP_INPUT_NEXT_##s] = n,
     383             :       foreach_udp_input_next
     384             : #undef _
     385             :   },
     386             : };
     387             : /* *INDENT-ON* */
     388             : 
     389             : /*
     390             :  * fd.io coding-style-patch-verification: ON
     391             :  *
     392             :  * Local Variables:
     393             :  * eval: (c-set-style "gnu")
     394             :  * End:
     395             :  */