Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2020 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #include <vlib/vlib.h>
      17             : #include <vnet/vnet.h>
      18             : #include <vnet/fib/ip4_fib.h>
      19             : #include <vppinfra/error.h>
      20             : 
      21             : #include <nat/nat44-ei/nat44_ei.h>
      22             : 
      23             : typedef struct
      24             : {
      25             :   u32 next_worker_index;
      26             :   u32 trace_index;
      27             :   u8 in2out;
      28             :   u8 output;
      29             : } nat44_ei_handoff_trace_t;
      30             : 
      31             : #define foreach_nat44_ei_handoff_error                                        \
      32             :   _ (CONGESTION_DROP, "congestion drop")                                      \
      33             :   _ (SAME_WORKER, "same worker")                                              \
      34             :   _ (DO_HANDOFF, "do handoff")
      35             : 
      36             : typedef enum
      37             : {
      38             : #define _(sym, str) NAT44_EI_HANDOFF_ERROR_##sym,
      39             :   foreach_nat44_ei_handoff_error
      40             : #undef _
      41             :     NAT44_EI_HANDOFF_N_ERROR,
      42             : } nat44_ei_handoff_error_t;
      43             : 
      44             : static char *nat44_ei_handoff_error_strings[] = {
      45             : #define _(sym, string) string,
      46             :   foreach_nat44_ei_handoff_error
      47             : #undef _
      48             : };
      49             : 
      50             : static u8 *
      51           4 : format_nat44_ei_handoff_trace (u8 *s, va_list *args)
      52             : {
      53           4 :   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
      54           4 :   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
      55           4 :   nat44_ei_handoff_trace_t *t = va_arg (*args, nat44_ei_handoff_trace_t *);
      56             :   char *tag, *output;
      57             : 
      58           4 :   tag = t->in2out ? "IN2OUT" : "OUT2IN";
      59           4 :   output = t->output ? "OUTPUT-FEATURE" : "";
      60             :   s =
      61           4 :     format (s, "NAT44_EI_%s_WORKER_HANDOFF %s: next-worker %d trace index %d",
      62             :             tag, output, t->next_worker_index, t->trace_index);
      63             : 
      64           4 :   return s;
      65             : }
      66             : 
      67             : static inline uword
      68           6 : nat44_ei_worker_handoff_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
      69             :                                    vlib_frame_t *frame, u8 is_output,
      70             :                                    u8 is_in2out)
      71             : {
      72           6 :   u32 n_enq, n_left_from, *from, do_handoff = 0, same_worker = 0;
      73             : 
      74           6 :   u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
      75           6 :   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
      76           6 :   nat44_ei_main_t *nm = &nat44_ei_main;
      77             : 
      78           6 :   u32 fq_index, thread_index = vm->thread_index;
      79             : 
      80           6 :   from = vlib_frame_vector_args (frame);
      81           6 :   n_left_from = frame->n_vectors;
      82             : 
      83           6 :   vlib_get_buffers (vm, from, b, n_left_from);
      84             : 
      85           6 :   if (is_in2out)
      86             :     {
      87           6 :       fq_index = is_output ? nm->fq_in2out_output_index : nm->fq_in2out_index;
      88             :     }
      89             :   else
      90             :     {
      91           0 :       fq_index = nm->fq_out2in_index;
      92             :     }
      93             : 
      94           6 :   while (n_left_from >= 4)
      95             :     {
      96             :       u32 arc_next0, arc_next1, arc_next2, arc_next3;
      97             :       u32 sw_if_index0, sw_if_index1, sw_if_index2, sw_if_index3;
      98             :       u32 rx_fib_index0, rx_fib_index1, rx_fib_index2, rx_fib_index3;
      99           0 :       u32 iph_offset0 = 0, iph_offset1 = 0, iph_offset2 = 0, iph_offset3 = 0;
     100             :       ip4_header_t *ip0, *ip1, *ip2, *ip3;
     101             : 
     102           0 :       if (PREDICT_TRUE (n_left_from >= 8))
     103             :         {
     104           0 :           vlib_prefetch_buffer_header (b[4], LOAD);
     105           0 :           vlib_prefetch_buffer_header (b[5], LOAD);
     106           0 :           vlib_prefetch_buffer_header (b[6], LOAD);
     107           0 :           vlib_prefetch_buffer_header (b[7], LOAD);
     108           0 :           clib_prefetch_load (&b[4]->data);
     109           0 :           clib_prefetch_load (&b[5]->data);
     110           0 :           clib_prefetch_load (&b[6]->data);
     111           0 :           clib_prefetch_load (&b[7]->data);
     112             :         }
     113             : 
     114           0 :       if (is_output)
     115             :         {
     116           0 :           iph_offset0 = vnet_buffer (b[0])->ip.save_rewrite_length;
     117           0 :           iph_offset1 = vnet_buffer (b[1])->ip.save_rewrite_length;
     118           0 :           iph_offset2 = vnet_buffer (b[2])->ip.save_rewrite_length;
     119           0 :           iph_offset3 = vnet_buffer (b[3])->ip.save_rewrite_length;
     120             :         }
     121             : 
     122           0 :       ip0 =
     123           0 :         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[0]) + iph_offset0);
     124           0 :       ip1 =
     125           0 :         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[1]) + iph_offset1);
     126           0 :       ip2 =
     127           0 :         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[2]) + iph_offset2);
     128           0 :       ip3 =
     129           0 :         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[3]) + iph_offset3);
     130             : 
     131           0 :       vnet_feature_next (&arc_next0, b[0]);
     132           0 :       vnet_feature_next (&arc_next1, b[1]);
     133           0 :       vnet_feature_next (&arc_next2, b[2]);
     134           0 :       vnet_feature_next (&arc_next3, b[3]);
     135             : 
     136           0 :       vnet_buffer2 (b[0])->nat.arc_next = arc_next0;
     137           0 :       vnet_buffer2 (b[1])->nat.arc_next = arc_next1;
     138           0 :       vnet_buffer2 (b[2])->nat.arc_next = arc_next2;
     139           0 :       vnet_buffer2 (b[3])->nat.arc_next = arc_next3;
     140             : 
     141           0 :       sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
     142           0 :       sw_if_index1 = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
     143           0 :       sw_if_index2 = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
     144           0 :       sw_if_index3 = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
     145             : 
     146           0 :       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
     147           0 :       rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index1);
     148           0 :       rx_fib_index2 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index2);
     149           0 :       rx_fib_index3 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index3);
     150             : 
     151           0 :       if (is_in2out)
     152             :         {
     153           0 :           ti[0] =
     154           0 :             nat44_ei_get_in2out_worker_index (ip0, rx_fib_index0, is_output);
     155           0 :           ti[1] =
     156           0 :             nat44_ei_get_in2out_worker_index (ip1, rx_fib_index1, is_output);
     157           0 :           ti[2] =
     158           0 :             nat44_ei_get_in2out_worker_index (ip2, rx_fib_index2, is_output);
     159           0 :           ti[3] =
     160           0 :             nat44_ei_get_in2out_worker_index (ip3, rx_fib_index3, is_output);
     161             :         }
     162             :       else
     163             :         {
     164           0 :           ti[0] = nat44_ei_get_out2in_worker_index (b[0], ip0, rx_fib_index0,
     165             :                                                     is_output);
     166           0 :           ti[1] = nat44_ei_get_out2in_worker_index (b[1], ip1, rx_fib_index1,
     167             :                                                     is_output);
     168           0 :           ti[2] = nat44_ei_get_out2in_worker_index (b[2], ip2, rx_fib_index2,
     169             :                                                     is_output);
     170           0 :           ti[3] = nat44_ei_get_out2in_worker_index (b[3], ip3, rx_fib_index3,
     171             :                                                     is_output);
     172             :         }
     173             : 
     174           0 :       if (ti[0] == thread_index)
     175           0 :         same_worker++;
     176             :       else
     177           0 :         do_handoff++;
     178             : 
     179           0 :       if (ti[1] == thread_index)
     180           0 :         same_worker++;
     181             :       else
     182           0 :         do_handoff++;
     183             : 
     184           0 :       if (ti[2] == thread_index)
     185           0 :         same_worker++;
     186             :       else
     187           0 :         do_handoff++;
     188             : 
     189           0 :       if (ti[3] == thread_index)
     190           0 :         same_worker++;
     191             :       else
     192           0 :         do_handoff++;
     193             : 
     194           0 :       b += 4;
     195           0 :       ti += 4;
     196           0 :       n_left_from -= 4;
     197             :     }
     198             : 
     199          20 :   while (n_left_from > 0)
     200             :     {
     201             :       u32 arc_next0;
     202             :       u32 sw_if_index0;
     203             :       u32 rx_fib_index0;
     204          14 :       u32 iph_offset0 = 0;
     205             :       ip4_header_t *ip0;
     206             : 
     207          14 :       if (is_output)
     208           0 :         iph_offset0 = vnet_buffer (b[0])->ip.save_rewrite_length;
     209             : 
     210          14 :       ip0 =
     211          14 :         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b[0]) + iph_offset0);
     212             : 
     213          14 :       vnet_feature_next (&arc_next0, b[0]);
     214          14 :       vnet_buffer2 (b[0])->nat.arc_next = arc_next0;
     215             : 
     216          14 :       sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
     217          14 :       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
     218             : 
     219          14 :       if (is_in2out)
     220             :         {
     221          14 :           ti[0] =
     222          14 :             nat44_ei_get_in2out_worker_index (ip0, rx_fib_index0, is_output);
     223             :         }
     224             :       else
     225             :         {
     226           0 :           ti[0] = nat44_ei_get_out2in_worker_index (b[0], ip0, rx_fib_index0,
     227             :                                                     is_output);
     228             :         }
     229             : 
     230          14 :       if (ti[0] == thread_index)
     231           7 :         same_worker++;
     232             :       else
     233           7 :         do_handoff++;
     234             : 
     235          14 :       b += 1;
     236          14 :       ti += 1;
     237          14 :       n_left_from -= 1;
     238             :     }
     239             : 
     240           6 :   if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
     241             :     {
     242             :       u32 i;
     243           6 :       b = bufs;
     244           6 :       ti = thread_indices;
     245             : 
     246          20 :       for (i = 0; i < frame->n_vectors; i++)
     247             :         {
     248          14 :           if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
     249             :             {
     250             :               nat44_ei_handoff_trace_t *t =
     251          14 :                 vlib_add_trace (vm, node, b[0], sizeof (*t));
     252          14 :               t->next_worker_index = ti[0];
     253          14 :               t->trace_index = vlib_buffer_get_trace_index (b[0]);
     254          14 :               t->in2out = is_in2out;
     255          14 :               t->output = is_output;
     256             : 
     257          14 :               b += 1;
     258          14 :               ti += 1;
     259             :             }
     260             :           else
     261           0 :             break;
     262             :         }
     263             :     }
     264             : 
     265           6 :   n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
     266           6 :                                          thread_indices, frame->n_vectors, 1);
     267             : 
     268           6 :   if (n_enq < frame->n_vectors)
     269             :     {
     270           0 :       vlib_node_increment_counter (vm, node->node_index,
     271             :                                    NAT44_EI_HANDOFF_ERROR_CONGESTION_DROP,
     272           0 :                                    frame->n_vectors - n_enq);
     273             :     }
     274             : 
     275           6 :   vlib_node_increment_counter (
     276             :     vm, node->node_index, NAT44_EI_HANDOFF_ERROR_SAME_WORKER, same_worker);
     277           6 :   vlib_node_increment_counter (vm, node->node_index,
     278             :                                NAT44_EI_HANDOFF_ERROR_DO_HANDOFF, do_handoff);
     279           6 :   return frame->n_vectors;
     280             : }
     281             : 
     282         581 : VLIB_NODE_FN (nat44_ei_in2out_worker_handoff_node)
     283             : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
     284             : {
     285           6 :   return nat44_ei_worker_handoff_fn_inline (vm, node, frame, 0, 1);
     286             : }
     287             : 
     288         575 : VLIB_NODE_FN (nat44_ei_in2out_output_worker_handoff_node)
     289             : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
     290             : {
     291           0 :   return nat44_ei_worker_handoff_fn_inline (vm, node, frame, 1, 1);
     292             : }
     293             : 
     294         575 : VLIB_NODE_FN (nat44_ei_out2in_worker_handoff_node)
     295             : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
     296             : {
     297           0 :   return nat44_ei_worker_handoff_fn_inline (vm, node, frame, 0, 0);
     298             : }
     299             : 
     300       72026 : VLIB_REGISTER_NODE (nat44_ei_in2out_output_worker_handoff_node) = {
     301             :   .name = "nat44-ei-in2out-output-worker-handoff",
     302             :   .vector_size = sizeof (u32),
     303             :   .format_trace = format_nat44_ei_handoff_trace,
     304             :   .type = VLIB_NODE_TYPE_INTERNAL,
     305             :   .n_errors = ARRAY_LEN (nat44_ei_handoff_error_strings),
     306             :   .error_strings = nat44_ei_handoff_error_strings,
     307             : };
     308             : 
     309       72026 : VLIB_REGISTER_NODE (nat44_ei_in2out_worker_handoff_node) = {
     310             :   .name = "nat44-ei-in2out-worker-handoff",
     311             :   .vector_size = sizeof (u32),
     312             :   .format_trace = format_nat44_ei_handoff_trace,
     313             :   .type = VLIB_NODE_TYPE_INTERNAL,
     314             :   .n_errors = ARRAY_LEN (nat44_ei_handoff_error_strings),
     315             :   .error_strings = nat44_ei_handoff_error_strings,
     316             : };
     317             : 
     318       72026 : VLIB_REGISTER_NODE (nat44_ei_out2in_worker_handoff_node) = {
     319             :   .name = "nat44-ei-out2in-worker-handoff",
     320             :   .vector_size = sizeof (u32),
     321             :   .format_trace = format_nat44_ei_handoff_trace,
     322             :   .type = VLIB_NODE_TYPE_INTERNAL,
     323             :   .n_errors = ARRAY_LEN (nat44_ei_handoff_error_strings),
     324             :   .error_strings = nat44_ei_handoff_error_strings,
     325             : };
     326             : 
     327             : /*
     328             :  * fd.io coding-style-patch-verification: ON
     329             :  *
     330             :  * Local Variables:
     331             :  * eval: (c-set-style "gnu")
     332             :  * End:
     333             :  */