Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2015-2019 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : /*
      16             :  * ip/ip4_forward.h: IP v4 forwarding
      17             :  *
      18             :  * Copyright (c) 2008 Eliot Dresselhaus
      19             :  *
      20             :  * Permission is hereby granted, free of charge, to any person obtaining
      21             :  * a copy of this software and associated documentation files (the
      22             :  * "Software"), to deal in the Software without restriction, including
      23             :  * without limitation the rights to use, copy, modify, merge, publish,
      24             :  * distribute, sublicense, and/or sell copies of the Software, and to
      25             :  * permit persons to whom the Software is furnished to do so, subject to
      26             :  * the following conditions:
      27             :  *
      28             :  * The above copyright notice and this permission notice shall be
      29             :  * included in all copies or substantial portions of the Software.
      30             :  *
      31             :  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
      32             :  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
      33             :  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
      34             :  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
      35             :  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
      36             :  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
      37             :  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
      38             :  */
      39             : 
      40             : #ifndef __included_ip4_forward_h__
      41             : #define __included_ip4_forward_h__
      42             : 
      43             : #include <vppinfra/cache.h>
      44             : #include <vnet/fib/ip4_fib.h>
      45             : #include <vnet/dpo/load_balance_map.h>
      46             : #include <vnet/ip/ip4_inlines.h>
      47             : 
      48             : /**
      49             :  * @file
      50             :  * @brief IPv4 Forwarding.
      51             :  *
      52             :  * This file contains the source code for IPv4 forwarding.
      53             :  */
      54             : 
      55             : always_inline uword
      56      340773 : ip4_lookup_inline (vlib_main_t * vm,
      57             :                    vlib_node_runtime_t * node, vlib_frame_t * frame)
      58             : {
      59      340773 :   ip4_main_t *im = &ip4_main;
      60      340773 :   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
      61             :   u32 n_left, *from;
      62      340773 :   u32 thread_index = vm->thread_index;
      63             :   vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
      64      340773 :   vlib_buffer_t **b = bufs;
      65             :   u16 nexts[VLIB_FRAME_SIZE], *next;
      66             : 
      67      340773 :   from = vlib_frame_vector_args (frame);
      68      340773 :   n_left = frame->n_vectors;
      69      340773 :   next = nexts;
      70      340773 :   vlib_get_buffers (vm, from, bufs, n_left);
      71             : 
      72             : #if (CLIB_N_PREFETCHES >= 8)
      73     3852540 :   while (n_left >= 4)
      74             :     {
      75             :       ip4_header_t *ip0, *ip1, *ip2, *ip3;
      76             :       const load_balance_t *lb0, *lb1, *lb2, *lb3;
      77             :       ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
      78             :       u32 lb_index0, lb_index1, lb_index2, lb_index3;
      79             :       flow_hash_config_t flow_hash_config0, flow_hash_config1;
      80             :       flow_hash_config_t flow_hash_config2, flow_hash_config3;
      81             :       u32 hash_c0, hash_c1, hash_c2, hash_c3;
      82             :       const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
      83             : 
      84             :       /* Prefetch next iteration. */
      85     3511770 :       if (n_left >= 8)
      86             :         {
      87     3277190 :           vlib_prefetch_buffer_header (b[4], LOAD);
      88     3277190 :           vlib_prefetch_buffer_header (b[5], LOAD);
      89     3277190 :           vlib_prefetch_buffer_header (b[6], LOAD);
      90     3277190 :           vlib_prefetch_buffer_header (b[7], LOAD);
      91             : 
      92     3277190 :           CLIB_PREFETCH (b[4]->data, sizeof (ip0[0]), LOAD);
      93     3277190 :           CLIB_PREFETCH (b[5]->data, sizeof (ip0[0]), LOAD);
      94     3277190 :           CLIB_PREFETCH (b[6]->data, sizeof (ip0[0]), LOAD);
      95     3277190 :           CLIB_PREFETCH (b[7]->data, sizeof (ip0[0]), LOAD);
      96             :         }
      97             : 
      98     3511770 :       ip0 = vlib_buffer_get_current (b[0]);
      99     3511770 :       ip1 = vlib_buffer_get_current (b[1]);
     100     3511770 :       ip2 = vlib_buffer_get_current (b[2]);
     101     3511770 :       ip3 = vlib_buffer_get_current (b[3]);
     102             : 
     103     3511770 :       dst_addr0 = &ip0->dst_address;
     104     3511770 :       dst_addr1 = &ip1->dst_address;
     105     3511770 :       dst_addr2 = &ip2->dst_address;
     106     3511770 :       dst_addr3 = &ip3->dst_address;
     107             : 
     108     3511770 :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[0]);
     109     3511770 :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[1]);
     110     3511770 :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[2]);
     111     3511770 :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[3]);
     112             : 
     113     3511770 :       ip4_fib_forwarding_lookup_x4 (
     114     3511770 :         vnet_buffer (b[0])->ip.fib_index, vnet_buffer (b[1])->ip.fib_index,
     115     3511770 :         vnet_buffer (b[2])->ip.fib_index, vnet_buffer (b[3])->ip.fib_index,
     116             :         dst_addr0, dst_addr1, dst_addr2, dst_addr3, &lb_index0, &lb_index1,
     117             :         &lb_index2, &lb_index3);
     118             : 
     119     3511770 :       ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
     120     3511770 :       lb0 = load_balance_get (lb_index0);
     121     3511770 :       lb1 = load_balance_get (lb_index1);
     122     3511770 :       lb2 = load_balance_get (lb_index2);
     123     3511770 :       lb3 = load_balance_get (lb_index3);
     124             : 
     125     3511770 :       ASSERT (lb0->lb_n_buckets > 0);
     126     3511770 :       ASSERT (is_pow2 (lb0->lb_n_buckets));
     127     3511770 :       ASSERT (lb1->lb_n_buckets > 0);
     128     3511770 :       ASSERT (is_pow2 (lb1->lb_n_buckets));
     129     3511770 :       ASSERT (lb2->lb_n_buckets > 0);
     130     3511770 :       ASSERT (is_pow2 (lb2->lb_n_buckets));
     131     3511770 :       ASSERT (lb3->lb_n_buckets > 0);
     132     3511770 :       ASSERT (is_pow2 (lb3->lb_n_buckets));
     133             : 
     134             :       /* Use flow hash to compute multipath adjacency. */
     135     3511770 :       hash_c0 = vnet_buffer (b[0])->ip.flow_hash = 0;
     136     3511770 :       hash_c1 = vnet_buffer (b[1])->ip.flow_hash = 0;
     137     3511770 :       hash_c2 = vnet_buffer (b[2])->ip.flow_hash = 0;
     138     3511770 :       hash_c3 = vnet_buffer (b[3])->ip.flow_hash = 0;
     139     3511770 :       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
     140             :         {
     141         619 :           flow_hash_config0 = lb0->lb_hash_config;
     142        1238 :           hash_c0 = vnet_buffer (b[0])->ip.flow_hash =
     143         619 :             ip4_compute_flow_hash (ip0, flow_hash_config0);
     144             :           dpo0 =
     145         619 :             load_balance_get_fwd_bucket (lb0,
     146         619 :                                          (hash_c0 &
     147         619 :                                           (lb0->lb_n_buckets_minus_1)));
     148             :         }
     149             :       else
     150             :         {
     151     3511150 :           dpo0 = load_balance_get_bucket_i (lb0, 0);
     152             :         }
     153     3511770 :       if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
     154             :         {
     155         619 :           flow_hash_config1 = lb1->lb_hash_config;
     156        1238 :           hash_c1 = vnet_buffer (b[1])->ip.flow_hash =
     157         619 :             ip4_compute_flow_hash (ip1, flow_hash_config1);
     158             :           dpo1 =
     159         619 :             load_balance_get_fwd_bucket (lb1,
     160         619 :                                          (hash_c1 &
     161         619 :                                           (lb1->lb_n_buckets_minus_1)));
     162             :         }
     163             :       else
     164             :         {
     165     3511150 :           dpo1 = load_balance_get_bucket_i (lb1, 0);
     166             :         }
     167     3511770 :       if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
     168             :         {
     169         619 :           flow_hash_config2 = lb2->lb_hash_config;
     170        1238 :           hash_c2 = vnet_buffer (b[2])->ip.flow_hash =
     171         619 :             ip4_compute_flow_hash (ip2, flow_hash_config2);
     172             :           dpo2 =
     173         619 :             load_balance_get_fwd_bucket (lb2,
     174         619 :                                          (hash_c2 &
     175         619 :                                           (lb2->lb_n_buckets_minus_1)));
     176             :         }
     177             :       else
     178             :         {
     179     3511150 :           dpo2 = load_balance_get_bucket_i (lb2, 0);
     180             :         }
     181     3511770 :       if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
     182             :         {
     183         619 :           flow_hash_config3 = lb3->lb_hash_config;
     184        1238 :           hash_c3 = vnet_buffer (b[3])->ip.flow_hash =
     185         619 :             ip4_compute_flow_hash (ip3, flow_hash_config3);
     186             :           dpo3 =
     187         619 :             load_balance_get_fwd_bucket (lb3,
     188         619 :                                          (hash_c3 &
     189         619 :                                           (lb3->lb_n_buckets_minus_1)));
     190             :         }
     191             :       else
     192             :         {
     193     3511150 :           dpo3 = load_balance_get_bucket_i (lb3, 0);
     194             :         }
     195             : 
     196     3511770 :       next[0] = dpo0->dpoi_next_node;
     197     3511770 :       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
     198     3511770 :       next[1] = dpo1->dpoi_next_node;
     199     3511770 :       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
     200     3511770 :       next[2] = dpo2->dpoi_next_node;
     201     3511770 :       vnet_buffer (b[2])->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
     202     3511770 :       next[3] = dpo3->dpoi_next_node;
     203     3511770 :       vnet_buffer (b[3])->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
     204             : 
     205     3511770 :       vlib_increment_combined_counter
     206             :         (cm, thread_index, lb_index0, 1,
     207             :          vlib_buffer_length_in_chain (vm, b[0]));
     208     3511770 :       vlib_increment_combined_counter
     209             :         (cm, thread_index, lb_index1, 1,
     210     3511770 :          vlib_buffer_length_in_chain (vm, b[1]));
     211     3511770 :       vlib_increment_combined_counter
     212             :         (cm, thread_index, lb_index2, 1,
     213     3511770 :          vlib_buffer_length_in_chain (vm, b[2]));
     214     3511770 :       vlib_increment_combined_counter
     215             :         (cm, thread_index, lb_index3, 1,
     216     3511770 :          vlib_buffer_length_in_chain (vm, b[3]));
     217             : 
     218     3511770 :       b += 4;
     219     3511770 :       next += 4;
     220     3511770 :       n_left -= 4;
     221             :     }
     222             : #elif (CLIB_N_PREFETCHES >= 4)
     223             :   while (n_left >= 4)
     224             :     {
     225             :       ip4_header_t *ip0, *ip1;
     226             :       const load_balance_t *lb0, *lb1;
     227             :       ip4_address_t *dst_addr0, *dst_addr1;
     228             :       u32 lb_index0, lb_index1;
     229             :       flow_hash_config_t flow_hash_config0, flow_hash_config1;
     230             :       u32 hash_c0, hash_c1;
     231             :       const dpo_id_t *dpo0, *dpo1;
     232             : 
     233             :       /* Prefetch next iteration. */
     234             :       {
     235             :         vlib_prefetch_buffer_header (b[2], LOAD);
     236             :         vlib_prefetch_buffer_header (b[3], LOAD);
     237             : 
     238             :         CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), LOAD);
     239             :         CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), LOAD);
     240             :       }
     241             : 
     242             :       ip0 = vlib_buffer_get_current (b[0]);
     243             :       ip1 = vlib_buffer_get_current (b[1]);
     244             : 
     245             :       dst_addr0 = &ip0->dst_address;
     246             :       dst_addr1 = &ip1->dst_address;
     247             : 
     248             :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[0]);
     249             :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[1]);
     250             : 
     251             :       ip4_fib_forwarding_lookup_x2 (
     252             :         vnet_buffer (b[0])->ip.fib_index, vnet_buffer (b[1])->ip.fib_index,
     253             :         dst_addr0, dst_addr1, &lb_index0, &lb_index1);
     254             : 
     255             :       ASSERT (lb_index0 && lb_index1);
     256             :       lb0 = load_balance_get (lb_index0);
     257             :       lb1 = load_balance_get (lb_index1);
     258             : 
     259             :       ASSERT (lb0->lb_n_buckets > 0);
     260             :       ASSERT (is_pow2 (lb0->lb_n_buckets));
     261             :       ASSERT (lb1->lb_n_buckets > 0);
     262             :       ASSERT (is_pow2 (lb1->lb_n_buckets));
     263             : 
     264             :       /* Use flow hash to compute multipath adjacency. */
     265             :       hash_c0 = vnet_buffer (b[0])->ip.flow_hash = 0;
     266             :       hash_c1 = vnet_buffer (b[1])->ip.flow_hash = 0;
     267             :       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
     268             :         {
     269             :           flow_hash_config0 = lb0->lb_hash_config;
     270             :           hash_c0 = vnet_buffer (b[0])->ip.flow_hash =
     271             :             ip4_compute_flow_hash (ip0, flow_hash_config0);
     272             :           dpo0 =
     273             :             load_balance_get_fwd_bucket (lb0,
     274             :                                          (hash_c0 &
     275             :                                           (lb0->lb_n_buckets_minus_1)));
     276             :         }
     277             :       else
     278             :         {
     279             :           dpo0 = load_balance_get_bucket_i (lb0, 0);
     280             :         }
     281             :       if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
     282             :         {
     283             :           flow_hash_config1 = lb1->lb_hash_config;
     284             :           hash_c1 = vnet_buffer (b[1])->ip.flow_hash =
     285             :             ip4_compute_flow_hash (ip1, flow_hash_config1);
     286             :           dpo1 =
     287             :             load_balance_get_fwd_bucket (lb1,
     288             :                                          (hash_c1 &
     289             :                                           (lb1->lb_n_buckets_minus_1)));
     290             :         }
     291             :       else
     292             :         {
     293             :           dpo1 = load_balance_get_bucket_i (lb1, 0);
     294             :         }
     295             : 
     296             :       next[0] = dpo0->dpoi_next_node;
     297             :       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
     298             :       next[1] = dpo1->dpoi_next_node;
     299             :       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
     300             : 
     301             :       vlib_increment_combined_counter
     302             :         (cm, thread_index, lb_index0, 1,
     303             :          vlib_buffer_length_in_chain (vm, b[0]));
     304             :       vlib_increment_combined_counter
     305             :         (cm, thread_index, lb_index1, 1,
     306             :          vlib_buffer_length_in_chain (vm, b[1]));
     307             : 
     308             :       b += 2;
     309             :       next += 2;
     310             :       n_left -= 2;
     311             :     }
     312             : #endif
     313      780055 :   while (n_left > 0)
     314             :     {
     315             :       ip4_header_t *ip0;
     316             :       const load_balance_t *lb0;
     317             :       ip4_address_t *dst_addr0;
     318             :       u32 lbi0;
     319             :       flow_hash_config_t flow_hash_config0;
     320             :       const dpo_id_t *dpo0;
     321             :       u32 hash_c0;
     322             : 
     323      439282 :       ip0 = vlib_buffer_get_current (b[0]);
     324      439282 :       dst_addr0 = &ip0->dst_address;
     325      439282 :       ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, b[0]);
     326             : 
     327      439282 :       lbi0 = ip4_fib_forwarding_lookup (vnet_buffer (b[0])->ip.fib_index,
     328             :                                         dst_addr0);
     329             : 
     330      439282 :       ASSERT (lbi0);
     331      439282 :       lb0 = load_balance_get (lbi0);
     332             : 
     333      439282 :       ASSERT (lb0->lb_n_buckets > 0);
     334      439282 :       ASSERT (is_pow2 (lb0->lb_n_buckets));
     335             : 
     336             :       /* Use flow hash to compute multipath adjacency. */
     337      439282 :       hash_c0 = vnet_buffer (b[0])->ip.flow_hash = 0;
     338      439282 :       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
     339             :         {
     340          36 :           flow_hash_config0 = lb0->lb_hash_config;
     341             : 
     342          72 :           hash_c0 = vnet_buffer (b[0])->ip.flow_hash =
     343          36 :             ip4_compute_flow_hash (ip0, flow_hash_config0);
     344             :           dpo0 =
     345          36 :             load_balance_get_fwd_bucket (lb0,
     346          36 :                                          (hash_c0 &
     347          36 :                                           (lb0->lb_n_buckets_minus_1)));
     348             :         }
     349             :       else
     350             :         {
     351      439246 :           dpo0 = load_balance_get_bucket_i (lb0, 0);
     352             :         }
     353             : 
     354      439282 :       next[0] = dpo0->dpoi_next_node;
     355      439282 :       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
     356             : 
     357      439282 :       vlib_increment_combined_counter (cm, thread_index, lbi0, 1,
     358             :                                        vlib_buffer_length_in_chain (vm,
     359             :                                                                     b[0]));
     360             : 
     361      439282 :       b += 1;
     362      439282 :       next += 1;
     363      439282 :       n_left -= 1;
     364             :     }
     365             : 
     366      340773 :   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
     367             : 
     368      340773 :   if (node->flags & VLIB_NODE_FLAG_TRACE)
     369        7585 :     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
     370             : 
     371      340773 :   return frame->n_vectors;
     372             : }
     373             : 
     374             : #endif /* __included_ip4_forward_h__ */
     375             : 
     376             : /*
     377             :  * fd.io coding-style-patch-verification: ON
     378             :  *
     379             :  * Local Variables:
     380             :  * eval: (c-set-style "gnu")
     381             :  * End:
     382             :  */