Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2021 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #include <vlib/vlib.h>
      17             : #include <vnet/ethernet/ethernet.h>
      18             : #include <vnet/hash/hash.h>
      19             : #include <vnet/ip/ip4_packet.h>
      20             : #include <vnet/ip/ip6_packet.h>
      21             : #include <vnet/mpls/packet.h>
      22             : #include <vppinfra/crc32.h>
      23             : #include <vppinfra/xxhash.h>
      24             : 
      25             : always_inline u32
      26           0 : ho_hash (u64 key)
      27             : {
      28             : #ifdef clib_crc32c_uses_intrinsics
      29           0 :   return clib_crc32c ((u8 *) &key, sizeof (key));
      30             : #else
      31             :   return clib_xxhash (key);
      32             : #endif
      33             : }
      34             : 
      35             : static inline u64
      36           0 : ipv4_get_key (ip4_header_t * ip)
      37             : {
      38             :   u64 hash_key;
      39             : 
      40           0 :   hash_key = *((u64 *) (&ip->address_pair)) ^ ip->protocol;
      41             : 
      42           0 :   return hash_key;
      43             : }
      44             : 
      45             : static inline u64
      46           0 : ipv6_get_key (ip6_header_t * ip)
      47             : {
      48             :   u64 hash_key;
      49             : 
      50           0 :   hash_key = ip->src_address.as_u64[0] ^
      51           0 :     rotate_left (ip->src_address.as_u64[1], 13) ^
      52           0 :     rotate_left (ip->dst_address.as_u64[0], 26) ^
      53           0 :     rotate_left (ip->dst_address.as_u64[1], 39) ^ ip->protocol;
      54             : 
      55           0 :   return hash_key;
      56             : }
      57             : 
      58             : #define MPLS_BOTTOM_OF_STACK_BIT_MASK   0x00000100U
      59             : #define MPLS_LABEL_MASK                 0xFFFFF000U
      60             : 
      61             : static inline u64
      62           0 : mpls_get_key (mpls_unicast_header_t * m)
      63             : {
      64             :   u64 hash_key;
      65             :   u8 ip_ver;
      66             : 
      67             : 
      68             :   /* find the bottom of the MPLS label stack. */
      69           0 :   if (PREDICT_TRUE (m->label_exp_s_ttl &
      70             :                     clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)))
      71             :     {
      72           0 :       goto bottom_lbl_found;
      73             :     }
      74           0 :   m++;
      75             : 
      76           0 :   if (PREDICT_TRUE (m->label_exp_s_ttl &
      77             :                     clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)))
      78             :     {
      79           0 :       goto bottom_lbl_found;
      80             :     }
      81           0 :   m++;
      82             : 
      83           0 :   if (m->label_exp_s_ttl &
      84           0 :       clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))
      85             :     {
      86           0 :       goto bottom_lbl_found;
      87             :     }
      88           0 :   m++;
      89             : 
      90           0 :   if (m->label_exp_s_ttl &
      91           0 :       clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))
      92             :     {
      93           0 :       goto bottom_lbl_found;
      94             :     }
      95           0 :   m++;
      96             : 
      97           0 :   if (m->label_exp_s_ttl &
      98           0 :       clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))
      99             :     {
     100           0 :       goto bottom_lbl_found;
     101             :     }
     102             : 
     103             :   /* the bottom label was not found - use the last label */
     104           0 :   hash_key = m->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK);
     105             : 
     106           0 :   return hash_key;
     107             : 
     108           0 : bottom_lbl_found:
     109           0 :   m++;
     110           0 :   ip_ver = (*((u8 *) m) >> 4);
     111             : 
     112             :   /* find out if it is IPV4 or IPV6 header */
     113           0 :   if (PREDICT_TRUE (ip_ver == 4))
     114             :     {
     115           0 :       hash_key = ipv4_get_key ((ip4_header_t *) m);
     116             :     }
     117           0 :   else if (PREDICT_TRUE (ip_ver == 6))
     118             :     {
     119           0 :       hash_key = ipv6_get_key ((ip6_header_t *) m);
     120             :     }
     121             :   else
     122             :     {
     123             :       /* use the bottom label */
     124           0 :       hash_key =
     125           0 :         (m - 1)->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK);
     126             :     }
     127             : 
     128           0 :   return hash_key;
     129             : 
     130             : }
     131             : 
     132             : static inline u64
     133           0 : eth_get_sym_key (ethernet_header_t * h0)
     134             : {
     135             :   u64 hash_key;
     136             : 
     137           0 :   if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
     138             :     {
     139           0 :       ip4_header_t *ip = (ip4_header_t *) (h0 + 1);
     140           0 :       hash_key =
     141           0 :         (u64) (ip->src_address.as_u32 ^
     142           0 :                ip->dst_address.as_u32 ^ ip->protocol);
     143             :     }
     144           0 :   else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
     145             :     {
     146           0 :       ip6_header_t *ip = (ip6_header_t *) (h0 + 1);
     147           0 :       hash_key = (u64) (ip->src_address.as_u64[0] ^
     148           0 :                         ip->src_address.as_u64[1] ^
     149           0 :                         ip->dst_address.as_u64[0] ^
     150           0 :                         ip->dst_address.as_u64[1] ^ ip->protocol);
     151             :     }
     152           0 :   else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     153             :     {
     154           0 :       hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1));
     155             :     }
     156             :   else
     157           0 :     if (PREDICT_FALSE
     158             :         ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN))
     159             :          || (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD))))
     160           0 :     {
     161           0 :       ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1);
     162             : 
     163           0 :       outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ?
     164           0 :         outer + 1 : outer;
     165           0 :       if (PREDICT_TRUE (outer->type) ==
     166           0 :           clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
     167             :         {
     168           0 :           ip4_header_t *ip = (ip4_header_t *) (outer + 1);
     169           0 :           hash_key =
     170           0 :             (u64) (ip->src_address.as_u32 ^
     171           0 :                    ip->dst_address.as_u32 ^ ip->protocol);
     172             :         }
     173           0 :       else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
     174             :         {
     175           0 :           ip6_header_t *ip = (ip6_header_t *) (outer + 1);
     176           0 :           hash_key =
     177           0 :             (u64) (ip->src_address.as_u64[0] ^ ip->src_address.as_u64[1] ^
     178           0 :                    ip->dst_address.as_u64[0] ^
     179           0 :                    ip->dst_address.as_u64[1] ^ ip->protocol);
     180             :         }
     181           0 :       else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     182             :         {
     183           0 :           hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1));
     184             :         }
     185             :       else
     186             :         {
     187           0 :           hash_key = outer->type;
     188             :         }
     189             :     }
     190             :   else
     191             :     {
     192           0 :       hash_key = 0;
     193             :     }
     194             : 
     195           0 :   return hash_key;
     196             : }
     197             : 
     198             : static inline u64
     199           0 : eth_get_key (ethernet_header_t * h0)
     200             : {
     201             :   u64 hash_key;
     202             : 
     203           0 :   if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
     204             :     {
     205           0 :       hash_key = ipv4_get_key ((ip4_header_t *) (h0 + 1));
     206             :     }
     207           0 :   else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
     208             :     {
     209           0 :       hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1));
     210             :     }
     211           0 :   else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     212             :     {
     213           0 :       hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1));
     214             :     }
     215           0 :   else if ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ||
     216           0 :            (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD)))
     217           0 :     {
     218           0 :       ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1);
     219             : 
     220           0 :       outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ?
     221           0 :         outer + 1 : outer;
     222           0 :       if (PREDICT_TRUE (outer->type) ==
     223           0 :           clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
     224             :         {
     225           0 :           hash_key = ipv4_get_key ((ip4_header_t *) (outer + 1));
     226             :         }
     227           0 :       else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
     228             :         {
     229           0 :           hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1));
     230             :         }
     231           0 :       else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     232             :         {
     233           0 :           hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1));
     234             :         }
     235             :       else
     236             :         {
     237           0 :           hash_key = outer->type;
     238             :         }
     239             :     }
     240             :   else
     241             :     {
     242           0 :       hash_key = 0;
     243             :     }
     244             : 
     245           0 :   return hash_key;
     246             : }
     247             : 
     248             : void
     249           0 : handoff_eth_func (void **p, u32 *hash, u32 n_packets)
     250             : {
     251           0 :   u32 n_left_from = n_packets;
     252             : 
     253           0 :   while (n_left_from >= 8)
     254             :     {
     255           0 :       u64 key[4] = {};
     256             : 
     257           0 :       clib_prefetch_load (p[4]);
     258           0 :       clib_prefetch_load (p[5]);
     259           0 :       clib_prefetch_load (p[6]);
     260           0 :       clib_prefetch_load (p[7]);
     261             : 
     262           0 :       key[0] = eth_get_key ((ethernet_header_t *) p[0]);
     263           0 :       key[1] = eth_get_key ((ethernet_header_t *) p[1]);
     264           0 :       key[2] = eth_get_key ((ethernet_header_t *) p[2]);
     265           0 :       key[3] = eth_get_key ((ethernet_header_t *) p[3]);
     266             : 
     267           0 :       hash[0] = ho_hash (key[0]);
     268           0 :       hash[1] = ho_hash (key[1]);
     269           0 :       hash[2] = ho_hash (key[2]);
     270           0 :       hash[3] = ho_hash (key[3]);
     271             : 
     272           0 :       hash += 4;
     273           0 :       n_left_from -= 4;
     274           0 :       p += 4;
     275             :     }
     276             : 
     277           0 :   while (n_left_from > 0)
     278             :     {
     279             :       u64 key;
     280             : 
     281           0 :       key = eth_get_key ((ethernet_header_t *) p[0]);
     282           0 :       hash[0] = ho_hash (key);
     283             : 
     284           0 :       hash += 1;
     285           0 :       n_left_from -= 1;
     286           0 :       p += 1;
     287             :     }
     288           0 : }
     289             : 
     290         575 : VNET_REGISTER_HASH_FUNCTION (handoff_eth, static) = {
     291             :   .name = "handoff-eth",
     292             :   .description = "Ethernet/IPv4/IPv6/MPLS headers",
     293             :   .priority = 2,
     294             :   .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_func,
     295             : };
     296             : 
     297             : void
     298           0 : handoff_eth_sym_func (void **p, u32 *hash, u32 n_packets)
     299             : {
     300           0 :   u32 n_left_from = n_packets;
     301             : 
     302           0 :   while (n_left_from >= 8)
     303             :     {
     304           0 :       u64 key[4] = {};
     305             : 
     306           0 :       clib_prefetch_load (p[4]);
     307           0 :       clib_prefetch_load (p[5]);
     308           0 :       clib_prefetch_load (p[6]);
     309           0 :       clib_prefetch_load (p[7]);
     310             : 
     311           0 :       key[0] = eth_get_sym_key ((ethernet_header_t *) p[0]);
     312           0 :       key[1] = eth_get_sym_key ((ethernet_header_t *) p[1]);
     313           0 :       key[2] = eth_get_sym_key ((ethernet_header_t *) p[2]);
     314           0 :       key[3] = eth_get_sym_key ((ethernet_header_t *) p[3]);
     315             : 
     316           0 :       hash[0] = ho_hash (key[0]);
     317           0 :       hash[1] = ho_hash (key[1]);
     318           0 :       hash[2] = ho_hash (key[2]);
     319           0 :       hash[3] = ho_hash (key[3]);
     320             : 
     321           0 :       hash += 4;
     322           0 :       n_left_from -= 4;
     323           0 :       p += 4;
     324             :     }
     325             : 
     326           0 :   while (n_left_from > 0)
     327             :     {
     328             :       u64 key;
     329             : 
     330           0 :       key = eth_get_sym_key ((ethernet_header_t *) p[0]);
     331           0 :       hash[0] = ho_hash (key);
     332             : 
     333           0 :       hash += 1;
     334           0 :       n_left_from -= 1;
     335           0 :       p += 1;
     336             :     }
     337           0 : }
     338             : 
     339         575 : VNET_REGISTER_HASH_FUNCTION (handoff_eth_sym, static) = {
     340             :   .name = "handoff-eth-sym",
     341             :   .description = "Ethernet/IPv4/IPv6/MPLS headers Symmetric",
     342             :   .priority = 1,
     343             :   .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_sym_func,
     344             : };
     345             : 
     346             : /*
     347             :  * fd.io coding-style-patch-verification: ON
     348             :  *
     349             :  * Local Variables:
     350             :  * eval: (c-set-style "gnu")
     351             :  * End:
     352             :  */