Line data    Source code

       1             : /*
       2             :  *------------------------------------------------------------------
       3             :  * Copyright (c) 2021 Cisco and/or its affiliates.
       4             :  * Licensed under the Apache License, Version 2.0 (the "License");
       5             :  * you may not use this file except in compliance with the License.
       6             :  * You may obtain a copy of the License at:
       7             :  *
       8             :  *     http://www.apache.org/licenses/LICENSE-2.0
       9             :  *
      10             :  * Unless required by applicable law or agreed to in writing, software
      11             :  * distributed under the License is distributed on an "AS IS" BASIS,
      12             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      13             :  * See the License for the specific language governing permissions and
      14             :  * limitations under the License.
      15             :  *------------------------------------------------------------------
      16             :  */
      17             : 
      18             : #include <vlib/vlib.h>
      19             : #include <vnet/gso/gro_func.h>
      20             : #include <vnet/interface/tx_queue_funcs.h>
      21             : #include <vnet/devices/virtio/virtio.h>
      22             : #include <vnet/devices/virtio/virtio_inline.h>
      23             : 
      24             : static_always_inline uword
      25   159937000 : virtio_pre_input_inline (vlib_main_t *vm, vnet_virtio_vring_t *txq_vring,
      26             :                          vnet_hw_if_tx_queue_t *txq, u8 packet_coalesce,
      27             :                          u8 packet_buffering)
      28             : {
      29   159937000 :   if (txq->shared_queue)
      30             :     {
      31           0 :       if (clib_spinlock_trylock (&txq_vring->lockp))
      32             :         {
      33           0 :           if (virtio_txq_is_scheduled (txq_vring))
      34           0 :             goto unlock;
      35           0 :           if (packet_coalesce)
      36           0 :             vnet_gro_flow_table_schedule_node_on_dispatcher (
      37             :               vm, txq, txq_vring->flow_table);
      38           0 :           else if (packet_buffering)
      39           0 :             virtio_vring_buffering_schedule_node_on_dispatcher (
      40             :               vm, txq, txq_vring->buffering);
      41           0 :           virtio_txq_set_scheduled (txq_vring);
      42           0 :         unlock:
      43           0 :           clib_spinlock_unlock (&txq_vring->lockp);
      44             :         }
      45             :     }
      46             :   else
      47             :     {
      48   159937000 :       if (packet_coalesce)
      49   159937000 :         vnet_gro_flow_table_schedule_node_on_dispatcher (
      50             :           vm, txq, txq_vring->flow_table);
      51           0 :       else if (packet_buffering)
      52           0 :         virtio_vring_buffering_schedule_node_on_dispatcher (
      53             :           vm, txq, txq_vring->buffering);
      54             :     }
      55   159937000 :   return 0;
      56             : }
      57             : 
      58             : static uword
      59   148663000 : virtio_pre_input (vlib_main_t *vm, vlib_node_runtime_t *node,
      60             :                   vlib_frame_t *frame)
      61             : {
      62   148663000 :   virtio_main_t *vim = &virtio_main;
      63   148663000 :   vnet_main_t *vnm = vnet_get_main ();
      64             :   virtio_if_t *vif;
      65             : 
      66   407297000 :   pool_foreach (vif, vim->interfaces)
      67             :     {
      68   258634000 :       if (vif->packet_coalesce || vif->packet_buffering)
      69             :         {
      70             :           vnet_virtio_vring_t *txq_vring;
      71   319874000 :           vec_foreach (txq_vring, vif->txq_vrings)
      72             :             {
      73             :               vnet_hw_if_tx_queue_t *txq =
      74   159937000 :                 vnet_hw_if_get_tx_queue (vnm, txq_vring->queue_index);
      75   159937000 :               if (clib_bitmap_get (txq->threads, vm->thread_index) == 1)
      76   159937000 :                 virtio_pre_input_inline (vm, txq_vring, txq,
      77   159937000 :                                          vif->packet_coalesce,
      78   159937000 :                                          vif->packet_buffering);
      79             :             }
      80             :         }
      81             :     }
      82             : 
      83   148663000 :   return 0;
      84             : }
      85             : 
      86             : /**
      87             :  * virtio interfaces support packet coalescing and buffering which
      88             :  * depends on timer expiry to flush the stored packets periodically.
      89             :  * Previously, virtio input node checked timer expiry and scheduled
      90             :  * tx queue accordingly.
      91             :  *
      92             :  * In poll mode, timer expiry was handled naturally, as input node
      93             :  * runs periodically. In interrupt mode, virtio input node was dependent
      94             :  * on the interrupts send from backend. Stored packets could starve,
      95             :  * if there would not be interrupts to input node.
      96             :  *
      97             :  * This problem had been solved through a dedicated process node which
      98             :  * periodically sends interrupt to virtio input node given coalescing
      99             :  * or buffering feature were enabled on an interface.
     100             :  *
     101             :  * But that approach worked with following limitations:
     102             :  * 1) Each VPP thread should have (atleast) 1 rx queue of an interface
     103             :  * (with buffering enabled). And rxqs and txqs should be placed on the
     104             :  * same thread.
     105             :  *
     106             :  * New design provides solution to above problem(s) without any limitation
     107             :  * through (dedicated) pre-input node running on each VPP thread when
     108             :  * atleast 1 virtio interface is enabled with coalescing or buffering.
     109             :  */
     110      178120 : VLIB_REGISTER_NODE (virtio_pre_input_node) = {
     111             :   .function = virtio_pre_input,
     112             :   .type = VLIB_NODE_TYPE_PRE_INPUT,
     113             :   .name = "virtio-pre-input",
     114             :   .state = VLIB_NODE_STATE_DISABLED,
     115             : };
     116             : 
     117             : void
     118         300 : virtio_pre_input_node_enable (vlib_main_t *vm, virtio_if_t *vif)
     119             : {
     120         300 :   virtio_main_t *vim = &virtio_main;
     121         300 :   if (vif->packet_coalesce || vif->packet_buffering)
     122             :     {
     123         108 :       vim->gro_or_buffering_if_count++;
     124         108 :       if (vim->gro_or_buffering_if_count == 1)
     125             :         {
     126         192 :           foreach_vlib_main ()
     127             :             {
     128          96 :               vlib_node_set_state (this_vlib_main, virtio_pre_input_node.index,
     129             :                                    VLIB_NODE_STATE_POLLING);
     130             :             }
     131             :         }
     132             :     }
     133         300 : }
     134             : 
     135             : void
     136         300 : virtio_pre_input_node_disable (vlib_main_t *vm, virtio_if_t *vif)
     137             : {
     138         300 :   virtio_main_t *vim = &virtio_main;
     139         300 :   if (vif->packet_coalesce || vif->packet_buffering)
     140             :     {
     141         108 :       if (vim->gro_or_buffering_if_count > 0)
     142         108 :         vim->gro_or_buffering_if_count--;
     143         108 :       if (vim->gro_or_buffering_if_count == 0)
     144             :         {
     145         192 :           foreach_vlib_main ()
     146             :             {
     147          96 :               vlib_node_set_state (this_vlib_main, virtio_pre_input_node.index,
     148             :                                    VLIB_NODE_STATE_DISABLED);
     149             :             }
     150             :         }
     151             :     }
     152         300 : }
     153             : 
     154             : /*
     155             :  * fd.io coding-style-patch-verification: ON
     156             :  *
     157             :  * Local Variables:
     158             :  * eval: (c-set-style "gnu")
     159             :  * End:
     160             :  */