Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2020 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #include "vppinfra/string.h"
      17             : #include <vnet/vnet.h>
      18             : 
      19             : #include <vlibapi/api.h>
      20             : #include <vlibmemory/api.h>
      21             : #include <vnet/plugin/plugin.h>
      22             : #include <vpp/app/version.h>
      23             : #include <linux/limits.h>
      24             : #include <sys/ioctl.h>
      25             : 
      26             : #include <perfmon/perfmon.h>
      27             : 
      28             : vlib_node_function_t *perfmon_dispatch_wrappers[PERF_MAX_EVENTS + 1];
      29             : 
      30             : static_always_inline void
      31           0 : perfmon_read_pmcs (u64 *counters, u32 *indexes, u8 n_counters)
      32             : {
      33           0 :   for (int i = 0; i < n_counters; i++)
      34           0 :     counters[i] = _rdpmc (indexes[i] - 1);
      35           0 : }
      36             : 
      37             : static_always_inline uword
      38           0 : perfmon_dispatch_wrapper_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
      39             :                                  vlib_frame_t *frame, u8 n_events)
      40             : {
      41           0 :   perfmon_main_t *pm = &perfmon_main;
      42           0 :   perfmon_thread_runtime_t *rt =
      43           0 :     vec_elt_at_index (pm->thread_runtimes, vm->thread_index);
      44           0 :   perfmon_node_stats_t *s =
      45           0 :     vec_elt_at_index (rt->node_stats, node->node_index);
      46             : 
      47             :   struct
      48             :   {
      49             :     u64 t[2][PERF_MAX_EVENTS];
      50             :   } samples;
      51             :   uword rv;
      52             : 
      53           0 :   clib_prefetch_load (s);
      54             : 
      55           0 :   perfmon_read_pmcs (&samples.t[0][0], &rt->indexes[0], n_events);
      56           0 :   rv = node->function (vm, node, frame);
      57           0 :   perfmon_read_pmcs (&samples.t[1][0], &rt->indexes[0], n_events);
      58             : 
      59           0 :   if (rv == 0)
      60           0 :     return rv;
      61             : 
      62           0 :   s->n_calls += 1;
      63           0 :   s->n_packets += rv;
      64             : 
      65           0 :   for (int i = 0; i < n_events; i++)
      66             :     {
      67           0 :       if (!(rt->preserve_samples & 1 << i))
      68             :         {
      69           0 :           s->value[i] += samples.t[1][i] - samples.t[0][i];
      70             :         }
      71             :       else
      72             :         {
      73           0 :           s->t[0].value[i] = samples.t[0][i];
      74           0 :           s->t[1].value[i] = samples.t[1][i];
      75             :         }
      76             :     }
      77             : 
      78           0 :   return rv;
      79             : }
      80             : 
      81             : static_always_inline u32
      82           0 : perfmon_mmap_read_index (const struct perf_event_mmap_page *mmap_page)
      83             : {
      84             :   u32 idx;
      85             :   u32 seq;
      86             : 
      87             :   /* See documentation in /usr/include/linux/perf_event.h, for more details
      88             :    * but the 2 main important things are:
      89             :    *  1) if seq != mmap_page->lock, it means the kernel is currently updating
      90             :    *     the user page and we need to read it again
      91             :    *  2) if idx == 0, it means the perf event is currently turned off and we
      92             :    *     just need to read the kernel-updated 'offset', otherwise we must also
      93             :    *     add the current hw value (hence rdmpc) */
      94             :   do
      95             :     {
      96           0 :       seq = mmap_page->lock;
      97           0 :       CLIB_COMPILER_BARRIER ();
      98             : 
      99           0 :       idx = mmap_page->index;
     100             : 
     101           0 :       CLIB_COMPILER_BARRIER ();
     102             :     }
     103           0 :   while (mmap_page->lock != seq);
     104             : 
     105           0 :   return idx;
     106             : }
     107             : 
     108             : static_always_inline clib_error_t *
     109           0 : read_mmap_indexes (perfmon_bundle_t *b)
     110             : {
     111           0 :   perfmon_main_t *pm = &perfmon_main;
     112           0 :   for (int i = 0; i < vec_len (pm->thread_runtimes); i++)
     113             :     {
     114             :       perfmon_thread_runtime_t *tr;
     115           0 :       tr = vec_elt_at_index (pm->thread_runtimes, i);
     116             : 
     117           0 :       for (int j = 0; j < b->n_events; j++)
     118             :         {
     119           0 :           tr->indexes[j] = perfmon_mmap_read_index (tr->mmap_pages[j]);
     120             : 
     121             :           /* if a zero index is returned generate error */
     122           0 :           if (!tr->indexes[j])
     123             :             {
     124           0 :               return clib_error_return (0, "invalid rdpmc index");
     125             :             }
     126             :         }
     127             :     }
     128           0 :   return 0;
     129             : }
     130             : 
     131             : clib_error_t *
     132           0 : intel_config_dispatch_wrapper (perfmon_bundle_t *b,
     133             :                                vlib_node_function_t **dispatch_wrapper)
     134             : {
     135           0 :   clib_error_t *err = 0;
     136           0 :   if ((err = read_mmap_indexes (b)) != 0)
     137           0 :     return err;
     138             : 
     139           0 :   (*dispatch_wrapper) = perfmon_dispatch_wrappers[b->n_events];
     140           0 :   return 0;
     141             : }
     142             : 
     143             : #define foreach_n_events                                                      \
     144             :   _ (1) _ (2) _ (3) _ (4) _ (5) _ (6) _ (7) _ (8) _ (9) _ (10) _ (11) _ (12)
     145             : 
     146             : #define _(x)                                                                  \
     147             :   static uword perfmon_dispatch_wrapper##x (                                  \
     148             :     vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)          \
     149             :   {                                                                           \
     150             :     return perfmon_dispatch_wrapper_inline (vm, node, frame, x);              \
     151             :   }
     152             : 
     153           0 : foreach_n_events
     154             : #undef _
     155             : 
     156             :   vlib_node_function_t *perfmon_dispatch_wrappers[PERF_MAX_EVENTS + 1] = {
     157             : #define _(x) [x] = &perfmon_dispatch_wrapper##x,
     158             :     foreach_n_events
     159             : #undef _
     160             :   };