Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2020 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #ifndef included_clib_interrupt_h
      17             : #define included_clib_interrupt_h
      18             : 
      19             : #include <vppinfra/clib.h>
      20             : #include <vppinfra/vec.h>
      21             : 
      22             : typedef struct
      23             : {
      24             :   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
      25             :   int n_int;
      26             :   uword n_uword_alloc;
      27             : } clib_interrupt_header_t;
      28             : 
      29             : void clib_interrupt_init (void **data, uword n_interrupts);
      30             : void clib_interrupt_resize (void **data, uword n_interrupts);
      31             : 
      32             : static_always_inline void
      33        1312 : clib_interrupt_free (void **data)
      34             : {
      35        1312 :   if (data[0])
      36             :     {
      37         144 :       clib_mem_free (data[0]);
      38         144 :       data[0] = 0;
      39             :     }
      40        1312 : }
      41             : 
      42             : static_always_inline int
      43       17774 : clib_interrupt_get_n_int (void *d)
      44             : {
      45       17774 :   clib_interrupt_header_t *h = d;
      46       17774 :   if (h)
      47         828 :     return h->n_int;
      48       16946 :   return 0;
      49             : }
      50             : 
      51             : static_always_inline uword *
      52     3975967 : clib_interrupt_get_bitmap (void *d)
      53             : {
      54     3975967 :   return d + sizeof (clib_interrupt_header_t);
      55             : }
      56             : 
      57             : static_always_inline uword *
      58     1662647 : clib_interrupt_get_atomic_bitmap (void *d)
      59             : {
      60     1662647 :   clib_interrupt_header_t *h = d;
      61     1662647 :   return clib_interrupt_get_bitmap (d) + h->n_uword_alloc;
      62             : }
      63             : 
      64             : static_always_inline void
      65      651142 : clib_interrupt_set (void *in, int int_num)
      66             : {
      67      651142 :   uword *bmp = clib_interrupt_get_bitmap (in);
      68      651142 :   uword mask = 1ULL << (int_num & (uword_bits - 1));
      69      651142 :   bmp += int_num >> log2_uword_bits;
      70             : 
      71      651142 :   ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
      72             : 
      73      651142 :   *bmp |= mask;
      74      651142 : }
      75             : 
      76             : static_always_inline void
      77         459 : clib_interrupt_set_atomic (void *in, int int_num)
      78             : {
      79         459 :   uword *bmp = clib_interrupt_get_atomic_bitmap (in);
      80         459 :   uword mask = 1ULL << (int_num & (uword_bits - 1));
      81         459 :   bmp += int_num >> log2_uword_bits;
      82             : 
      83         459 :   ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
      84             : 
      85         459 :   __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED);
      86         459 : }
      87             : 
      88             : static_always_inline void
      89      576485 : clib_interrupt_clear (void *in, int int_num)
      90             : {
      91      576485 :   uword *bmp = clib_interrupt_get_bitmap (in);
      92      576480 :   uword *abm = clib_interrupt_get_atomic_bitmap (in);
      93      576476 :   uword mask = 1ULL << (int_num & (uword_bits - 1));
      94      576476 :   uword off = int_num >> log2_uword_bits;
      95             : 
      96      576476 :   ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
      97             : 
      98      576476 :   bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
      99      576476 :   bmp[off] &= ~mask;
     100      576476 : }
     101             : 
     102             : static_always_inline int
     103     1085770 : clib_interrupt_get_next (void *in, int last)
     104             : {
     105     1085770 :   uword *bmp = clib_interrupt_get_bitmap (in);
     106     1085766 :   uword *abm = clib_interrupt_get_atomic_bitmap (in);
     107     1085768 :   clib_interrupt_header_t *h = in;
     108             :   uword bmp_uword, off;
     109             : 
     110     1085768 :   ASSERT (last >= -1 && last < h->n_int);
     111             : 
     112     1085768 :   off = (last + 1) >> log2_uword_bits;
     113     1085768 :   if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
     114           0 :     return -1;
     115             : 
     116     1085768 :   last -= off << log2_uword_bits;
     117     1085768 :   bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
     118     1085768 :   bmp_uword = bmp[off] & ~pow2_mask (last + 1);
     119             : 
     120     1085919 : next:
     121     1085919 :   if (bmp_uword)
     122      576243 :     return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword);
     123             : 
     124      509676 :   off++;
     125             : 
     126      509676 :   if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
     127      509674 :     return -1;
     128             : 
     129           2 :   bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
     130           2 :   bmp_uword = bmp[off];
     131             : 
     132           2 :   goto next;
     133             : }
     134             : 
     135             : #endif /* included_clib_interrupt_h */
     136             : 
     137             : /*
     138             :  * fd.io coding-style-patch-verification: ON
     139             :  *
     140             :  * Local Variables:
     141             :  * eval: (c-set-style "gnu")
     142             :  * End:
     143             :  */