Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2020 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #ifndef included_clib_interrupt_h
      17             : #define included_clib_interrupt_h
      18             : 
      19             : #include <vppinfra/clib.h>
      20             : #include <vppinfra/vec.h>
      21             : 
      22             : typedef struct
      23             : {
      24             :   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
      25             :   int n_int;
      26             :   uword n_uword_alloc;
      27             : } clib_interrupt_header_t;
      28             : 
      29             : void clib_interrupt_init (void **data, uword n_interrupts);
      30             : void clib_interrupt_resize (void **data, uword n_interrupts);
      31             : 
      32             : static_always_inline void
      33        1474 : clib_interrupt_free (void **data)
      34             : {
      35        1474 :   if (data[0])
      36             :     {
      37         144 :       clib_mem_free (data[0]);
      38         144 :       data[0] = 0;
      39             :     }
      40        1474 : }
      41             : 
      42             : static_always_inline int
      43       18144 : clib_interrupt_get_n_int (void *d)
      44             : {
      45       18144 :   clib_interrupt_header_t *h = d;
      46       18144 :   if (h)
      47         828 :     return h->n_int;
      48       17316 :   return 0;
      49             : }
      50             : 
      51             : static_always_inline uword *
      52     5546870 : clib_interrupt_get_bitmap (void *d)
      53             : {
      54     5546870 :   return d + sizeof (clib_interrupt_header_t);
      55             : }
      56             : 
      57             : static_always_inline uword *
      58     2403909 : clib_interrupt_get_atomic_bitmap (void *d)
      59             : {
      60     2403909 :   clib_interrupt_header_t *h = d;
      61     2403909 :   return clib_interrupt_get_bitmap (d) + h->n_uword_alloc;
      62             : }
      63             : 
      64             : static_always_inline void
      65      739542 : clib_interrupt_set (void *in, int int_num)
      66             : {
      67      739542 :   uword *bmp = clib_interrupt_get_bitmap (in);
      68      739542 :   uword mask = 1ULL << (int_num & (uword_bits - 1));
      69      739542 :   bmp += int_num >> log2_uword_bits;
      70             : 
      71      739542 :   ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
      72             : 
      73      739542 :   *bmp |= mask;
      74      739542 : }
      75             : 
      76             : static_always_inline void
      77         450 : clib_interrupt_set_atomic (void *in, int int_num)
      78             : {
      79         450 :   uword *bmp = clib_interrupt_get_atomic_bitmap (in);
      80         450 :   uword mask = 1ULL << (int_num & (uword_bits - 1));
      81         450 :   bmp += int_num >> log2_uword_bits;
      82             : 
      83         450 :   ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
      84             : 
      85         450 :   __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED);
      86         450 : }
      87             : 
      88             : static_always_inline void
      89      664004 : clib_interrupt_clear (void *in, int int_num)
      90             : {
      91      664004 :   uword *bmp = clib_interrupt_get_bitmap (in);
      92      664004 :   uword *abm = clib_interrupt_get_atomic_bitmap (in);
      93      664001 :   uword mask = 1ULL << (int_num & (uword_bits - 1));
      94      664001 :   uword off = int_num >> log2_uword_bits;
      95             : 
      96      664001 :   ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
      97             : 
      98      664001 :   bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
      99      664001 :   bmp[off] &= ~mask;
     100      664001 : }
     101             : 
     102             : static_always_inline int
     103     1739517 : clib_interrupt_get_next (void *in, int last)
     104             : {
     105     1739517 :   uword *bmp = clib_interrupt_get_bitmap (in);
     106     1739507 :   uword *abm = clib_interrupt_get_atomic_bitmap (in);
     107     1739507 :   clib_interrupt_header_t *h = in;
     108             :   uword bmp_uword, off;
     109             : 
     110     1739507 :   ASSERT (last >= -1 && last < h->n_int);
     111             : 
     112     1739507 :   off = (last + 1) >> log2_uword_bits;
     113     1739507 :   if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
     114           0 :     return -1;
     115             : 
     116     1739507 :   last -= off << log2_uword_bits;
     117     1739507 :   bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
     118     1739507 :   bmp_uword = bmp[off] & ~pow2_mask (last + 1);
     119             : 
     120     1739737 : next:
     121     1739737 :   if (bmp_uword)
     122      663767 :     return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword);
     123             : 
     124     1075970 :   off++;
     125             : 
     126     1075970 :   if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
     127     1075959 :     return -1;
     128             : 
     129          11 :   bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
     130          11 :   bmp_uword = bmp[off];
     131             : 
     132          11 :   goto next;
     133             : }
     134             : 
     135             : #endif /* included_clib_interrupt_h */
     136             : 
     137             : /*
     138             :  * fd.io coding-style-patch-verification: ON
     139             :  *
     140             :  * Local Variables:
     141             :  * eval: (c-set-style "gnu")
     142             :  * End:
     143             :  */