Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2020 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : 
      16             : #ifndef SRC_SVM_FIFO_TYPES_H_
      17             : #define SRC_SVM_FIFO_TYPES_H_
      18             : 
      19             : #include <svm/ssvm.h>
      20             : #include <vppinfra/clib.h>
      21             : #include <vppinfra/rbtree.h>
      22             : #include <vppinfra/lock.h>
      23             : 
      24             : #define FS_MIN_LOG2_CHUNK_SZ 12 /**< also min fifo size */
      25             : #define FS_MAX_LOG2_CHUNK_SZ 22 /**< 4MB max chunk size */
      26             : #define FS_CHUNK_VEC_LEN     11 /**< number of chunk sizes */
      27             : 
      28             : STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) ==
      29             :                  FS_CHUNK_VEC_LEN - 1,
      30             :                "update chunk sizes");
      31             : 
      32             : #define SVM_FIFO_TRACE                  (0)
      33             : #define SVM_FIFO_MAX_EVT_SUBSCRIBERS    7
      34             : 
      35             : typedef struct fifo_segment_header_ fifo_segment_header_t;
      36             : typedef uword fs_sptr_t;
      37             : 
      38             : typedef struct svm_fifo_chunk_
      39             : {
      40             :   u32 start_byte;               /**< chunk start byte */
      41             :   u32 length;                   /**< length of chunk in bytes */
      42             :   fs_sptr_t next;               /**< pointer to next chunk in linked-lists */
      43             :   rb_node_index_t enq_rb_index; /**< enq node index if chunk in rbtree */
      44             :   rb_node_index_t deq_rb_index; /**< deq node index if chunk in rbtree */
      45             :   u8 data[0];                   /**< start of chunk data */
      46             : } svm_fifo_chunk_t;
      47             : 
      48             : typedef struct
      49             : {
      50             :   u32 next;     /**< Next linked-list element pool index */
      51             :   u32 prev;     /**< Previous linked-list element pool index */
      52             :   u32 start;    /**< Start of segment, normalized*/
      53             :   u32 length;   /**< Length of segment */
      54             : } ooo_segment_t;
      55             : 
      56             : typedef struct
      57             : {
      58             :   u32 offset;
      59             :   u32 len;
      60             :   u32 action;
      61             : } svm_fifo_trace_elem_t;
      62             : 
      63             : typedef struct svm_fifo_shr_
      64             : {
      65             :   CLIB_CACHE_LINE_ALIGN_MARK (shared);
      66             :   fs_sptr_t start_chunk;        /**< first chunk in fifo chunk list */
      67             :   fs_sptr_t end_chunk;          /**< end chunk in fifo chunk list */
      68             :   volatile u32 has_event;       /**< non-zero if deq event exists */
      69             :   u32 min_alloc;                /**< min chunk alloc if space available */
      70             :   u32 size;                     /**< size of the fifo in bytes */
      71             :   u32 master_session_index;     /**< session layer session index */
      72             :   u32 client_session_index;     /**< app session index */
      73             :   u8 slice_index;               /**< segment slice for fifo */
      74             :   fs_sptr_t next;               /**< next in freelist */
      75             : 
      76             :   CLIB_CACHE_LINE_ALIGN_MARK (consumer);
      77             :   fs_sptr_t head_chunk;         /**< tracks chunk where head lands */
      78             :   u32 head;                     /**< fifo head position/byte */
      79             :   volatile u32 want_deq_ntf;    /**< producer wants nudge */
      80             :   volatile u32 has_deq_ntf;
      81             :   u32 deq_thresh; /**< fifo threshold used for notifications */
      82             : 
      83             :   CLIB_CACHE_LINE_ALIGN_MARK (producer);
      84             :   u32 tail;                     /**< fifo tail position/byte */
      85             :   fs_sptr_t tail_chunk;         /**< tracks chunk where tail lands */
      86             :   volatile u8 n_subscribers;    /**< Number of subscribers for io events */
      87             :   u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
      88             : } svm_fifo_shared_t;
      89             : 
      90             : typedef struct _svm_fifo
      91             : {
      92             :   CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
      93             :   svm_fifo_shared_t *shr;        /**< shared fifo in fifo segment memory */
      94             :   fifo_segment_header_t *fs_hdr; /**< fifo segment header for fifo */
      95             :   rb_tree_t ooo_enq_lookup;      /**< rbtree for ooo enq chunk lookup */
      96             :   rb_tree_t ooo_deq_lookup;      /**< rbtree for ooo deq chunk lookup */
      97             :   svm_fifo_chunk_t *ooo_deq;     /**< last chunk used for ooo dequeue */
      98             :   svm_fifo_chunk_t *ooo_enq;     /**< last chunk used for ooo enqueue */
      99             :   ooo_segment_t *ooo_segments;   /**< Pool of ooo segments */
     100             :   u32 ooos_list_head;            /**< Head of out-of-order linked-list */
     101             :   u32 ooos_newest;               /**< Last segment to have been updated */
     102             : 
     103             :   u8 flags;               /**< fifo flags */
     104             :   u8 master_thread_index; /**< session layer thread index */
     105             :   u8 client_thread_index; /**< app worker index */
     106             :   i8 refcnt;              /**< reference count  */
     107             :   u32 segment_manager;    /**< session layer segment manager index */
     108             :   u32 segment_index;      /**< segment index in segment manager */
     109             : 
     110             :   struct _svm_fifo *next; /**< prev in active chain */
     111             :   struct _svm_fifo *prev; /**< prev in active chain */
     112             : 
     113             :   svm_fifo_chunk_t *chunks_at_attach; /**< chunks to be accounted at detach */
     114             :   svm_fifo_shared_t *hdr_at_attach;   /**< hdr to be freed at detach */
     115             : 
     116             : #if SVM_FIFO_TRACE
     117             :   svm_fifo_trace_elem_t *trace;
     118             : #endif
     119             : } svm_fifo_t;
     120             : 
     121             : typedef struct fifo_segment_slice_
     122             : {
     123             :   CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
     124             :   fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
     125             :   fs_sptr_t free_fifos;                 /**< Freelists of fifo shared hdrs  */
     126             :   uword n_fl_chunk_bytes;               /**< Chunk bytes on freelist */
     127             :   uword virtual_mem;                    /**< Slice sum of all fifo sizes */
     128             :   u32 num_chunks[FS_CHUNK_VEC_LEN];     /**< Allocated chunks by chunk size */
     129             : } fifo_segment_slice_t;
     130             : 
     131             : typedef struct fifo_slice_private_
     132             : {
     133             :   clib_mem_bulk_handle_t fifos; /**< Bulk fifo allocator */
     134             :   uword virtual_mem;            /**< Slice sum of all fifo sizes */
     135             :   svm_fifo_t *active_fifos;     /**< Linked list of active RX fifos */
     136             : } fifo_slice_private_t;
     137             : 
     138             : struct fifo_segment_header_
     139             : {
     140             :   uword n_cached_bytes;                 /**< Cached bytes */
     141             :   u32 n_active_fifos;                   /**< Number of active fifos */
     142             :   u32 n_reserved_bytes;                 /**< Bytes not to be allocated */
     143             :   u32 max_log2_fifo_size;               /**< Max log2(chunk size) for fs */
     144             :   u8 n_slices;                          /**< Number of slices */
     145             :   u8 pct_first_alloc;                   /**< Pct of fifo size to alloc */
     146             :   u8 n_mqs;                             /**< Num mqs for mqs segment */
     147             :   CLIB_CACHE_LINE_ALIGN_MARK (allocator);
     148             :   uword byte_index;
     149             :   uword max_byte_index;
     150             :   uword start_byte_index;
     151             :   CLIB_CACHE_LINE_ALIGN_MARK (slice);
     152             :   fifo_segment_slice_t slices[0]; /** Fixed array of slices */
     153             : };
     154             : 
     155             : void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
     156             :                              int n_bytes);
     157             : 
     158             : always_inline void *
     159        1111 : fs_ptr (fifo_segment_header_t *fsh, fs_sptr_t sp)
     160             : {
     161        1111 :   return sp ? (void *) ((u8 *) fsh + sp) : 0;
     162             : }
     163             : 
     164             : always_inline fs_sptr_t
     165        6003 : fs_sptr (fifo_segment_header_t *fsh, void *p)
     166             : {
     167        6003 :   return p ? (fs_sptr_t) ((u8 *) p - (u8 *) fsh) : 0;
     168             : }
     169             : 
     170             : always_inline svm_fifo_chunk_t *
     171    10679153 : fs_chunk_ptr (fifo_segment_header_t *fsh, fs_sptr_t cp)
     172             : {
     173    10679153 :   return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + cp) : 0;
     174             : }
     175             : 
     176             : always_inline fs_sptr_t
     177     1249782 : fs_chunk_sptr (fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
     178             : {
     179     1249782 :   return c ? (fs_sptr_t) ((u8 *) c - (u8 *) fsh) : 0;
     180             : }
     181             : 
     182             : #endif /* SRC_SVM_FIFO_TYPES_H_ */
     183             : 
     184             : /*
     185             :  * fd.io coding-style-patch-verification: ON
     186             :  *
     187             :  * Local Variables:
     188             :  * eval: (c-set-style "gnu")
     189             :  * End:
     190             :  */