Line data Source code
1 : /*
2 : *------------------------------------------------------------------
3 : * Copyright (c) 2017 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : *------------------------------------------------------------------
16 : */
17 :
18 : #include <vppinfra/lock.h>
19 : #include <vlib/dma/dma.h>
20 : #include <vlib/log.h>
21 :
22 : #define MEMIF_DEFAULT_SOCKET_FILENAME "memif.sock"
23 : #define MEMIF_DEFAULT_RING_SIZE 1024
24 : #define MEMIF_DEFAULT_RX_QUEUES 1
25 : #define MEMIF_DEFAULT_TX_QUEUES 1
26 : #define MEMIF_DEFAULT_BUFFER_SIZE 2048
27 :
28 : #define MEMIF_MAX_M2S_RING 256
29 : #define MEMIF_MAX_S2M_RING 256
30 : #define MEMIF_MAX_REGION 256
31 : #define MEMIF_MAX_LOG2_RING_SIZE 14
32 :
33 :
34 : #define memif_log_debug(dev, f, ...) do { \
35 : memif_if_t *_dev = (memif_if_t *) dev; \
36 : if (_dev) \
37 : vlib_log(VLIB_LOG_LEVEL_DEBUG, memif_main.log_class, "%U: " f, \
38 : format_vnet_hw_if_index_name, vnet_get_main(), \
39 : _dev->hw_if_index, ##__VA_ARGS__); \
40 : else \
41 : vlib_log(VLIB_LOG_LEVEL_DEBUG, memif_main.log_class, f, \
42 : ##__VA_ARGS__); \
43 : } while (0)
44 :
45 : #define memif_log_warn(dev, f, ...) do { \
46 : memif_if_t *_dev = (memif_if_t *) dev; \
47 : if (_dev) \
48 : vlib_log(VLIB_LOG_LEVEL_WARNING, memif_main.log_class, "%U: " f, \
49 : format_vnet_hw_if_index_name, vnet_get_main(), \
50 : _dev->hw_if_index, ##__VA_ARGS__); \
51 : else \
52 : vlib_log(VLIB_LOG_LEVEL_WARNING, memif_main.log_class, f, \
53 : ##__VA_ARGS__); \
54 : } while (0)
55 :
56 : #define memif_log_err(dev, f, ...) do { \
57 : memif_if_t *_dev = (memif_if_t *) dev; \
58 : if (_dev) \
59 : vlib_log(VLIB_LOG_LEVEL_ERR, memif_main.log_class, "%U: " f, \
60 : format_vnet_hw_if_index_name, vnet_get_main(), \
61 : _dev->hw_if_index, ##__VA_ARGS__); \
62 : else \
63 : vlib_log(VLIB_LOG_LEVEL_ERR, memif_main.log_class, f, \
64 : ##__VA_ARGS__); \
65 : } while (0)
66 :
67 : #define memif_file_add(a, b) \
68 : do \
69 : { \
70 : *a = clib_file_add (&file_main, b); \
71 : memif_log_debug (0, "clib_file_add fd %d private_data %u idx %u", \
72 : (b)->file_descriptor, (b)->private_data, *a); \
73 : } \
74 : while (0)
75 :
76 : #define memif_file_del(a) \
77 : do \
78 : { \
79 : memif_log_debug (0, "clib_file_del idx %u", a - file_main.file_pool); \
80 : clib_file_del (&file_main, a); \
81 : } \
82 : while (0)
83 :
84 : #define memif_file_del_by_index(a) \
85 : do \
86 : { \
87 : memif_log_debug (0, "clib_file_del idx %u", a); \
88 : clib_file_del_by_index (&file_main, a); \
89 : } \
90 : while (0)
91 :
92 : typedef struct
93 : {
94 : u8 *filename;
95 : u32 socket_id;
96 : clib_socket_t *sock;
97 : clib_socket_t **pending_clients;
98 : int ref_cnt;
99 : int is_listener;
100 :
101 : /* hash of all registered id */
102 : mhash_t dev_instance_by_id;
103 :
104 : /* hash of all registered fds */
105 : uword *dev_instance_by_fd;
106 : } memif_socket_file_t;
107 :
108 : typedef struct
109 : {
110 : /* Required for vec_validate_aligned */
111 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
112 : void *shm;
113 : memif_region_size_t region_size;
114 : int fd;
115 : u8 is_external;
116 : } memif_region_t;
117 :
118 : typedef struct
119 : {
120 : memif_msg_t msg;
121 : int fd;
122 : } memif_msg_fifo_elt_t;
123 :
124 : #define MEMIF_RX_VECTOR_SZ VLIB_FRAME_SIZE
125 : #define MEMIF_DMA_INFO_SIZE VLIB_FRAME_SIZE
126 :
127 : struct memif_dma_info;
128 :
129 : typedef struct
130 : {
131 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
132 : clib_spinlock_t lockp;
133 : /* ring data */
134 : memif_ring_t *ring;
135 : memif_log2_ring_size_t log2_ring_size;
136 : memif_region_index_t region;
137 : memif_region_offset_t offset;
138 :
139 : u16 last_head;
140 : u16 last_tail;
141 : u32 *buffers;
142 : u8 buffer_pool_index;
143 :
144 : /* dma data */
145 : u16 dma_head;
146 : u16 dma_tail;
147 : struct memif_dma_info *dma_info;
148 : u16 dma_info_head;
149 : u16 dma_info_tail;
150 : u16 dma_info_size;
151 : u8 dma_info_full;
152 :
153 : /* interrupts */
154 : int int_fd;
155 : uword int_clib_file_index;
156 : u64 int_count;
157 :
158 : /* queue type */
159 : memif_ring_type_t type;
160 : u32 queue_index;
161 : } memif_queue_t;
162 :
163 : #define foreach_memif_if_flag \
164 : _ (0, ADMIN_UP, "admin-up") \
165 : _ (1, IS_SLAVE, "slave") \
166 : _ (2, CONNECTING, "connecting") \
167 : _ (3, CONNECTED, "connected") \
168 : _ (4, DELETING, "deleting") \
169 : _ (5, ZERO_COPY, "zero-copy") \
170 : _ (6, ERROR, "error") \
171 : _ (7, USE_DMA, "use_dma")
172 :
173 : typedef enum
174 : {
175 : #define _(a, b, c) MEMIF_IF_FLAG_##b = (1 << a),
176 : foreach_memif_if_flag
177 : #undef _
178 : } memif_if_flag_t;
179 :
180 : typedef struct
181 : {
182 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
183 : u32 flags;
184 : memif_interface_id_t id;
185 : u32 hw_if_index;
186 : u32 sw_if_index;
187 : uword dev_instance;
188 : memif_interface_mode_t mode:8;
189 :
190 : u32 per_interface_next_index;
191 :
192 : /* socket connection */
193 : clib_socket_t *sock;
194 : uword socket_file_index;
195 : memif_msg_fifo_elt_t *msg_queue;
196 : u8 *secret;
197 :
198 : memif_region_t *regions;
199 :
200 : memif_queue_t *rx_queues;
201 : memif_queue_t *tx_queues;
202 :
203 : /* remote info */
204 : u8 *remote_name;
205 : u8 *remote_if_name;
206 :
207 : struct
208 : {
209 : memif_log2_ring_size_t log2_ring_size;
210 : u8 num_s2m_rings;
211 : u8 num_m2s_rings;
212 : u16 buffer_size;
213 : } cfg;
214 :
215 : struct
216 : {
217 : memif_log2_ring_size_t log2_ring_size;
218 : u8 num_s2m_rings;
219 : u8 num_m2s_rings;
220 : u16 buffer_size;
221 : } run;
222 :
223 : /* disconnect strings */
224 : u8 *local_disc_string;
225 : u8 *remote_disc_string;
226 :
227 : /* dma config index */
228 : int dma_input_config;
229 : int dma_tx_config;
230 : } memif_if_t;
231 :
232 : typedef struct
233 : {
234 : u16 packet_len;
235 : u16 first_buffer_vec_index;
236 : } memif_packet_op_t;
237 :
238 : typedef struct
239 : {
240 : CLIB_ALIGN_MARK (pad, 16); /* align up to 16 bytes for 32bit builds */
241 : void *data;
242 : u32 data_len;
243 : i16 buffer_offset;
244 : u16 buffer_vec_index;
245 : } memif_copy_op_t;
246 :
247 : typedef enum
248 : {
249 : MEMIF_DESC_STATUS_OK = 0,
250 : MEMIF_DESC_STATUS_ERR_BAD_REGION,
251 : MEMIF_DESC_STATUS_ERR_REGION_OVERRUN,
252 : MEMIF_DESC_STATUS_ERR_DATA_TOO_BIG,
253 : MEMIF_DESC_STATUS_ERR_ZERO_LENGTH
254 : } __clib_packed memif_desc_status_err_code_t;
255 :
256 : typedef union
257 : {
258 : struct
259 : {
260 : u8 next : 1;
261 : u8 err : 1;
262 : u8 reserved : 2;
263 : memif_desc_status_err_code_t err_code : 4;
264 : };
265 : u8 as_u8;
266 : } memif_desc_status_t;
267 :
268 : STATIC_ASSERT_SIZEOF (memif_desc_status_t, 1);
269 :
270 : typedef struct
271 : {
272 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
273 : u16 n_packets;
274 : u16 max_desc_len;
275 : u32 n_rx_bytes;
276 : u8 xor_status;
277 : /* copy vector */
278 : memif_copy_op_t *copy_ops;
279 : u32 *buffers;
280 : memif_packet_op_t packet_ops[MEMIF_RX_VECTOR_SZ];
281 :
282 : /* temp storage for compressed descriptors */
283 : void **desc_data;
284 : u16 *desc_len;
285 : memif_desc_status_t *desc_status;
286 :
287 : /* buffer template */
288 : vlib_buffer_t buffer_template;
289 : } memif_per_thread_data_t;
290 :
291 : typedef struct memif_dma_info
292 : {
293 : /* per thread data */
294 : memif_interface_mode_t mode;
295 : vlib_node_runtime_t *node;
296 : u32 dma_head;
297 : u32 dma_tail;
298 : u8 finished;
299 : memif_per_thread_data_t data;
300 : } memif_dma_info_t;
301 :
302 : typedef struct
303 : {
304 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
305 :
306 : /** API message ID base */
307 : u16 msg_id_base;
308 :
309 : /* pool of all memory interfaces */
310 : memif_if_t *interfaces;
311 :
312 : /* pool of all unix socket files */
313 : memif_socket_file_t *socket_files;
314 : uword *socket_file_index_by_sock_id; /* map user socket id to pool idx */
315 :
316 : /* per thread data */
317 : memif_per_thread_data_t *per_thread_data;
318 :
319 : vlib_log_class_t log_class;
320 :
321 : } memif_main_t;
322 :
323 : extern memif_main_t memif_main;
324 : extern vnet_device_class_t memif_device_class;
325 : extern vlib_node_registration_t memif_input_node;
326 :
327 : typedef enum
328 : {
329 : MEMIF_PROCESS_EVENT_START = 1,
330 : MEMIF_PROCESS_EVENT_STOP = 2,
331 : MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN = 3,
332 : } memif_process_event_t;
333 :
334 : typedef struct
335 : {
336 : memif_interface_id_t id;
337 : u32 socket_id;
338 : u8 *secret;
339 : u8 is_master;
340 : u8 is_zero_copy;
341 : u8 use_dma;
342 : memif_interface_mode_t mode:8;
343 : memif_log2_ring_size_t log2_ring_size;
344 : u16 buffer_size;
345 : u8 hw_addr_set;
346 : u8 hw_addr[6];
347 : u8 rx_queues;
348 : u8 tx_queues;
349 :
350 : /* return */
351 : u32 sw_if_index;
352 : } memif_create_if_args_t;
353 :
354 : u32 memif_get_unused_socket_id ();
355 : clib_error_t *memif_socket_filename_add_del (u8 is_add, u32 sock_id,
356 : char *sock_filename);
357 : clib_error_t *memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args);
358 : clib_error_t *memif_delete_if (vlib_main_t *vm, memif_if_t *mif);
359 : clib_error_t *memif_plugin_api_hookup (vlib_main_t * vm);
360 : clib_error_t *memif_interface_admin_up_down (vnet_main_t *vnm, u32 hw_if_index,
361 : u32 flags);
362 :
363 : static_always_inline void *
364 0 : memif_get_buffer (memif_if_t * mif, memif_ring_t * ring, u16 slot)
365 : {
366 0 : u16 region = ring->desc[slot].region;
367 0 : return mif->regions[region].shm + ring->desc[slot].offset;
368 : }
369 :
370 : /* memif.c */
371 : clib_error_t *memif_init_regions_and_queues (memif_if_t * mif);
372 : clib_error_t *memif_connect (memif_if_t * mif);
373 : void memif_disconnect (memif_if_t * mif, clib_error_t * err);
374 :
375 : /* socket.c */
376 : void memif_socket_close (clib_socket_t ** sock);
377 : clib_error_t *memif_conn_fd_accept_ready (clib_file_t * uf);
378 : clib_error_t *memif_master_conn_fd_read_ready (clib_file_t * uf);
379 : clib_error_t *memif_slave_conn_fd_read_ready (clib_file_t * uf);
380 : clib_error_t *memif_master_conn_fd_write_ready (clib_file_t * uf);
381 : clib_error_t *memif_slave_conn_fd_write_ready (clib_file_t * uf);
382 : clib_error_t *memif_master_conn_fd_error (clib_file_t * uf);
383 : clib_error_t *memif_slave_conn_fd_error (clib_file_t * uf);
384 : clib_error_t *memif_msg_send_disconnect (memif_if_t * mif,
385 : clib_error_t * err);
386 : u8 *format_memif_device_name (u8 * s, va_list * args);
387 : void memif_dma_completion_cb (vlib_main_t *vm, vlib_dma_batch_t *b);
388 : void memif_tx_dma_completion_cb (vlib_main_t *vm, vlib_dma_batch_t *b);
389 :
390 : /*
391 : * fd.io coding-style-patch-verification: ON
392 : *
393 : * Local Variables:
394 : * eval: (c-set-style "gnu")
395 : * End:
396 : */
|
