Line data Source code
1 : /*
2 : *------------------------------------------------------------------
3 : * Copyright (c) 2017 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : *------------------------------------------------------------------
16 : */
17 :
18 :
19 : #define _GNU_SOURCE
20 : #include <stdint.h>
21 : #include <net/if.h>
22 : #include <sys/types.h>
23 : #include <fcntl.h>
24 : #include <sys/ioctl.h>
25 : #include <sys/socket.h>
26 : #include <sys/un.h>
27 : #include <sys/uio.h>
28 : #include <sys/mman.h>
29 : #include <sys/prctl.h>
30 : #include <sys/eventfd.h>
31 : #include <inttypes.h>
32 : #include <limits.h>
33 :
34 : #include <vlib/vlib.h>
35 : #include <vlib/unix/unix.h>
36 : #include <vnet/plugin/plugin.h>
37 : #include <vnet/ethernet/ethernet.h>
38 : #include <vnet/interface/rx_queue_funcs.h>
39 : #include <vnet/interface/tx_queue_funcs.h>
40 : #include <vpp/app/version.h>
41 : #include <memif/memif.h>
42 : #include <memif/private.h>
43 :
44 : memif_main_t memif_main;
45 :
46 : static u32
47 6 : memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
48 : {
49 : /* nothing for now */
50 6 : return 0;
51 : }
52 :
53 : static clib_error_t *
54 0 : memif_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi,
55 : u32 flags)
56 : {
57 : /* nothing for now */
58 0 : return 0;
59 : }
60 :
61 : static void
62 16 : memif_queue_intfd_close (memif_queue_t * mq)
63 : {
64 16 : if (mq->int_clib_file_index != ~0)
65 : {
66 8 : memif_file_del_by_index (mq->int_clib_file_index);
67 8 : mq->int_clib_file_index = ~0;
68 8 : mq->int_fd = -1;
69 : }
70 8 : else if (mq->int_fd > -1)
71 : {
72 8 : close (mq->int_fd);
73 8 : mq->int_fd = -1;
74 : }
75 16 : }
76 :
77 : static void
78 8 : memif_disconnect_free_zc_queue_buffer (memif_queue_t * mq, u8 is_rx)
79 : {
80 8 : vlib_main_t *vm = vlib_get_main ();
81 : u16 ring_size, n_slots, mask, start;
82 :
83 8 : ring_size = 1 << mq->log2_ring_size;
84 8 : mask = ring_size - 1;
85 8 : n_slots = mq->ring->head - mq->last_tail;
86 8 : start = mq->last_tail & mask;
87 8 : if (is_rx)
88 4 : vlib_buffer_free_from_ring (vm, mq->buffers, start, ring_size, n_slots);
89 : else
90 4 : vlib_buffer_free_from_ring_no_next (vm, mq->buffers, start, ring_size,
91 : n_slots);
92 8 : vec_free (mq->buffers);
93 8 : }
94 :
95 : void
96 20 : memif_disconnect (memif_if_t * mif, clib_error_t * err)
97 : {
98 20 : memif_main_t *mm = &memif_main;
99 20 : vnet_main_t *vnm = vnet_get_main ();
100 : memif_region_t *mr;
101 : memif_queue_t *mq;
102 : int i;
103 :
104 20 : if (mif == 0)
105 0 : return;
106 :
107 20 : memif_log_debug (mif, "disconnect %u (%v)", mif->dev_instance,
108 : err ? err->what : 0);
109 :
110 20 : if (err)
111 : {
112 20 : clib_error_t *e = 0;
113 20 : mif->local_disc_string = vec_dup (err->what);
114 20 : if (mif->sock && clib_socket_is_connected (mif->sock))
115 8 : e = memif_msg_send_disconnect (mif, err);
116 20 : clib_error_free (e);
117 : }
118 :
119 : /* set interface down */
120 20 : mif->flags &= ~(MEMIF_IF_FLAG_CONNECTED | MEMIF_IF_FLAG_CONNECTING);
121 20 : if (mif->hw_if_index != ~0)
122 20 : vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0);
123 :
124 : /* close connection socket */
125 20 : if (mif->sock && mif->sock->fd)
126 8 : {
127 8 : memif_socket_file_t *msf = vec_elt_at_index (mm->socket_files,
128 : mif->socket_file_index);
129 8 : hash_unset (msf->dev_instance_by_fd, mif->sock->fd);
130 8 : memif_socket_close (&mif->sock);
131 : }
132 12 : else if (mif->sock)
133 : {
134 : clib_error_t *err;
135 0 : err = clib_socket_close (mif->sock);
136 0 : if (err)
137 : {
138 0 : memif_log_err (mif, "%U", format_clib_error, err);
139 0 : clib_error_free (err);
140 : }
141 0 : clib_mem_free (mif->sock);
142 : }
143 :
144 : /* *INDENT-OFF* */
145 28 : vec_foreach_index (i, mif->rx_queues)
146 : {
147 8 : mq = vec_elt_at_index (mif->rx_queues, i);
148 8 : if (mq->ring)
149 : {
150 8 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
151 : {
152 4 : memif_disconnect_free_zc_queue_buffer(mq, 1);
153 : }
154 8 : mq->ring = 0;
155 : }
156 : }
157 20 : vnet_hw_if_unregister_all_rx_queues (vnm, mif->hw_if_index);
158 :
159 : /* *INDENT-OFF* */
160 28 : vec_foreach_index (i, mif->tx_queues)
161 : {
162 8 : mq = vec_elt_at_index (mif->tx_queues, i);
163 8 : if (mq->ring)
164 : {
165 8 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
166 : {
167 4 : memif_disconnect_free_zc_queue_buffer(mq, 0);
168 : }
169 8 : clib_spinlock_free (&mq->lockp);
170 : }
171 8 : mq->ring = 0;
172 : }
173 20 : vnet_hw_if_unregister_all_tx_queues (vnm, mif->hw_if_index);
174 20 : vnet_hw_if_update_runtime_data (vnm, mif->hw_if_index);
175 :
176 : /* free tx and rx queues */
177 28 : vec_foreach (mq, mif->rx_queues)
178 8 : memif_queue_intfd_close (mq);
179 20 : vec_free (mif->rx_queues);
180 :
181 28 : vec_foreach (mq, mif->tx_queues)
182 8 : memif_queue_intfd_close (mq);
183 20 : vec_free (mif->tx_queues);
184 :
185 : /* free memory regions */
186 36 : vec_foreach (mr, mif->regions)
187 : {
188 : int rv;
189 16 : if (mr->is_external)
190 4 : continue;
191 12 : if ((rv = munmap (mr->shm, mr->region_size)))
192 0 : memif_log_err (mif, "munmap failed, rv = %d", rv);
193 12 : if (mr->fd > -1)
194 12 : close (mr->fd);
195 : }
196 : /* *INDENT-ON* */
197 20 : vec_free (mif->regions);
198 20 : vec_free (mif->remote_name);
199 20 : vec_free (mif->remote_if_name);
200 20 : clib_fifo_free (mif->msg_queue);
201 : }
202 :
203 : static clib_error_t *
204 0 : memif_int_fd_write_ready (clib_file_t * uf)
205 : {
206 0 : memif_main_t *mm = &memif_main;
207 0 : u16 qid = uf->private_data & 0xFFFF;
208 0 : memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 16);
209 :
210 0 : memif_log_warn (mif, "unexpected EPOLLOUT on RX for queue %u", qid);
211 0 : return 0;
212 : }
213 :
214 : static clib_error_t *
215 0 : memif_int_fd_read_ready (clib_file_t * uf)
216 : {
217 0 : memif_main_t *mm = &memif_main;
218 0 : vnet_main_t *vnm = vnet_get_main ();
219 0 : u16 qid = uf->private_data & 0xFFFF;
220 0 : memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 16);
221 0 : memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, qid);
222 : u64 b;
223 : ssize_t size;
224 :
225 0 : size = read (uf->file_descriptor, &b, sizeof (b));
226 0 : if (size < 0)
227 : {
228 0 : memif_log_debug (mif, "Failed to read from socket");
229 0 : return 0;
230 : }
231 :
232 0 : vnet_hw_if_rx_queue_set_int_pending (vnm, mq->queue_index);
233 0 : mq->int_count++;
234 :
235 0 : return 0;
236 : }
237 :
238 :
239 : clib_error_t *
240 14 : memif_connect (memif_if_t * mif)
241 : {
242 14 : memif_main_t *mm = &memif_main;
243 14 : vlib_main_t *vm = vlib_get_main ();
244 14 : vnet_main_t *vnm = vnet_get_main ();
245 14 : clib_file_t template = { 0 };
246 : memif_region_t *mr;
247 : int i, j;
248 14 : u32 n_txqs = 0, n_threads = vlib_get_n_threads ();
249 14 : clib_error_t *err = NULL;
250 14 : u8 max_log2_ring_sz = 0;
251 14 : int with_barrier = 0;
252 :
253 14 : memif_log_debug (mif, "connect %u", mif->dev_instance);
254 :
255 14 : vec_free (mif->local_disc_string);
256 14 : vec_free (mif->remote_disc_string);
257 :
258 : /* *INDENT-OFF* */
259 42 : vec_foreach (mr, mif->regions)
260 : {
261 28 : if (mr->shm)
262 14 : continue;
263 :
264 14 : if (mr->fd < 0)
265 : {
266 0 : err = clib_error_return (0, "no memory region fd");
267 0 : goto error;
268 : }
269 :
270 14 : if ((mr->shm = mmap (NULL, mr->region_size, PROT_READ | PROT_WRITE,
271 : MAP_SHARED, mr->fd, 0)) == MAP_FAILED)
272 : {
273 0 : err = clib_error_return_unix (0, "mmap");
274 0 : goto error;
275 : }
276 : }
277 : /* *INDENT-ON* */
278 :
279 14 : template.read_function = memif_int_fd_read_ready;
280 14 : template.write_function = memif_int_fd_write_ready;
281 :
282 14 : with_barrier = 1;
283 14 : if (vlib_worker_thread_barrier_held ())
284 14 : with_barrier = 0;
285 :
286 14 : if (with_barrier)
287 0 : vlib_worker_thread_barrier_sync (vm);
288 :
289 : /* *INDENT-OFF* */
290 28 : vec_foreach_index (i, mif->tx_queues)
291 : {
292 14 : memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i);
293 14 : max_log2_ring_sz = clib_max (max_log2_ring_sz, mq->log2_ring_size);
294 :
295 14 : mq->ring = mif->regions[mq->region].shm + mq->offset;
296 14 : if (mq->ring->cookie != MEMIF_COOKIE)
297 : {
298 0 : err = clib_error_return (0, "wrong cookie on tx ring %u", i);
299 0 : goto error;
300 : }
301 14 : mq->queue_index =
302 14 : vnet_hw_if_register_tx_queue (vnm, mif->hw_if_index, i);
303 14 : clib_spinlock_init (&mq->lockp);
304 :
305 14 : if (mif->flags & MEMIF_IF_FLAG_USE_DMA)
306 : {
307 : memif_dma_info_t *dma_info;
308 0 : mq->dma_head = 0;
309 0 : mq->dma_tail = 0;
310 0 : mq->dma_info_head = 0;
311 0 : mq->dma_info_tail = 0;
312 0 : mq->dma_info_size = MEMIF_DMA_INFO_SIZE;
313 0 : vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE,
314 : CLIB_CACHE_LINE_BYTES);
315 :
316 0 : vec_foreach (dma_info, mq->dma_info)
317 : {
318 0 : vec_validate_aligned (dma_info->data.desc_data,
319 : pow2_mask (max_log2_ring_sz),
320 : CLIB_CACHE_LINE_BYTES);
321 0 : vec_validate_aligned (dma_info->data.desc_len,
322 : pow2_mask (max_log2_ring_sz),
323 : CLIB_CACHE_LINE_BYTES);
324 0 : vec_validate_aligned (dma_info->data.desc_status,
325 : pow2_mask (max_log2_ring_sz),
326 : CLIB_CACHE_LINE_BYTES);
327 0 : vec_validate_aligned (dma_info->data.copy_ops, 0,
328 : CLIB_CACHE_LINE_BYTES);
329 0 : vec_reset_length (dma_info->data.copy_ops);
330 0 : vec_validate_aligned (dma_info->data.buffers, 0,
331 : CLIB_CACHE_LINE_BYTES);
332 0 : vec_reset_length (dma_info->data.buffers);
333 : }
334 : }
335 : }
336 :
337 14 : if (vec_len (mif->tx_queues) > 0)
338 : {
339 14 : n_txqs = vec_len (mif->tx_queues);
340 28 : for (j = 0; j < n_threads; j++)
341 : {
342 14 : u32 qi = mif->tx_queues[j % n_txqs].queue_index;
343 14 : vnet_hw_if_tx_queue_assign_thread (vnm, qi, j);
344 : }
345 : }
346 :
347 28 : vec_foreach_index (i, mif->rx_queues)
348 : {
349 14 : memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i);
350 : u32 ti;
351 : u32 qi;
352 : int rv;
353 :
354 14 : max_log2_ring_sz = clib_max (max_log2_ring_sz, mq->log2_ring_size);
355 :
356 14 : mq->ring = mif->regions[mq->region].shm + mq->offset;
357 14 : if (mq->ring->cookie != MEMIF_COOKIE)
358 : {
359 0 : err = clib_error_return (0, "wrong cookie on tx ring %u", i);
360 0 : goto error;
361 : }
362 14 : qi = vnet_hw_if_register_rx_queue (vnm, mif->hw_if_index, i,
363 : VNET_HW_IF_RXQ_THREAD_ANY);
364 14 : mq->queue_index = qi;
365 :
366 14 : if (mif->flags & MEMIF_IF_FLAG_USE_DMA)
367 : {
368 : memif_dma_info_t *dma_info;
369 0 : mq->dma_head = 0;
370 0 : mq->dma_tail = 0;
371 0 : mq->dma_info_head = 0;
372 0 : mq->dma_info_tail = 0;
373 0 : mq->dma_info_size = MEMIF_DMA_INFO_SIZE;
374 0 : vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE,
375 : CLIB_CACHE_LINE_BYTES);
376 0 : vec_foreach (dma_info, mq->dma_info)
377 : {
378 0 : vec_validate_aligned (dma_info->data.desc_data,
379 : pow2_mask (max_log2_ring_sz),
380 : CLIB_CACHE_LINE_BYTES);
381 0 : vec_validate_aligned (dma_info->data.desc_len,
382 : pow2_mask (max_log2_ring_sz),
383 : CLIB_CACHE_LINE_BYTES);
384 0 : vec_validate_aligned (dma_info->data.desc_status,
385 : pow2_mask (max_log2_ring_sz),
386 : CLIB_CACHE_LINE_BYTES);
387 0 : vec_validate_aligned (dma_info->data.copy_ops, 0,
388 : CLIB_CACHE_LINE_BYTES);
389 0 : vec_reset_length (dma_info->data.copy_ops);
390 0 : vec_validate_aligned (dma_info->data.buffers, 0,
391 : CLIB_CACHE_LINE_BYTES);
392 0 : vec_reset_length (dma_info->data.buffers);
393 : }
394 : }
395 :
396 14 : if (mq->int_fd > -1)
397 : {
398 14 : template.file_descriptor = mq->int_fd;
399 14 : template.private_data = (mif->dev_instance << 16) | (i & 0xFFFF);
400 14 : template.description = format (0, "%U rx %u int",
401 : format_memif_device_name,
402 : mif->dev_instance, i);
403 14 : memif_file_add (&mq->int_clib_file_index, &template);
404 14 : vnet_hw_if_set_rx_queue_file_index (vnm, qi,
405 14 : mq->int_clib_file_index);
406 : }
407 14 : ti = vnet_hw_if_get_rx_queue_thread_index (vnm, qi);
408 14 : mq->buffer_pool_index = vlib_buffer_pool_get_default_for_numa (
409 14 : vm, vlib_get_main_by_index (ti)->numa_node);
410 14 : rv = vnet_hw_if_set_rx_queue_mode (vnm, qi, VNET_HW_IF_RX_MODE_DEFAULT);
411 14 : vnet_hw_if_update_runtime_data (vnm, mif->hw_if_index);
412 :
413 14 : if (rv)
414 0 : memif_log_err
415 : (mif, "Warning: unable to set rx mode for interface %d queue %d: "
416 : "rc=%d", mif->hw_if_index, i, rv);
417 : else
418 : {
419 14 : vnet_hw_if_rx_mode rxmode = vnet_hw_if_get_rx_queue_mode (vnm, qi);
420 :
421 14 : if (rxmode == VNET_HW_IF_RX_MODE_POLLING)
422 14 : mq->ring->flags |= MEMIF_RING_FLAG_MASK_INT;
423 : else
424 0 : vnet_hw_if_rx_queue_set_int_pending (vnm, qi);
425 : }
426 : }
427 : /* *INDENT-ON* */
428 :
429 14 : if (1 << max_log2_ring_sz > vec_len (mm->per_thread_data[0].desc_data))
430 : {
431 : memif_per_thread_data_t *ptd;
432 :
433 8 : vec_foreach (ptd, mm->per_thread_data)
434 : {
435 4 : vec_validate_aligned (ptd->desc_data, pow2_mask (max_log2_ring_sz),
436 : CLIB_CACHE_LINE_BYTES);
437 4 : vec_validate_aligned (ptd->desc_len, pow2_mask (max_log2_ring_sz),
438 : CLIB_CACHE_LINE_BYTES);
439 4 : vec_validate_aligned (ptd->desc_status, pow2_mask (max_log2_ring_sz),
440 : CLIB_CACHE_LINE_BYTES);
441 : }
442 : }
443 14 : if (with_barrier)
444 0 : vlib_worker_thread_barrier_release (vm);
445 :
446 14 : mif->flags &= ~MEMIF_IF_FLAG_CONNECTING;
447 14 : mif->flags |= MEMIF_IF_FLAG_CONNECTED;
448 :
449 14 : vnet_hw_interface_set_flags (vnm, mif->hw_if_index,
450 : VNET_HW_INTERFACE_FLAG_LINK_UP);
451 14 : return 0;
452 :
453 0 : error:
454 0 : if (with_barrier)
455 0 : vlib_worker_thread_barrier_release (vm);
456 0 : memif_log_err (mif, "%U", format_clib_error, err);
457 0 : return err;
458 : }
459 :
460 : static_always_inline memif_ring_t *
461 28 : memif_get_ring (memif_if_t * mif, memif_ring_type_t type, u16 ring_num)
462 : {
463 28 : if (vec_len (mif->regions) == 0)
464 0 : return NULL;
465 28 : void *p = mif->regions[0].shm;
466 28 : int ring_size =
467 28 : sizeof (memif_ring_t) +
468 28 : sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size);
469 28 : p += (ring_num + type * mif->run.num_s2m_rings) * ring_size;
470 :
471 28 : return (memif_ring_t *) p;
472 : }
473 :
474 : clib_error_t *
475 7 : memif_init_regions_and_queues (memif_if_t * mif)
476 : {
477 7 : vlib_main_t *vm = vlib_get_main ();
478 : memif_socket_file_t *msf;
479 7 : memif_ring_t *ring = NULL;
480 : int fd, i, j;
481 : u64 buffer_offset;
482 : memif_region_t *r;
483 : clib_error_t *err;
484 :
485 7 : ASSERT (vec_len (mif->regions) == 0);
486 7 : vec_add2_aligned (mif->regions, r, 1, CLIB_CACHE_LINE_BYTES);
487 :
488 7 : buffer_offset = (mif->run.num_s2m_rings + mif->run.num_m2s_rings) *
489 7 : (sizeof (memif_ring_t) +
490 7 : sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size));
491 :
492 7 : r->region_size = buffer_offset;
493 :
494 7 : if ((mif->flags & MEMIF_IF_FLAG_ZERO_COPY) == 0)
495 0 : r->region_size += mif->run.buffer_size * (1 << mif->run.log2_ring_size) *
496 0 : (mif->run.num_s2m_rings + mif->run.num_m2s_rings);
497 :
498 7 : if ((fd = clib_mem_vm_create_fd (CLIB_MEM_PAGE_SZ_DEFAULT, "%U region 0",
499 : format_memif_device_name,
500 : mif->dev_instance)) == -1)
501 : {
502 0 : err = clib_mem_get_last_error ();
503 0 : goto error;
504 : }
505 :
506 7 : if ((ftruncate (fd, r->region_size)) == -1)
507 : {
508 0 : err = clib_error_return_unix (0, "ftruncate");
509 0 : goto error;
510 : }
511 :
512 7 : msf = pool_elt_at_index (memif_main.socket_files, mif->socket_file_index);
513 7 : r->shm = clib_mem_vm_map_shared (0, r->region_size, fd, 0, "memif%lu/%lu:0",
514 : msf->socket_id, mif->id);
515 :
516 7 : if (r->shm == CLIB_MEM_VM_MAP_FAILED)
517 : {
518 0 : err = clib_error_return_unix (0, "memif shared region map failed");
519 0 : goto error;
520 : }
521 :
522 7 : r->fd = fd;
523 :
524 7 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
525 : {
526 : vlib_buffer_pool_t *bp;
527 : /* *INDENT-OFF* */
528 14 : vec_foreach (bp, vm->buffer_main->buffer_pools)
529 : {
530 : vlib_physmem_map_t *pm;
531 7 : pm = vlib_physmem_get_map (vm, bp->physmem_map_index);
532 7 : vec_add2_aligned (mif->regions, r, 1, CLIB_CACHE_LINE_BYTES);
533 7 : r->fd = pm->fd;
534 7 : r->region_size = pm->n_pages << pm->log2_page_size;
535 7 : r->shm = pm->base;
536 7 : r->is_external = 1;
537 : }
538 : /* *INDENT-ON* */
539 : }
540 :
541 14 : for (i = 0; i < mif->run.num_s2m_rings; i++)
542 : {
543 7 : ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
544 7 : ring->head = ring->tail = 0;
545 7 : ring->cookie = MEMIF_COOKIE;
546 :
547 7 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
548 7 : continue;
549 :
550 0 : for (j = 0; j < (1 << mif->run.log2_ring_size); j++)
551 : {
552 0 : u16 slot = i * (1 << mif->run.log2_ring_size) + j;
553 0 : ring->desc[j].region = 0;
554 0 : ring->desc[j].offset =
555 0 : buffer_offset + (u32) (slot * mif->run.buffer_size);
556 0 : ring->desc[j].length = mif->run.buffer_size;
557 : }
558 : }
559 14 : for (i = 0; i < mif->run.num_m2s_rings; i++)
560 : {
561 7 : ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
562 7 : ring->head = ring->tail = 0;
563 7 : ring->cookie = MEMIF_COOKIE;
564 :
565 7 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
566 7 : continue;
567 :
568 0 : for (j = 0; j < (1 << mif->run.log2_ring_size); j++)
569 : {
570 0 : u16 slot =
571 0 : (i + mif->run.num_s2m_rings) * (1 << mif->run.log2_ring_size) + j;
572 0 : ring->desc[j].region = 0;
573 0 : ring->desc[j].offset =
574 0 : buffer_offset + (u32) (slot * mif->run.buffer_size);
575 0 : ring->desc[j].length = mif->run.buffer_size;
576 : }
577 : }
578 :
579 7 : ASSERT (mif->tx_queues == 0);
580 7 : vec_validate_aligned (mif->tx_queues, mif->run.num_s2m_rings - 1,
581 : CLIB_CACHE_LINE_BYTES);
582 :
583 : /* *INDENT-OFF* */
584 14 : vec_foreach_index (i, mif->tx_queues)
585 : {
586 7 : memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i);
587 7 : if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
588 : {
589 0 : err = clib_error_return_unix (0, "eventfd[tx queue %u]", i);
590 0 : goto error;
591 : }
592 :
593 7 : mq->int_clib_file_index = ~0;
594 7 : mq->ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
595 7 : mq->log2_ring_size = mif->cfg.log2_ring_size;
596 7 : mq->region = 0;
597 7 : mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm;
598 7 : mq->last_head = 0;
599 7 : mq->type = MEMIF_RING_S2M;
600 7 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
601 7 : vec_validate_aligned (mq->buffers, 1 << mq->log2_ring_size,
602 : CLIB_CACHE_LINE_BYTES);
603 : }
604 : /* *INDENT-ON* */
605 :
606 7 : ASSERT (mif->rx_queues == 0);
607 7 : vec_validate_aligned (mif->rx_queues, mif->run.num_m2s_rings - 1,
608 : CLIB_CACHE_LINE_BYTES);
609 :
610 : /* *INDENT-OFF* */
611 14 : vec_foreach_index (i, mif->rx_queues)
612 : {
613 7 : memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i);
614 7 : if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
615 : {
616 0 : err = clib_error_return_unix (0, "eventfd[rx queue %u]", i);
617 0 : goto error;
618 : }
619 7 : mq->int_clib_file_index = ~0;
620 7 : mq->ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
621 7 : mq->log2_ring_size = mif->cfg.log2_ring_size;
622 7 : mq->region = 0;
623 7 : mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm;
624 7 : mq->last_head = 0;
625 7 : mq->type = MEMIF_RING_M2S;
626 7 : if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
627 7 : vec_validate_aligned (mq->buffers, 1 << mq->log2_ring_size,
628 : CLIB_CACHE_LINE_BYTES);
629 : }
630 : /* *INDENT-ON* */
631 :
632 7 : return 0;
633 :
634 0 : error:
635 0 : memif_log_err (mif, "%U", format_clib_error, err);
636 0 : return err;
637 : }
638 :
639 : static uword
640 559 : memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
641 : {
642 559 : memif_main_t *mm = &memif_main;
643 : memif_if_t *mif;
644 : clib_socket_t *sock;
645 559 : uword *event_data = 0, event_type;
646 559 : u8 enabled = 0;
647 559 : f64 start_time, last_run_duration = 0, now;
648 : clib_error_t *err;
649 :
650 559 : sock = clib_mem_alloc (sizeof (clib_socket_t));
651 559 : clib_memset (sock, 0, sizeof (clib_socket_t));
652 :
653 : while (1)
654 : {
655 622 : if (enabled)
656 47 : vlib_process_wait_for_event_or_clock (vm, (f64) 3 -
657 : last_run_duration);
658 : else
659 575 : vlib_process_wait_for_event (vm);
660 :
661 63 : event_type = vlib_process_get_events (vm, &event_data);
662 63 : vec_reset_length (event_data);
663 :
664 63 : switch (event_type)
665 : {
666 3 : case ~0:
667 3 : break;
668 18 : case MEMIF_PROCESS_EVENT_START:
669 18 : enabled = 1;
670 18 : break;
671 16 : case MEMIF_PROCESS_EVENT_STOP:
672 16 : enabled = 0;
673 16 : continue;
674 26 : case MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN:
675 26 : break;
676 0 : default:
677 0 : ASSERT (0);
678 : }
679 :
680 47 : last_run_duration = start_time = vlib_time_now (vm);
681 : /* *INDENT-OFF* */
682 96 : pool_foreach (mif, mm->interfaces)
683 : {
684 49 : memif_socket_file_t * msf = vec_elt_at_index (mm->socket_files, mif->socket_file_index);
685 : /* Allow no more than 10us without a pause */
686 49 : now = vlib_time_now (vm);
687 49 : if (now > start_time + 10e-6)
688 : {
689 0 : vlib_process_suspend (vm, 100e-6); /* suspend for 100 us */
690 0 : start_time = vlib_time_now (vm);
691 : }
692 :
693 49 : if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) == 0)
694 20 : continue;
695 :
696 29 : if (mif->flags & MEMIF_IF_FLAG_CONNECTING)
697 0 : continue;
698 :
699 29 : if (mif->flags & MEMIF_IF_FLAG_CONNECTED)
700 13 : continue;
701 :
702 16 : if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
703 : {
704 8 : clib_memset (sock, 0, sizeof(clib_socket_t));
705 8 : sock->config = (char *) msf->filename;
706 8 : sock->is_seqpacket = 1;
707 8 : sock->is_blocking = 1;
708 :
709 8 : if ((err = clib_socket_init (sock)))
710 : {
711 1 : clib_error_free (err);
712 : }
713 : else
714 : {
715 7 : clib_file_t t = { 0 };
716 :
717 7 : t.read_function = memif_slave_conn_fd_read_ready;
718 7 : t.write_function = memif_slave_conn_fd_write_ready;
719 7 : t.error_function = memif_slave_conn_fd_error;
720 7 : t.file_descriptor = sock->fd;
721 7 : t.private_data = mif->dev_instance;
722 7 : memif_file_add (&sock->private_data, &t);
723 7 : t.description = format (0, "%U ctl",
724 : format_memif_device_name,
725 : mif->dev_instance);
726 7 : hash_set (msf->dev_instance_by_fd, sock->fd, mif->dev_instance);
727 :
728 7 : mif->flags |= MEMIF_IF_FLAG_CONNECTING;
729 7 : mif->sock = sock;
730 7 : sock = clib_mem_alloc (sizeof(clib_socket_t));
731 : }
732 : }
733 : }
734 : /* *INDENT-ON* */
735 47 : last_run_duration = vlib_time_now (vm) - last_run_duration;
736 : }
737 : return 0;
738 : }
739 :
740 76184 : VLIB_REGISTER_NODE (memif_process_node,static) = {
741 : .function = memif_process,
742 : .type = VLIB_NODE_TYPE_PROCESS,
743 : .name = "memif-process",
744 : };
745 :
746 : /*
747 : * Returns an unused socket id, and ~0 if it can't find one.
748 : */
749 : u32
750 0 : memif_get_unused_socket_id ()
751 : {
752 0 : memif_main_t *mm = &memif_main;
753 : uword *p;
754 : int i, j;
755 :
756 : static u32 seed = 0;
757 : /* limit to 1M tries */
758 0 : for (j = 0; j < 1 << 10; j++)
759 : {
760 0 : seed = random_u32 (&seed);
761 0 : for (i = 0; i < 1 << 10; i++)
762 : {
763 : /* look around randomly generated id */
764 0 : seed += (2 * (i % 2) - 1) * i;
765 0 : if (seed == (u32) ~0)
766 0 : continue;
767 0 : p = hash_get (mm->socket_file_index_by_sock_id, seed);
768 0 : if (!p)
769 0 : return seed;
770 : }
771 : }
772 :
773 0 : return ~0;
774 : }
775 :
776 : clib_error_t *
777 583 : memif_socket_filename_add_del (u8 is_add, u32 sock_id, char *sock_filename)
778 : {
779 583 : memif_main_t *mm = &memif_main;
780 : uword *p;
781 : memif_socket_file_t *msf;
782 583 : clib_error_t *err = 0;
783 583 : char *dir = 0, *tmp;
784 583 : u32 idx = 0;
785 583 : u8 *name = 0;
786 :
787 : /* allow adding socket id 0 */
788 583 : if (sock_id == 0 && is_add == 0)
789 0 : return vnet_error (VNET_ERR_INVALID_ARGUMENT, "cannot delete socket id 0");
790 :
791 583 : if (sock_id == ~0)
792 0 : return vnet_error (VNET_ERR_INVALID_ARGUMENT,
793 : "socked id is not specified");
794 :
795 583 : if (is_add == 0)
796 : {
797 9 : p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
798 9 : if (!p)
799 : /* Don't delete non-existent entries. */
800 0 : return vnet_error (VNET_ERR_INVALID_ARGUMENT,
801 : "socket file with id %u does not exist", sock_id);
802 :
803 9 : msf = pool_elt_at_index (mm->socket_files, *p);
804 9 : if (msf->ref_cnt > 0)
805 0 : return vnet_error (VNET_ERR_UNEXPECTED_INTF_STATE,
806 0 : "socket file '%s' is in use", msf->filename);
807 :
808 9 : vec_free (msf->filename);
809 9 : pool_put (mm->socket_files, msf);
810 :
811 9 : hash_unset (mm->socket_file_index_by_sock_id, sock_id);
812 :
813 9 : return 0;
814 : }
815 :
816 574 : if (sock_filename == 0 || sock_filename[0] == 0)
817 0 : return vnet_error (VNET_ERR_INVALID_ARGUMENT,
818 : "socket filename not specified");
819 :
820 574 : if (clib_socket_prefix_is_valid (sock_filename))
821 : {
822 0 : name = format (0, "%s%c", sock_filename, 0);
823 : }
824 574 : else if (sock_filename[0] == '/')
825 : {
826 11 : name = format (0, "%s%c", sock_filename, 0);
827 : }
828 : else
829 : {
830 : /* copy runtime dir path */
831 563 : vec_add (dir, vlib_unix_get_runtime_dir (),
832 : strlen (vlib_unix_get_runtime_dir ()));
833 563 : vec_add1 (dir, '/');
834 :
835 : /* if sock_filename contains dirs, add them to path */
836 563 : tmp = strrchr (sock_filename, '/');
837 563 : if (tmp)
838 : {
839 2 : idx = tmp - sock_filename;
840 2 : vec_add (dir, sock_filename, idx);
841 : }
842 :
843 563 : vec_add1 (dir, '\0');
844 : /* create socket dir */
845 563 : if ((err = vlib_unix_recursive_mkdir (dir)))
846 : {
847 0 : clib_error_free (err);
848 0 : err = vnet_error (VNET_ERR_SYSCALL_ERROR_1,
849 : "unable to create socket dir");
850 0 : goto done;
851 : }
852 :
853 563 : name =
854 563 : format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), sock_filename, 0);
855 : }
856 :
857 574 : p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
858 574 : if (p)
859 : {
860 0 : msf = pool_elt_at_index (mm->socket_files, *p);
861 0 : if (strcmp ((char *) msf->filename, (char *) name) == 0)
862 : {
863 : /* Silently accept identical "add". */
864 0 : goto done;
865 : }
866 :
867 : /* But don't allow a direct add of a different filename. */
868 0 : err = vnet_error (VNET_ERR_ENTRY_ALREADY_EXISTS, "entry already exists");
869 0 : goto done;
870 : }
871 :
872 574 : pool_get (mm->socket_files, msf);
873 574 : clib_memset (msf, 0, sizeof (memif_socket_file_t));
874 :
875 574 : msf->filename = name;
876 574 : msf->socket_id = sock_id;
877 574 : name = 0;
878 :
879 574 : hash_set (mm->socket_file_index_by_sock_id, sock_id, msf - mm->socket_files);
880 :
881 574 : done:
882 574 : vec_free (name);
883 574 : vec_free (dir);
884 574 : return err;
885 : }
886 :
887 : clib_error_t *
888 16 : memif_delete_if (vlib_main_t *vm, memif_if_t *mif)
889 : {
890 16 : vnet_main_t *vnm = vnet_get_main ();
891 16 : memif_main_t *mm = &memif_main;
892 16 : memif_socket_file_t *msf =
893 16 : vec_elt_at_index (mm->socket_files, mif->socket_file_index);
894 : clib_error_t *err;
895 :
896 16 : mif->flags |= MEMIF_IF_FLAG_DELETING;
897 16 : vec_free (mif->local_disc_string);
898 16 : vec_free (mif->remote_disc_string);
899 :
900 : /* bring down the interface */
901 16 : vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0);
902 16 : vnet_sw_interface_set_flags (vnm, mif->sw_if_index, 0);
903 :
904 16 : err = clib_error_return (0, "interface deleted");
905 16 : memif_disconnect (mif, err);
906 16 : clib_error_free (err);
907 :
908 16 : if (mif->hw_if_index != ~0)
909 : {
910 : /* remove the interface */
911 16 : if (mif->mode == MEMIF_INTERFACE_MODE_IP)
912 0 : vnet_delete_hw_interface (vnm, mif->hw_if_index);
913 : else
914 16 : ethernet_delete_interface (vnm, mif->hw_if_index);
915 16 : mif->hw_if_index = ~0;
916 : }
917 :
918 : /* free interface data structures */
919 16 : mhash_unset (&msf->dev_instance_by_id, &mif->id, 0);
920 :
921 : /* remove socket file */
922 16 : if (--(msf->ref_cnt) == 0)
923 : {
924 16 : if (msf->is_listener)
925 : {
926 : int i;
927 : /* *INDENT-OFF* */
928 8 : vec_foreach_index (i, msf->pending_clients)
929 0 : memif_socket_close (msf->pending_clients + i);
930 : /* *INDENT-ON* */
931 8 : memif_socket_close (&msf->sock);
932 8 : vec_free (msf->pending_clients);
933 : }
934 16 : mhash_free (&msf->dev_instance_by_id);
935 16 : hash_free (msf->dev_instance_by_fd);
936 16 : if (msf->sock)
937 : {
938 0 : err = clib_socket_close (msf->sock);
939 0 : if (err)
940 : {
941 0 : memif_log_err (mif, "%U", format_clib_error, err);
942 0 : clib_error_free (err);
943 : }
944 0 : clib_mem_free (msf->sock);
945 : }
946 : }
947 :
948 16 : vec_free (mif->local_disc_string);
949 16 : clib_memset (mif, 0, sizeof (*mif));
950 16 : pool_put (mm->interfaces, mif);
951 :
952 16 : if (pool_elts (mm->interfaces) == 0)
953 16 : vlib_process_signal_event (vm, memif_process_node.index,
954 : MEMIF_PROCESS_EVENT_STOP, 0);
955 :
956 16 : return 0;
957 : }
958 :
959 : /* *INDENT-OFF* */
960 2799 : VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = {
961 : .name = "memif-ip",
962 : .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
963 : .tx_hash_fn_type = VNET_HASH_FN_TYPE_IP,
964 : };
965 : /* *INDENT-ON* */
966 :
967 : static void
968 0 : memif_prepare_dma_args (vlib_dma_config_t *args)
969 : {
970 0 : args->max_batches = 256;
971 0 : args->max_transfer_size = VLIB_BUFFER_DEFAULT_DATA_SIZE;
972 0 : args->barrier_before_last = 1;
973 0 : args->sw_fallback = 1;
974 0 : args->callback_fn = NULL;
975 0 : }
976 :
977 : clib_error_t *
978 22 : memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args)
979 : {
980 22 : memif_main_t *mm = &memif_main;
981 22 : vlib_thread_main_t *tm = vlib_get_thread_main ();
982 22 : vnet_main_t *vnm = vnet_get_main ();
983 22 : vnet_eth_interface_registration_t eir = {};
984 22 : memif_if_t *mif = 0;
985 : vnet_sw_interface_t *sw;
986 : uword *p;
987 22 : memif_socket_file_t *msf = 0;
988 22 : clib_error_t *err = 0;
989 :
990 22 : p = hash_get (mm->socket_file_index_by_sock_id, args->socket_id);
991 22 : if (p == 0)
992 : {
993 0 : err = vnet_error (VNET_ERR_INVALID_ARGUMENT, "unknown socket id");
994 0 : goto done;
995 : }
996 :
997 22 : msf = vec_elt_at_index (mm->socket_files, p[0]);
998 :
999 : /* existing socket file can be either master or slave but cannot be both */
1000 22 : if (msf->ref_cnt > 0)
1001 : {
1002 0 : if ((!msf->is_listener != !args->is_master))
1003 : {
1004 : err =
1005 0 : vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS,
1006 : "socket file cannot be used by both master and slave");
1007 0 : goto done;
1008 : }
1009 :
1010 0 : p = mhash_get (&msf->dev_instance_by_id, &args->id);
1011 0 : if (p)
1012 : {
1013 0 : err = vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS,
1014 : "interface already exists");
1015 0 : goto done;
1016 : }
1017 : }
1018 :
1019 : /* Create new socket file */
1020 22 : if (msf->ref_cnt == 0)
1021 : {
1022 22 : mhash_init (&msf->dev_instance_by_id, sizeof (uword),
1023 : sizeof (memif_interface_id_t));
1024 22 : msf->dev_instance_by_fd = hash_create (0, sizeof (uword));
1025 22 : msf->is_listener = (args->is_master != 0);
1026 :
1027 22 : memif_log_debug (0, "initializing socket file %s", msf->filename);
1028 : }
1029 :
1030 22 : if (mm->per_thread_data == 0)
1031 : {
1032 : int i;
1033 :
1034 4 : vec_validate_aligned (mm->per_thread_data, tm->n_vlib_mains - 1,
1035 : CLIB_CACHE_LINE_BYTES);
1036 :
1037 8 : for (i = 0; i < tm->n_vlib_mains; i++)
1038 : {
1039 4 : memif_per_thread_data_t *ptd =
1040 4 : vec_elt_at_index (mm->per_thread_data, i);
1041 4 : vlib_buffer_t *bt = &ptd->buffer_template;
1042 4 : clib_memset (bt, 0, sizeof (vlib_buffer_t));
1043 4 : bt->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
1044 4 : bt->total_length_not_including_first_buffer = 0;
1045 4 : vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1046 :
1047 4 : vec_validate_aligned (ptd->copy_ops, 0, CLIB_CACHE_LINE_BYTES);
1048 4 : vec_reset_length (ptd->copy_ops);
1049 4 : vec_validate_aligned (ptd->buffers, 0, CLIB_CACHE_LINE_BYTES);
1050 4 : vec_reset_length (ptd->buffers);
1051 : }
1052 : }
1053 :
1054 22 : pool_get (mm->interfaces, mif);
1055 22 : clib_memset (mif, 0, sizeof (*mif));
1056 22 : mif->dev_instance = mif - mm->interfaces;
1057 22 : mif->socket_file_index = msf - mm->socket_files;
1058 22 : mif->id = args->id;
1059 22 : mif->sw_if_index = mif->hw_if_index = mif->per_interface_next_index = ~0;
1060 22 : mif->mode = args->mode;
1061 22 : if (args->secret)
1062 6 : mif->secret = vec_dup (args->secret);
1063 :
1064 : /* register dma config if enabled */
1065 22 : if (args->use_dma)
1066 : {
1067 : vlib_dma_config_t dma_args;
1068 0 : bzero (&dma_args, sizeof (dma_args));
1069 0 : memif_prepare_dma_args (&dma_args);
1070 :
1071 0 : dma_args.max_transfers = 1 << args->log2_ring_size;
1072 0 : dma_args.callback_fn = memif_dma_completion_cb;
1073 0 : mif->dma_input_config = vlib_dma_config_add (vm, &dma_args);
1074 0 : dma_args.callback_fn = memif_tx_dma_completion_cb;
1075 0 : mif->dma_tx_config = vlib_dma_config_add (vm, &dma_args);
1076 : }
1077 :
1078 22 : if (mif->mode == MEMIF_INTERFACE_MODE_ETHERNET)
1079 : {
1080 :
1081 22 : if (!args->hw_addr_set)
1082 : {
1083 22 : f64 now = vlib_time_now (vm);
1084 : u32 rnd;
1085 22 : rnd = (u32) (now * 1e6);
1086 22 : rnd = random_u32 (&rnd);
1087 :
1088 22 : memcpy (args->hw_addr + 2, &rnd, sizeof (rnd));
1089 22 : args->hw_addr[0] = 2;
1090 22 : args->hw_addr[1] = 0xfe;
1091 : }
1092 :
1093 22 : eir.dev_class_index = memif_device_class.index;
1094 22 : eir.dev_instance = mif->dev_instance;
1095 22 : eir.address = args->hw_addr;
1096 22 : eir.cb.flag_change = memif_eth_flag_change;
1097 22 : eir.cb.set_max_frame_size = memif_eth_set_max_frame_size;
1098 22 : mif->hw_if_index = vnet_eth_register_interface (vnm, &eir);
1099 : }
1100 0 : else if (mif->mode == MEMIF_INTERFACE_MODE_IP)
1101 : {
1102 0 : mif->hw_if_index =
1103 0 : vnet_register_interface (vnm, memif_device_class.index,
1104 0 : mif->dev_instance,
1105 : memif_ip_hw_if_class.index,
1106 0 : mif->dev_instance);
1107 : }
1108 : else
1109 : {
1110 : err =
1111 0 : vnet_error (VNET_ERR_SYSCALL_ERROR_2, "unsupported interface mode");
1112 0 : goto error;
1113 : }
1114 :
1115 22 : sw = vnet_get_hw_sw_interface (vnm, mif->hw_if_index);
1116 22 : mif->sw_if_index = sw->sw_if_index;
1117 :
1118 22 : mif->cfg.log2_ring_size = args->log2_ring_size;
1119 22 : mif->cfg.buffer_size = args->buffer_size;
1120 22 : mif->cfg.num_s2m_rings =
1121 22 : args->is_master ? args->rx_queues : args->tx_queues;
1122 22 : mif->cfg.num_m2s_rings =
1123 22 : args->is_master ? args->tx_queues : args->rx_queues;
1124 :
1125 22 : args->sw_if_index = mif->sw_if_index;
1126 :
1127 : /* If this is new one, start listening */
1128 22 : if (msf->is_listener && msf->ref_cnt == 0)
1129 : {
1130 11 : clib_socket_t *s = clib_mem_alloc (sizeof (clib_socket_t));
1131 :
1132 11 : ASSERT (msf->sock == 0);
1133 11 : msf->sock = s;
1134 :
1135 11 : clib_memset (s, 0, sizeof (clib_socket_t));
1136 11 : s->config = (char *) msf->filename;
1137 11 : s->local_only = 1;
1138 11 : s->is_server = 1;
1139 11 : s->allow_group_write = 1;
1140 11 : s->is_seqpacket = 1;
1141 11 : s->passcred = 1;
1142 :
1143 11 : if ((err = clib_socket_init (s)))
1144 : {
1145 0 : err->code = VNET_ERR_SYSCALL_ERROR_4;
1146 0 : goto error;
1147 : }
1148 :
1149 11 : clib_file_t template = { 0 };
1150 11 : template.read_function = memif_conn_fd_accept_ready;
1151 11 : template.file_descriptor = msf->sock->fd;
1152 11 : template.private_data = mif->socket_file_index;
1153 11 : template.description = format (0, "memif listener %s", msf->filename);
1154 11 : memif_file_add (&msf->sock->private_data, &template);
1155 : }
1156 :
1157 22 : msf->ref_cnt++;
1158 :
1159 22 : if (args->is_master == 0)
1160 : {
1161 11 : mif->flags |= MEMIF_IF_FLAG_IS_SLAVE;
1162 11 : if (args->is_zero_copy)
1163 11 : mif->flags |= MEMIF_IF_FLAG_ZERO_COPY;
1164 : }
1165 :
1166 22 : if (args->use_dma)
1167 0 : mif->flags |= MEMIF_IF_FLAG_USE_DMA;
1168 :
1169 22 : vnet_hw_if_set_caps (vnm, mif->hw_if_index, VNET_HW_IF_CAP_INT_MODE);
1170 22 : vnet_hw_if_set_input_node (vnm, mif->hw_if_index, memif_input_node.index);
1171 22 : mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0);
1172 :
1173 22 : if (pool_elts (mm->interfaces) == 1)
1174 : {
1175 18 : vlib_process_signal_event (vm, memif_process_node.index,
1176 : MEMIF_PROCESS_EVENT_START, 0);
1177 : }
1178 22 : goto done;
1179 :
1180 0 : error:
1181 0 : memif_delete_if (vm, mif);
1182 0 : if (err)
1183 0 : memif_log_err (mif, "%U", format_clib_error, err);
1184 0 : return err;
1185 :
1186 22 : done:
1187 22 : return err;
1188 : }
1189 :
1190 : clib_error_t *
1191 26 : memif_interface_admin_up_down (vnet_main_t *vnm, u32 hw_if_index, u32 flags)
1192 : {
1193 26 : memif_main_t *mm = &memif_main;
1194 26 : vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1195 26 : memif_if_t *mif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
1196 : static clib_error_t *error = 0;
1197 :
1198 26 : if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
1199 : {
1200 16 : if (mif->flags & MEMIF_IF_FLAG_CONNECTED)
1201 : {
1202 4 : vnet_hw_interface_set_flags (vnm, mif->hw_if_index,
1203 : VNET_HW_INTERFACE_FLAG_LINK_UP);
1204 : }
1205 16 : mif->flags |= MEMIF_IF_FLAG_ADMIN_UP;
1206 : }
1207 : else
1208 10 : mif->flags &= ~MEMIF_IF_FLAG_ADMIN_UP;
1209 :
1210 26 : vlib_process_signal_event (vnm->vlib_main, memif_process_node.index,
1211 : MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN, 0);
1212 26 : return error;
1213 : }
1214 :
1215 : static clib_error_t *
1216 559 : memif_init (vlib_main_t * vm)
1217 : {
1218 559 : memif_main_t *mm = &memif_main;
1219 :
1220 559 : clib_memset (mm, 0, sizeof (memif_main_t));
1221 :
1222 559 : mm->log_class = vlib_log_register_class ("memif_plugin", 0);
1223 559 : memif_log_debug (0, "initialized");
1224 :
1225 : /* initialize binary API */
1226 559 : memif_plugin_api_hookup (vm);
1227 :
1228 : /*
1229 : * Pre-stuff socket filename pool with a non-modifieable mapping
1230 : * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the
1231 : * default run-time directory.
1232 : */
1233 559 : return memif_socket_filename_add_del (1, 0, MEMIF_DEFAULT_SOCKET_FILENAME);
1234 : }
1235 :
1236 1119 : VLIB_INIT_FUNCTION (memif_init);
1237 :
1238 : /* *INDENT-OFF* */
1239 : VLIB_PLUGIN_REGISTER () = {
1240 : .version = VPP_BUILD_VER,
1241 : .description = "Packet Memory Interface (memif) -- Experimental",
1242 : };
1243 : /* *INDENT-ON* */
1244 :
1245 : /*
1246 : * fd.io coding-style-patch-verification: ON
1247 : *
1248 : * Local Variables:
1249 : * eval: (c-set-style "gnu")
1250 : * End:
1251 : */
|
