Line data Source code
1 : /*
2 : * Copyright (c) 2018 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #include <fcntl.h>
17 : #include <sys/ioctl.h>
18 :
19 : #include <vppinfra/types.h>
20 : #include <vlib/vlib.h>
21 : #include <vlib/pci/pci.h>
22 : #include <vnet/ethernet/ethernet.h>
23 : #include <vnet/ip/ip4_packet.h>
24 : #include <vnet/ip/ip6_packet.h>
25 : #include <vnet/devices/virtio/virtio.h>
26 : #include <vnet/devices/virtio/pci.h>
27 : #include <vnet/interface/rx_queue_funcs.h>
28 :
29 : #define PCI_VENDOR_ID_VIRTIO 0x1af4
30 : #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
31 : /* Doesn't support modern device */
32 : #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
33 :
34 : #define PCI_CAPABILITY_LIST 0x34
35 : #define PCI_CAP_ID_VNDR 0x09
36 : #define PCI_CAP_ID_MSIX 0x11
37 :
38 : #define PCI_MSIX_ENABLE 0x8000
39 :
40 : static pci_device_id_t virtio_pci_device_ids[] = {
41 : {
42 : .vendor_id = PCI_VENDOR_ID_VIRTIO,
43 : .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
44 : {
45 : .vendor_id = PCI_VENDOR_ID_VIRTIO,
46 : .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
47 : {0},
48 : };
49 :
50 : static u32
51 0 : virtio_pci_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw,
52 : u32 flags)
53 : {
54 0 : return 0;
55 : }
56 :
57 : static clib_error_t *
58 0 : virtio_pci_get_max_virtqueue_pairs (vlib_main_t * vm, virtio_if_t * vif)
59 : {
60 0 : clib_error_t *error = 0;
61 0 : u16 max_queue_pairs = 1;
62 :
63 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
64 : {
65 0 : max_queue_pairs = vif->virtio_pci_func->get_max_queue_pairs (vm, vif);
66 : }
67 :
68 0 : virtio_log_debug (vif, "max queue pair is %x", max_queue_pairs);
69 0 : if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
70 0 : return clib_error_return (error, "max queue pair is %x,"
71 : " should be in range [1, 0x8000]",
72 : max_queue_pairs);
73 :
74 0 : vif->max_queue_pairs = max_queue_pairs;
75 0 : return error;
76 : }
77 :
78 : static void
79 0 : virtio_pci_set_mac (vlib_main_t * vm, virtio_if_t * vif)
80 : {
81 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
82 0 : vif->virtio_pci_func->set_mac (vm, vif);
83 0 : }
84 :
85 : static u32
86 0 : virtio_pci_get_mac (vlib_main_t * vm, virtio_if_t * vif)
87 : {
88 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
89 : {
90 0 : vif->virtio_pci_func->get_mac (vm, vif);
91 0 : return 0;
92 : }
93 0 : return 1;
94 : }
95 :
96 : static u16
97 0 : virtio_pci_is_link_up (vlib_main_t * vm, virtio_if_t * vif)
98 : {
99 : /*
100 : * Minimal driver: assumes link is up
101 : */
102 0 : u16 status = 1;
103 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
104 0 : status = vif->virtio_pci_func->get_device_status (vm, vif);
105 0 : return status;
106 : }
107 :
108 : static void
109 0 : virtio_pci_irq_queue_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
110 : u16 line)
111 : {
112 0 : vnet_main_t *vnm = vnet_get_main ();
113 0 : virtio_main_t *vim = &virtio_main;
114 0 : uword pd = vlib_pci_get_private_data (vm, h);
115 0 : virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
116 0 : line--;
117 0 : u16 qid = line;
118 :
119 0 : vnet_virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
120 0 : vnet_hw_if_rx_queue_set_int_pending (vnm, vring->queue_index);
121 0 : }
122 :
123 : static void
124 0 : virtio_pci_irq_config_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
125 : u16 line)
126 : {
127 0 : vnet_main_t *vnm = vnet_get_main ();
128 0 : virtio_main_t *vim = &virtio_main;
129 0 : uword pd = vlib_pci_get_private_data (vm, h);
130 0 : virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
131 :
132 0 : if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
133 : {
134 0 : vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
135 : VNET_HW_INTERFACE_FLAG_LINK_UP);
136 : }
137 : else
138 : {
139 0 : vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
140 : }
141 0 : }
142 :
143 : static void
144 0 : virtio_pci_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h)
145 : {
146 0 : virtio_main_t *vim = &virtio_main;
147 0 : uword pd = vlib_pci_get_private_data (vm, h);
148 0 : virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
149 0 : u8 isr = 0;
150 0 : u16 line = 0;
151 :
152 0 : isr = vif->virtio_pci_func->get_isr (vm, vif);
153 :
154 : /*
155 : * If the lower bit is set: look through the used rings of
156 : * all virtqueues for the device, to see if any progress has
157 : * been made by the device which requires servicing.
158 : */
159 0 : if (isr & VIRTIO_PCI_ISR_INTR)
160 : {
161 0 : for (; line < vif->num_rxqs; line++)
162 0 : virtio_pci_irq_queue_handler (vm, h, (line + 1));
163 : }
164 :
165 0 : if (isr & VIRTIO_PCI_ISR_CONFIG)
166 0 : virtio_pci_irq_config_handler (vm, h, line);
167 0 : }
168 :
169 : inline void
170 0 : device_status (vlib_main_t * vm, virtio_if_t * vif)
171 : {
172 : struct status_struct
173 : {
174 : u8 bit;
175 : char *str;
176 : };
177 : struct status_struct *status_entry;
178 : static struct status_struct status_array[] = {
179 : #define _(s,b) { .str = #s, .bit = b, },
180 : foreach_virtio_config_status_flags
181 : #undef _
182 : {.str = NULL}
183 : };
184 :
185 0 : vlib_cli_output (vm, " status 0x%x", vif->status);
186 :
187 0 : status_entry = (struct status_struct *) &status_array;
188 0 : while (status_entry->str)
189 : {
190 0 : if (vif->status & status_entry->bit)
191 0 : vlib_cli_output (vm, " %s (%x)", status_entry->str,
192 0 : status_entry->bit);
193 0 : status_entry++;
194 : }
195 0 : }
196 :
197 : static int
198 0 : virtio_pci_send_ctrl_msg_packed (vlib_main_t * vm, virtio_if_t * vif,
199 : virtio_ctrl_msg_t * data, u32 len)
200 : {
201 0 : vnet_virtio_vring_t *vring = vif->cxq_vring;
202 0 : virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
203 : virtio_ctrl_msg_t result;
204 : u32 buffer_index;
205 : vlib_buffer_t *b;
206 : u16 used, next;
207 0 : u16 sz = vring->queue_size;
208 0 : u16 flags = 0, first_desc_flags = 0;
209 :
210 0 : used = vring->desc_in_use;
211 0 : next = vring->desc_next;
212 0 : vnet_virtio_vring_packed_desc_t *d = &vring->packed_desc[next];
213 :
214 0 : if (vlib_buffer_alloc (vm, &buffer_index, 1))
215 0 : b = vlib_get_buffer (vm, buffer_index);
216 : else
217 0 : return VIRTIO_NET_ERR;
218 : /*
219 : * current_data may not be initialized with 0 and may contain
220 : * previous offset.
221 : */
222 0 : b->current_data = 0;
223 0 : clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t));
224 :
225 0 : first_desc_flags = VRING_DESC_F_NEXT;
226 0 : if (vring->avail_wrap_counter)
227 : {
228 0 : first_desc_flags |= VRING_DESC_F_AVAIL;
229 0 : first_desc_flags &= ~VRING_DESC_F_USED;
230 : }
231 : else
232 : {
233 0 : first_desc_flags &= ~VRING_DESC_F_AVAIL;
234 0 : first_desc_flags |= VRING_DESC_F_USED;
235 : }
236 0 : d->addr = vlib_buffer_get_current_pa (vm, b);
237 0 : d->len = sizeof (virtio_net_ctrl_hdr_t);
238 0 : d->id = next;
239 :
240 0 : next++;
241 0 : if (next >= sz)
242 : {
243 0 : next = 0;
244 0 : vring->avail_wrap_counter ^= 1;
245 : }
246 0 : used++;
247 :
248 0 : d = &vring->packed_desc[next];
249 0 : flags = VRING_DESC_F_NEXT;
250 0 : if (vring->avail_wrap_counter)
251 : {
252 0 : flags |= VRING_DESC_F_AVAIL;
253 0 : flags &= ~VRING_DESC_F_USED;
254 : }
255 : else
256 : {
257 0 : flags &= ~VRING_DESC_F_AVAIL;
258 0 : flags |= VRING_DESC_F_USED;
259 : }
260 0 : d->addr = vlib_buffer_get_current_pa (vm, b) +
261 : STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data);
262 0 : d->len = len;
263 0 : d->id = next;
264 0 : d->flags = flags;
265 :
266 0 : next++;
267 0 : if (next >= sz)
268 : {
269 0 : next = 0;
270 0 : vring->avail_wrap_counter ^= 1;
271 : }
272 0 : used++;
273 :
274 0 : d = &vring->packed_desc[next];
275 0 : flags = VRING_DESC_F_WRITE;
276 0 : if (vring->avail_wrap_counter)
277 : {
278 0 : flags |= VRING_DESC_F_AVAIL;
279 0 : flags &= ~VRING_DESC_F_USED;
280 : }
281 : else
282 : {
283 0 : flags &= ~VRING_DESC_F_AVAIL;
284 0 : flags |= VRING_DESC_F_USED;
285 : }
286 0 : d->addr = vlib_buffer_get_current_pa (vm, b) +
287 : STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status);
288 0 : d->len = sizeof (data->status);
289 0 : d->id = next;
290 0 : d->flags = flags;
291 :
292 0 : next++;
293 0 : if (next >= sz)
294 : {
295 0 : next = 0;
296 0 : vring->avail_wrap_counter ^= 1;
297 : }
298 0 : used++;
299 :
300 0 : CLIB_MEMORY_STORE_BARRIER ();
301 0 : vring->packed_desc[vring->desc_next].flags = first_desc_flags;
302 0 : vring->desc_next = next;
303 0 : vring->desc_in_use = used;
304 0 : CLIB_MEMORY_BARRIER ();
305 0 : if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
306 : {
307 0 : virtio_kick (vm, vring, vif);
308 : }
309 :
310 0 : u16 last = vring->last_used_idx;
311 0 : d = &vring->packed_desc[last];
312 : do
313 : {
314 0 : flags = d->flags;
315 : }
316 0 : while ((flags & VRING_DESC_F_AVAIL) != (vring->used_wrap_counter << 7)
317 0 : || (flags & VRING_DESC_F_USED) != (vring->used_wrap_counter << 15));
318 :
319 0 : last += 3;
320 0 : if (last >= vring->queue_size)
321 : {
322 0 : last = last - vring->queue_size;
323 0 : vring->used_wrap_counter ^= 1;
324 : }
325 0 : vring->desc_in_use -= 3;
326 0 : vring->last_used_idx = last;
327 :
328 0 : CLIB_MEMORY_BARRIER ();
329 0 : clib_memcpy (&result, vlib_buffer_get_current (b),
330 : sizeof (virtio_ctrl_msg_t));
331 0 : virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
332 0 : status = result.status;
333 0 : vlib_buffer_free (vm, &buffer_index, 1);
334 0 : return status;
335 : }
336 :
337 : static int
338 0 : virtio_pci_send_ctrl_msg_split (vlib_main_t * vm, virtio_if_t * vif,
339 : virtio_ctrl_msg_t * data, u32 len)
340 : {
341 0 : vnet_virtio_vring_t *vring = vif->cxq_vring;
342 0 : virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
343 : virtio_ctrl_msg_t result;
344 : u32 buffer_index;
345 : vlib_buffer_t *b;
346 : u16 used, next, avail;
347 0 : u16 sz = vring->queue_size;
348 0 : u16 mask = sz - 1;
349 :
350 0 : used = vring->desc_in_use;
351 0 : next = vring->desc_next;
352 0 : avail = vring->avail->idx;
353 0 : vnet_virtio_vring_desc_t *d = &vring->desc[next];
354 :
355 0 : if (vlib_buffer_alloc (vm, &buffer_index, 1))
356 0 : b = vlib_get_buffer (vm, buffer_index);
357 : else
358 0 : return VIRTIO_NET_ERR;
359 : /*
360 : * current_data may not be initialized with 0 and may contain
361 : * previous offset.
362 : */
363 0 : b->current_data = 0;
364 0 : clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t));
365 0 : d->flags = VRING_DESC_F_NEXT;
366 0 : d->addr = vlib_buffer_get_current_pa (vm, b);
367 0 : d->len = sizeof (virtio_net_ctrl_hdr_t);
368 0 : vring->avail->ring[avail & mask] = next;
369 0 : avail++;
370 0 : next = (next + 1) & mask;
371 0 : d->next = next;
372 0 : used++;
373 :
374 0 : d = &vring->desc[next];
375 0 : d->flags = VRING_DESC_F_NEXT;
376 0 : d->addr = vlib_buffer_get_current_pa (vm, b) +
377 : STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data);
378 0 : d->len = len;
379 0 : next = (next + 1) & mask;
380 0 : d->next = next;
381 0 : used++;
382 :
383 0 : d = &vring->desc[next];
384 0 : d->flags = VRING_DESC_F_WRITE;
385 0 : d->addr = vlib_buffer_get_current_pa (vm, b) +
386 : STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status);
387 0 : d->len = sizeof (data->status);
388 0 : next = (next + 1) & mask;
389 0 : used++;
390 :
391 0 : CLIB_MEMORY_STORE_BARRIER ();
392 0 : vring->avail->idx = avail;
393 0 : vring->desc_next = next;
394 0 : vring->desc_in_use = used;
395 :
396 0 : if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
397 : {
398 0 : virtio_kick (vm, vring, vif);
399 : }
400 :
401 0 : u16 last = vring->last_used_idx, n_left = 0;
402 0 : n_left = vring->used->idx - last;
403 :
404 0 : while (n_left)
405 : {
406 0 : vnet_virtio_vring_used_elem_t *e = &vring->used->ring[last & mask];
407 0 : u16 slot = e->id;
408 :
409 0 : d = &vring->desc[slot];
410 0 : while (d->flags & VRING_DESC_F_NEXT)
411 : {
412 0 : used--;
413 0 : slot = d->next;
414 0 : d = &vring->desc[slot];
415 : }
416 0 : used--;
417 0 : last++;
418 0 : n_left--;
419 : }
420 0 : vring->desc_in_use = used;
421 0 : vring->last_used_idx = last;
422 :
423 0 : CLIB_MEMORY_BARRIER ();
424 0 : clib_memcpy (&result, vlib_buffer_get_current (b),
425 : sizeof (virtio_ctrl_msg_t));
426 0 : virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
427 0 : status = result.status;
428 0 : vlib_buffer_free (vm, &buffer_index, 1);
429 0 : return status;
430 : }
431 :
432 : static int
433 0 : virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif,
434 : virtio_ctrl_msg_t * data, u32 len)
435 : {
436 0 : if (vif->is_packed)
437 0 : return virtio_pci_send_ctrl_msg_packed (vm, vif, data, len);
438 : else
439 0 : return virtio_pci_send_ctrl_msg_split (vm, vif, data, len);
440 : }
441 :
442 : static int
443 0 : virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif)
444 : {
445 : virtio_ctrl_msg_t offload_hdr;
446 0 : virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
447 :
448 0 : offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
449 0 : offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
450 0 : offload_hdr.status = VIRTIO_NET_ERR;
451 0 : u64 offloads = 0ULL;
452 0 : clib_memcpy (offload_hdr.data, &offloads, sizeof (offloads));
453 :
454 0 : status =
455 0 : virtio_pci_send_ctrl_msg (vm, vif, &offload_hdr, sizeof (offloads));
456 0 : virtio_log_debug (vif, "disable offloads");
457 0 : vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
458 0 : vif->virtio_pci_func->get_driver_features (vm, vif);
459 0 : return status;
460 : }
461 :
462 : static int
463 0 : virtio_pci_enable_checksum_offload (vlib_main_t * vm, virtio_if_t * vif)
464 : {
465 : virtio_ctrl_msg_t csum_offload_hdr;
466 0 : virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
467 :
468 0 : csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
469 0 : csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
470 0 : csum_offload_hdr.status = VIRTIO_NET_ERR;
471 0 : u64 offloads = 0ULL;
472 0 : offloads |= VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM);
473 0 : clib_memcpy (csum_offload_hdr.data, &offloads, sizeof (offloads));
474 :
475 0 : status =
476 0 : virtio_pci_send_ctrl_msg (vm, vif, &csum_offload_hdr, sizeof (offloads));
477 0 : virtio_log_debug (vif, "enable checksum offload");
478 0 : vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
479 0 : vif->features = vif->virtio_pci_func->get_driver_features (vm, vif);
480 0 : return status;
481 : }
482 :
483 : static int
484 0 : virtio_pci_enable_gso (vlib_main_t * vm, virtio_if_t * vif)
485 : {
486 : virtio_ctrl_msg_t gso_hdr;
487 0 : virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
488 :
489 0 : gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
490 0 : gso_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
491 0 : gso_hdr.status = VIRTIO_NET_ERR;
492 0 : u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
493 : | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
494 : | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6);
495 0 : clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads));
496 :
497 0 : status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads));
498 0 : virtio_log_debug (vif, "enable gso");
499 0 : vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
500 0 : vif->virtio_pci_func->get_driver_features (vm, vif);
501 0 : return status;
502 : }
503 :
504 : static int
505 0 : virtio_pci_offloads (vlib_main_t * vm, virtio_if_t * vif, int gso_enabled,
506 : int csum_offload_enabled)
507 : {
508 0 : vnet_main_t *vnm = vnet_get_main ();
509 0 : vnet_hw_if_caps_change_t cc = {};
510 :
511 0 : if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
512 0 : (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)))
513 : {
514 0 : if (gso_enabled
515 0 : && (vif->features & (VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) |
516 : VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6))))
517 : {
518 0 : if (virtio_pci_enable_gso (vm, vif))
519 : {
520 0 : virtio_log_warning (vif, "gso is not enabled");
521 : }
522 : else
523 : {
524 0 : vif->gso_enabled = 1;
525 0 : vif->csum_offload_enabled = 1;
526 0 : cc.val = cc.mask = VNET_HW_IF_CAP_TCP_GSO |
527 : VNET_HW_IF_CAP_TX_TCP_CKSUM |
528 : VNET_HW_IF_CAP_TX_UDP_CKSUM;
529 : }
530 : }
531 0 : else if (csum_offload_enabled
532 0 : && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)))
533 : {
534 0 : if (virtio_pci_enable_checksum_offload (vm, vif))
535 : {
536 0 : virtio_log_warning (vif, "checksum offload is not enabled");
537 : }
538 : else
539 : {
540 0 : vif->csum_offload_enabled = 1;
541 0 : vif->gso_enabled = 0;
542 0 : cc.val =
543 : VNET_HW_IF_CAP_TX_TCP_CKSUM | VNET_HW_IF_CAP_TX_UDP_CKSUM;
544 0 : cc.mask = VNET_HW_IF_CAP_TCP_GSO | VNET_HW_IF_CAP_TX_TCP_CKSUM |
545 : VNET_HW_IF_CAP_TX_UDP_CKSUM;
546 : }
547 : }
548 : else
549 : {
550 0 : if (virtio_pci_disable_offload (vm, vif))
551 : {
552 0 : virtio_log_warning (vif, "offloads are not disabled");
553 : }
554 : else
555 : {
556 0 : vif->csum_offload_enabled = 0;
557 0 : vif->gso_enabled = 0;
558 0 : cc.val = 0;
559 0 : cc.mask = VNET_HW_IF_CAP_L4_TX_CKSUM | VNET_HW_IF_CAP_TCP_GSO;
560 : }
561 : }
562 : }
563 :
564 0 : if (cc.mask)
565 0 : vnet_hw_if_change_caps (vnm, vif->hw_if_index, &cc);
566 :
567 0 : return 0;
568 : }
569 :
570 : static int
571 0 : virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif,
572 : u16 num_queues)
573 : {
574 : virtio_ctrl_msg_t mq_hdr;
575 0 : virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
576 :
577 0 : mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
578 0 : mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
579 0 : mq_hdr.status = VIRTIO_NET_ERR;
580 0 : clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues));
581 :
582 0 : status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues));
583 0 : virtio_log_debug (vif, "multi-queue enable %u queues", num_queues);
584 0 : return status;
585 : }
586 :
587 : static u8
588 0 : virtio_pci_queue_size_valid (u16 qsz)
589 : {
590 0 : if (qsz < 64 || qsz > 4096)
591 0 : return 0;
592 0 : if ((qsz % 64) != 0)
593 0 : return 0;
594 0 : return 1;
595 : }
596 :
597 : clib_error_t *
598 0 : virtio_pci_control_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif,
599 : u16 queue_num)
600 : {
601 0 : clib_error_t *error = 0;
602 0 : u16 queue_size = 0;
603 : vnet_virtio_vring_t *vring;
604 0 : u32 i = 0;
605 0 : void *ptr = NULL;
606 :
607 0 : queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
608 :
609 0 : if (queue_size > 32768)
610 0 : return clib_error_return (0, "ring size must be 32768 or lower");
611 :
612 0 : if (queue_size == 0)
613 0 : queue_size = 256;
614 :
615 0 : vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES);
616 0 : vring = vec_elt_at_index (vif->cxq_vring, 0);
617 :
618 0 : i = (((queue_size * sizeof (vnet_virtio_vring_packed_desc_t)) +
619 0 : sizeof (vnet_virtio_vring_desc_event_t) + VNET_VIRTIO_PCI_VRING_ALIGN -
620 0 : 1) &
621 : ~(VNET_VIRTIO_PCI_VRING_ALIGN - 1)) +
622 : sizeof (vnet_virtio_vring_desc_event_t);
623 :
624 0 : ptr = vlib_physmem_alloc_aligned_on_numa (vm, i, VNET_VIRTIO_PCI_VRING_ALIGN,
625 : vif->numa_node);
626 0 : if (!ptr)
627 0 : return vlib_physmem_last_error (vm);
628 0 : clib_memset (ptr, 0, i);
629 :
630 0 : vring->packed_desc = ptr;
631 :
632 0 : vring->driver_event =
633 0 : ptr + (queue_size * sizeof (vnet_virtio_vring_packed_desc_t));
634 0 : vring->driver_event->off_wrap = 0;
635 0 : vring->driver_event->flags = VRING_EVENT_F_DISABLE;
636 :
637 0 : vring->device_event =
638 0 : ptr + (((queue_size * sizeof (vnet_virtio_vring_packed_desc_t)) +
639 : sizeof (vnet_virtio_vring_desc_event_t) +
640 0 : VNET_VIRTIO_PCI_VRING_ALIGN - 1) &
641 : ~(VNET_VIRTIO_PCI_VRING_ALIGN - 1));
642 0 : vring->device_event->off_wrap = 0;
643 0 : vring->device_event->flags = 0;
644 :
645 0 : vring->total_packets = 0;
646 0 : vring->queue_id = queue_num;
647 0 : vring->queue_size = queue_size;
648 0 : vring->avail_wrap_counter = 1;
649 0 : vring->used_wrap_counter = 1;
650 :
651 0 : ASSERT (vring->buffers == 0);
652 :
653 0 : virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num,
654 : queue_size);
655 0 : vif->virtio_pci_func->setup_queue (vm, vif, queue_num, vring);
656 0 : vring->queue_notify_offset =
657 0 : vif->notify_off_multiplier *
658 0 : vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
659 0 : virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
660 : queue_num, vring->queue_notify_offset);
661 0 : return error;
662 : }
663 :
664 : clib_error_t *
665 0 : virtio_pci_control_vring_split_init (vlib_main_t * vm, virtio_if_t * vif,
666 : u16 queue_num)
667 : {
668 0 : clib_error_t *error = 0;
669 0 : u16 queue_size = 0;
670 : vnet_virtio_vring_t *vring;
671 0 : u32 i = 0;
672 0 : void *ptr = NULL;
673 :
674 0 : queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
675 0 : if (!virtio_pci_queue_size_valid (queue_size))
676 0 : clib_warning ("queue size is not valid");
677 :
678 0 : if (!is_pow2 (queue_size))
679 0 : return clib_error_return (0, "ring size must be power of 2");
680 :
681 0 : if (queue_size > 32768)
682 0 : return clib_error_return (0, "ring size must be 32768 or lower");
683 :
684 0 : if (queue_size == 0)
685 0 : queue_size = 256;
686 :
687 0 : vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES);
688 0 : vring = vec_elt_at_index (vif->cxq_vring, 0);
689 0 : i = vnet_virtio_vring_size (queue_size, VNET_VIRTIO_PCI_VRING_ALIGN);
690 0 : i = round_pow2 (i, VNET_VIRTIO_PCI_VRING_ALIGN);
691 0 : ptr = vlib_physmem_alloc_aligned_on_numa (vm, i, VNET_VIRTIO_PCI_VRING_ALIGN,
692 : vif->numa_node);
693 0 : if (!ptr)
694 0 : return vlib_physmem_last_error (vm);
695 0 : clib_memset (ptr, 0, i);
696 0 : vnet_virtio_vring_init (vring, queue_size, ptr, VNET_VIRTIO_PCI_VRING_ALIGN);
697 0 : vring->queue_id = queue_num;
698 0 : vring->total_packets = 0;
699 :
700 0 : ASSERT (vring->buffers == 0);
701 0 : virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num,
702 : queue_size);
703 0 : vif->virtio_pci_func->setup_queue (vm, vif, queue_num, vring);
704 0 : vring->queue_notify_offset =
705 0 : vif->notify_off_multiplier *
706 0 : vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
707 0 : virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
708 : queue_num, vring->queue_notify_offset);
709 :
710 0 : return error;
711 : }
712 :
713 : clib_error_t *
714 0 : virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif,
715 : u16 queue_num)
716 : {
717 0 : if (vif->is_packed)
718 0 : return virtio_pci_control_vring_packed_init (vm, vif, queue_num);
719 : else
720 0 : return virtio_pci_control_vring_split_init (vm, vif, queue_num);
721 : }
722 :
723 : clib_error_t *
724 0 : virtio_pci_vring_split_init (vlib_main_t * vm, virtio_if_t * vif,
725 : u16 queue_num)
726 : {
727 0 : clib_error_t *error = 0;
728 0 : u16 queue_size = 0;
729 : vnet_virtio_vring_t *vring;
730 0 : u32 i = 0;
731 0 : void *ptr = NULL;
732 :
733 0 : queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
734 0 : if (!virtio_pci_queue_size_valid (queue_size))
735 0 : clib_warning ("queue size is not valid");
736 :
737 0 : if (!is_pow2 (queue_size))
738 0 : return clib_error_return (0, "ring size must be power of 2");
739 :
740 0 : if (queue_size > 32768)
741 0 : return clib_error_return (0, "ring size must be 32768 or lower");
742 :
743 0 : if (queue_size == 0)
744 0 : queue_size = 256;
745 :
746 0 : if (queue_num % 2)
747 : {
748 0 : vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num),
749 : CLIB_CACHE_LINE_BYTES);
750 0 : vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
751 0 : clib_spinlock_init (&vring->lockp);
752 : }
753 : else
754 : {
755 0 : vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num),
756 : CLIB_CACHE_LINE_BYTES);
757 0 : vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
758 : }
759 0 : i = vnet_virtio_vring_size (queue_size, VNET_VIRTIO_PCI_VRING_ALIGN);
760 0 : i = round_pow2 (i, VNET_VIRTIO_PCI_VRING_ALIGN);
761 0 : ptr = vlib_physmem_alloc_aligned_on_numa (vm, i, VNET_VIRTIO_PCI_VRING_ALIGN,
762 : vif->numa_node);
763 0 : if (!ptr)
764 0 : return vlib_physmem_last_error (vm);
765 0 : clib_memset (ptr, 0, i);
766 0 : vnet_virtio_vring_init (vring, queue_size, ptr, VNET_VIRTIO_PCI_VRING_ALIGN);
767 0 : vring->queue_id = queue_num;
768 0 : vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
769 0 : vring->flow_table = 0;
770 0 : vring->total_packets = 0;
771 :
772 0 : ASSERT (vring->buffers == 0);
773 0 : vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
774 0 : if (queue_num % 2)
775 : {
776 0 : virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num,
777 : queue_size);
778 0 : clib_memset_u32 (vring->buffers, ~0, queue_size);
779 : }
780 : else
781 : {
782 0 : virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num,
783 : queue_size);
784 : }
785 0 : vring->queue_size = queue_size;
786 0 : if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, vring))
787 0 : return clib_error_return (0, "error in queue address setup");
788 :
789 0 : vring->queue_notify_offset =
790 0 : vif->notify_off_multiplier *
791 0 : vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
792 0 : virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
793 : queue_num, vring->queue_notify_offset);
794 0 : return error;
795 : }
796 :
797 : clib_error_t *
798 0 : virtio_pci_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif,
799 : u16 queue_num)
800 : {
801 0 : clib_error_t *error = 0;
802 0 : u16 queue_size = 0;
803 : vnet_virtio_vring_t *vring;
804 0 : u32 i = 0;
805 0 : void *ptr = NULL;
806 :
807 0 : queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
808 :
809 0 : if (queue_size > 32768)
810 0 : return clib_error_return (0, "ring size must be 32768 or lower");
811 :
812 0 : if (queue_size == 0)
813 0 : queue_size = 256;
814 :
815 0 : if (queue_num % 2)
816 : {
817 0 : vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num),
818 : CLIB_CACHE_LINE_BYTES);
819 0 : vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
820 0 : clib_spinlock_init (&vring->lockp);
821 : }
822 : else
823 : {
824 0 : vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num),
825 : CLIB_CACHE_LINE_BYTES);
826 0 : vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
827 : }
828 :
829 0 : i = (((queue_size * sizeof (vnet_virtio_vring_packed_desc_t)) +
830 0 : sizeof (vnet_virtio_vring_desc_event_t) + VNET_VIRTIO_PCI_VRING_ALIGN -
831 0 : 1) &
832 : ~(VNET_VIRTIO_PCI_VRING_ALIGN - 1)) +
833 : sizeof (vnet_virtio_vring_desc_event_t);
834 :
835 0 : ptr = vlib_physmem_alloc_aligned_on_numa (vm, i, VNET_VIRTIO_PCI_VRING_ALIGN,
836 : vif->numa_node);
837 0 : if (!ptr)
838 0 : return vlib_physmem_last_error (vm);
839 :
840 0 : clib_memset (ptr, 0, i);
841 0 : vring->packed_desc = ptr;
842 :
843 0 : vring->driver_event =
844 0 : ptr + (queue_size * sizeof (vnet_virtio_vring_packed_desc_t));
845 0 : vring->driver_event->off_wrap = 0;
846 0 : vring->driver_event->flags = VRING_EVENT_F_DISABLE;
847 :
848 0 : vring->device_event =
849 0 : ptr + (((queue_size * sizeof (vnet_virtio_vring_packed_desc_t)) +
850 : sizeof (vnet_virtio_vring_desc_event_t) +
851 0 : VNET_VIRTIO_PCI_VRING_ALIGN - 1) &
852 : ~(VNET_VIRTIO_PCI_VRING_ALIGN - 1));
853 0 : vring->device_event->off_wrap = 0;
854 0 : vring->device_event->flags = 0;
855 :
856 0 : vring->queue_id = queue_num;
857 :
858 0 : vring->avail_wrap_counter = 1;
859 0 : vring->used_wrap_counter = 1;
860 0 : vring->total_packets = 0;
861 :
862 0 : ASSERT (vring->buffers == 0);
863 0 : vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
864 0 : if (queue_num % 2)
865 : {
866 0 : virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num,
867 : queue_size);
868 0 : clib_memset_u32 (vring->buffers, ~0, queue_size);
869 : }
870 : else
871 : {
872 0 : virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num,
873 : queue_size);
874 : }
875 0 : vring->queue_size = queue_size;
876 0 : if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, vring))
877 0 : return clib_error_return (0, "error in queue address setup");
878 :
879 0 : vring->queue_notify_offset =
880 0 : vif->notify_off_multiplier *
881 0 : vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
882 0 : virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
883 : queue_num, vring->queue_notify_offset);
884 :
885 0 : return error;
886 : }
887 :
888 : clib_error_t *
889 0 : virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num)
890 : {
891 0 : if (vif->is_packed)
892 0 : return virtio_pci_vring_packed_init (vm, vif, queue_num);
893 : else
894 0 : return virtio_pci_vring_split_init (vm, vif, queue_num);
895 : }
896 :
897 : static void
898 0 : virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif,
899 : u64 req_features)
900 : {
901 : /*
902 : * if features are not requested
903 : * default: all supported features
904 : */
905 0 : u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)
906 : | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
907 : | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
908 : | VIRTIO_FEATURE (VIRTIO_NET_F_MTU)
909 : | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
910 : | VIRTIO_FEATURE (VIRTIO_NET_F_GSO)
911 : | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
912 : | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
913 : | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)
914 : | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4)
915 : | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)
916 : | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO)
917 : | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
918 : | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
919 : | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)
920 : | VIRTIO_FEATURE (VIRTIO_NET_F_MQ)
921 : | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY)
922 : | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
923 : | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
924 :
925 0 : if (vif->is_modern)
926 0 : supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1);
927 :
928 0 : if (vif->is_packed)
929 : {
930 0 : supported_features |=
931 : (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED) |
932 : VIRTIO_FEATURE (VIRTIO_F_IN_ORDER));
933 : }
934 :
935 0 : if (req_features == 0)
936 : {
937 0 : req_features = supported_features;
938 : }
939 :
940 0 : vif->features = req_features & vif->remote_features & supported_features;
941 :
942 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MTU))
943 : {
944 0 : u16 mtu = 0;
945 0 : mtu = vif->virtio_pci_func->get_mtu (vm, vif);
946 :
947 0 : if (mtu < 64)
948 0 : vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU);
949 : }
950 :
951 0 : if ((vif->features & (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED))) == 0)
952 0 : vif->is_packed = 0;
953 :
954 0 : vif->virtio_pci_func->set_driver_features (vm, vif, vif->features);
955 0 : vif->features = vif->virtio_pci_func->get_driver_features (vm, vif);
956 0 : }
957 :
958 : void
959 0 : virtio_pci_read_device_feature (vlib_main_t * vm, virtio_if_t * vif)
960 : {
961 0 : vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
962 0 : }
963 :
964 : int
965 0 : virtio_pci_reset_device (vlib_main_t * vm, virtio_if_t * vif)
966 : {
967 0 : u8 status = 0;
968 :
969 : /*
970 : * Reset the device
971 : */
972 0 : status = vif->virtio_pci_func->device_reset (vm, vif);
973 :
974 : /*
975 : * Set the Acknowledge status bit
976 : */
977 0 : vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
978 :
979 : /*
980 : * Set the Driver status bit
981 : */
982 0 : vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
983 :
984 : /*
985 : * Read the status and verify it
986 : */
987 0 : status = vif->virtio_pci_func->get_status (vm, vif);
988 0 : if ((status & VIRTIO_CONFIG_STATUS_ACK)
989 0 : && (status & VIRTIO_CONFIG_STATUS_DRIVER))
990 0 : vif->status = status;
991 : else
992 0 : return -1;
993 :
994 0 : return 0;
995 : }
996 :
997 : clib_error_t *
998 0 : virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif, void **bar)
999 : {
1000 0 : clib_error_t *error = 0;
1001 : virtio_pci_cap_t cap;
1002 0 : u8 pos, common_cfg = 0, notify = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
1003 0 : vlib_pci_dev_handle_t h = vif->pci_dev_handle;
1004 :
1005 0 : if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
1006 : {
1007 0 : virtio_log_error (vif, "error in reading capabilty list position");
1008 0 : return clib_error_return (error,
1009 : "error in reading capabilty list position");
1010 : }
1011 0 : while (pos)
1012 : {
1013 0 : if ((error =
1014 0 : vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
1015 : sizeof (cap))))
1016 : {
1017 0 : virtio_log_error (vif, "%s [%2x]",
1018 : "error in reading the capability at", pos);
1019 0 : return clib_error_return (error,
1020 : "error in reading the capability at [%2x]",
1021 : pos);
1022 : }
1023 :
1024 0 : if (cap.cap_vndr == PCI_CAP_ID_MSIX)
1025 : {
1026 0 : u16 flags, table_size, table_size_mask = 0x07FF;
1027 :
1028 0 : if ((error =
1029 0 : vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags,
1030 : sizeof (flags))))
1031 0 : return clib_error_return (error,
1032 : "error in reading the capability at [%2x]",
1033 : pos + 2);
1034 :
1035 0 : table_size = flags & table_size_mask;
1036 0 : virtio_log_debug (vif, "flags:0x%x %s 0x%x", flags,
1037 : "msix interrupt vector table-size", table_size);
1038 :
1039 0 : if (flags & PCI_MSIX_ENABLE)
1040 : {
1041 0 : virtio_log_debug (vif, "msix interrupt enabled");
1042 0 : vif->msix_enabled = VIRTIO_MSIX_ENABLED;
1043 0 : vif->msix_table_size = table_size;
1044 : }
1045 : else
1046 : {
1047 0 : virtio_log_debug (vif, "msix interrupt disabled");
1048 0 : vif->msix_enabled = VIRTIO_MSIX_DISABLED;
1049 0 : vif->msix_table_size = 0;
1050 : }
1051 : }
1052 :
1053 0 : if (cap.cap_vndr != PCI_CAP_ID_VNDR)
1054 : {
1055 0 : virtio_log_debug (vif, "[%2x] %s %2x ", pos,
1056 : "skipping non VNDR cap id:", cap.cap_vndr);
1057 0 : goto next;
1058 : }
1059 :
1060 0 : virtio_log_debug (vif,
1061 : "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
1062 : pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
1063 :
1064 0 : if (cap.bar >= 0 && cap.bar <= 5)
1065 : {
1066 0 : vif->bar = bar[cap.bar];
1067 0 : vif->bar_id = cap.bar;
1068 : }
1069 : else
1070 0 : return clib_error_return (error, "invalid bar %u", cap.bar);
1071 :
1072 0 : switch (cap.cfg_type)
1073 : {
1074 0 : case VIRTIO_PCI_CAP_COMMON_CFG:
1075 0 : vif->common_offset = cap.offset;
1076 0 : common_cfg = 1;
1077 0 : break;
1078 0 : case VIRTIO_PCI_CAP_NOTIFY_CFG:
1079 0 : if ((error =
1080 0 : vlib_pci_read_write_config (vm, h, VLIB_READ,
1081 : pos + sizeof (cap),
1082 0 : &vif->notify_off_multiplier,
1083 : sizeof
1084 : (vif->notify_off_multiplier))))
1085 : {
1086 0 : virtio_log_error (vif, "notify off multiplier is not given");
1087 : }
1088 : else
1089 : {
1090 0 : virtio_log_debug (vif, "notify off multiplier is %u",
1091 : vif->notify_off_multiplier);
1092 0 : vif->notify_offset = cap.offset;
1093 0 : notify = 1;
1094 : }
1095 0 : break;
1096 0 : case VIRTIO_PCI_CAP_DEVICE_CFG:
1097 0 : vif->device_offset = cap.offset;
1098 0 : dev_cfg = 1;
1099 0 : break;
1100 0 : case VIRTIO_PCI_CAP_ISR_CFG:
1101 0 : vif->isr_offset = cap.offset;
1102 0 : isr = 1;
1103 0 : break;
1104 0 : case VIRTIO_PCI_CAP_PCI_CFG:
1105 0 : if (cap.bar == 0)
1106 0 : pci_cfg = 1;
1107 0 : break;
1108 : }
1109 0 : next:
1110 0 : pos = cap.cap_next;
1111 : }
1112 :
1113 0 : if (common_cfg == 0 || notify == 0 || dev_cfg == 0 || isr == 0)
1114 : {
1115 0 : vif->virtio_pci_func = &virtio_pci_legacy_func;
1116 0 : vif->notify_off_multiplier = 0;
1117 0 : virtio_log_debug (vif, "legacy virtio pci device found");
1118 0 : return error;
1119 : }
1120 :
1121 0 : vif->is_modern = 1;
1122 0 : vif->virtio_pci_func = &virtio_pci_modern_func;
1123 :
1124 0 : if (!pci_cfg)
1125 : {
1126 0 : virtio_log_debug (vif, "modern virtio pci device found");
1127 : }
1128 : else
1129 : {
1130 0 : virtio_log_debug (vif, "transitional virtio pci device found");
1131 : }
1132 :
1133 0 : return error;
1134 : }
1135 :
1136 : static clib_error_t *
1137 0 : virtio_pci_device_init (vlib_main_t * vm, virtio_if_t * vif,
1138 : virtio_pci_create_if_args_t * args, void **bar)
1139 : {
1140 0 : clib_error_t *error = 0;
1141 0 : u8 status = 0;
1142 :
1143 0 : if ((error = virtio_pci_read_caps (vm, vif, bar)))
1144 : {
1145 0 : args->rv = VNET_API_ERROR_UNSUPPORTED;
1146 0 : virtio_log_error (vif, "Device is not supported");
1147 0 : return clib_error_return (error, "Device is not supported");
1148 : }
1149 :
1150 0 : if (virtio_pci_reset_device (vm, vif) < 0)
1151 : {
1152 0 : args->rv = VNET_API_ERROR_INIT_FAILED;
1153 0 : virtio_log_error (vif, "Failed to reset the device");
1154 0 : return clib_error_return (error, "Failed to reset the device");
1155 : }
1156 : /*
1157 : * read device features and negotiate (user) requested features
1158 : */
1159 0 : virtio_pci_read_device_feature (vm, vif);
1160 0 : if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) ==
1161 : 0)
1162 : {
1163 0 : virtio_log_warning (vif, "error encountered: vhost-net backend doesn't "
1164 : "support VIRTIO_RING_F_INDIRECT_DESC features");
1165 : }
1166 0 : if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)) == 0)
1167 : {
1168 0 : virtio_log_warning (vif, "error encountered: vhost-net backend doesn't "
1169 : "support VIRTIO_NET_F_MRG_RXBUF features");
1170 : }
1171 0 : virtio_negotiate_features (vm, vif, args->features);
1172 :
1173 : /*
1174 : * After FEATURE_OK, driver should not accept new feature bits
1175 : */
1176 0 : vif->virtio_pci_func->set_status (vm, vif,
1177 : VIRTIO_CONFIG_STATUS_FEATURES_OK);
1178 0 : status = vif->virtio_pci_func->get_status (vm, vif);
1179 0 : if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
1180 : {
1181 0 : args->rv = VNET_API_ERROR_UNSUPPORTED;
1182 0 : virtio_log_error (vif,
1183 : "error encountered: Device doesn't support requested features");
1184 0 : return clib_error_return (error,
1185 : "Device doesn't support requested features");
1186 : }
1187 0 : vif->status = status;
1188 :
1189 : /*
1190 : * get or set the mac address
1191 : */
1192 0 : if (virtio_pci_get_mac (vm, vif))
1193 : {
1194 0 : f64 now = vlib_time_now (vm);
1195 : u32 rnd;
1196 0 : rnd = (u32) (now * 1e6);
1197 0 : rnd = random_u32 (&rnd);
1198 :
1199 0 : memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
1200 0 : vif->mac_addr[0] = 2;
1201 0 : vif->mac_addr[1] = 0xfe;
1202 0 : virtio_pci_set_mac (vm, vif);
1203 : }
1204 :
1205 0 : virtio_set_net_hdr_size (vif);
1206 :
1207 : /*
1208 : * Initialize the virtqueues
1209 : */
1210 0 : if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif)))
1211 : {
1212 0 : args->rv = VNET_API_ERROR_EXCEEDED_NUMBER_OF_RANGES_CAPACITY;
1213 0 : goto err;
1214 : }
1215 :
1216 0 : if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1217 : {
1218 0 : if (vif->msix_table_size <= vif->max_queue_pairs)
1219 : {
1220 0 : virtio_log_error (vif,
1221 : "error MSIX lines (%u) <= Number of RXQs (%u)",
1222 : vif->msix_table_size, vif->max_queue_pairs);
1223 0 : return clib_error_return (error,
1224 : "error MSIX lines (%u) <= Number of RXQs (%u)",
1225 : vif->msix_table_size,
1226 : vif->max_queue_pairs);
1227 : }
1228 : }
1229 :
1230 0 : for (int i = 0; i < vif->max_queue_pairs; i++)
1231 : {
1232 0 : if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i))))
1233 : {
1234 0 : args->rv = VNET_API_ERROR_INIT_FAILED;
1235 0 : virtio_log_error (vif, "%s (%u) %s", "error in rxq-queue",
1236 : RX_QUEUE (i), "initialization");
1237 : error =
1238 0 : clib_error_return (error, "%s (%u) %s", "error in rxq-queue",
1239 : RX_QUEUE (i), "initialization");
1240 0 : goto err;
1241 : }
1242 : else
1243 : {
1244 0 : vif->num_rxqs++;
1245 : }
1246 :
1247 0 : if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i))))
1248 : {
1249 0 : args->rv = VNET_API_ERROR_INIT_FAILED;
1250 0 : virtio_log_error (vif, "%s (%u) %s", "error in txq-queue",
1251 : TX_QUEUE (i), "initialization");
1252 : error =
1253 0 : clib_error_return (error, "%s (%u) %s", "error in txq-queue",
1254 : TX_QUEUE (i), "initialization");
1255 0 : goto err;
1256 : }
1257 : else
1258 : {
1259 0 : vif->num_txqs++;
1260 : }
1261 : }
1262 :
1263 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1264 : {
1265 0 : if ((error =
1266 0 : virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2)))
1267 : {
1268 0 : virtio_log_warning (vif, "%s (%u) %s", "error in control-queue",
1269 : vif->max_queue_pairs * 2, "initialization");
1270 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1271 0 : vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ);
1272 : }
1273 : }
1274 : else
1275 : {
1276 0 : virtio_log_debug (vif, "control queue is not available");
1277 0 : vif->cxq_vring = NULL;
1278 : }
1279 :
1280 : /*
1281 : * set the msix interrupts
1282 : */
1283 0 : if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1284 : {
1285 : int i, j;
1286 0 : if (vif->virtio_pci_func->set_config_irq (vm, vif, 0) ==
1287 : VIRTIO_MSI_NO_VECTOR)
1288 : {
1289 0 : virtio_log_warning (vif, "config vector 0 is not set");
1290 : }
1291 : else
1292 : {
1293 0 : virtio_log_debug (vif, "config msix vector is set at 0");
1294 : }
1295 0 : for (i = 0, j = 1; i < vif->max_queue_pairs; i++, j++)
1296 : {
1297 0 : if (vif->virtio_pci_func->set_queue_irq (vm, vif, j,
1298 : RX_QUEUE (i)) ==
1299 : VIRTIO_MSI_NO_VECTOR)
1300 : {
1301 0 : virtio_log_warning (vif, "queue (%u) vector is not set at %u",
1302 : RX_QUEUE (i), j);
1303 : }
1304 : else
1305 : {
1306 0 : virtio_log_debug (vif, "%s (%u) %s %u", "queue",
1307 : RX_QUEUE (i), "msix vector is set at", j);
1308 : }
1309 : }
1310 : }
1311 :
1312 : /*
1313 : * set the driver status OK
1314 : */
1315 0 : vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1316 0 : vif->status = vif->virtio_pci_func->get_status (vm, vif);
1317 0 : err:
1318 0 : return error;
1319 : }
1320 :
1321 : void
1322 0 : virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args)
1323 : {
1324 0 : vnet_main_t *vnm = vnet_get_main ();
1325 0 : virtio_main_t *vim = &virtio_main;
1326 : virtio_if_t *vif;
1327 : vlib_pci_dev_handle_t h;
1328 0 : clib_error_t *error = 0;
1329 0 : u32 interrupt_count = 0;
1330 :
1331 : /* *INDENT-OFF* */
1332 0 : pool_foreach (vif, vim->interfaces) {
1333 0 : if (vif->pci_addr.as_u32 == args->addr)
1334 : {
1335 0 : args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
1336 0 : args->error =
1337 0 : clib_error_return (error, "PCI address in use");
1338 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1339 : format_vlib_pci_addr, &args->addr,
1340 : " PCI address in use");
1341 0 : return;
1342 : }
1343 : }
1344 : /* *INDENT-ON* */
1345 :
1346 0 : if (args->bind)
1347 : {
1348 0 : vlib_pci_addr_t pci = { .as_u32 = args->addr };
1349 0 : error = vlib_pci_bind_to_uio (vm, &pci, (char *) "auto",
1350 0 : VIRTIO_BIND_FORCE == args->bind);
1351 0 : if (error)
1352 : {
1353 0 : args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1354 0 : args->error =
1355 0 : clib_error_return (error, "%U: %s", format_vlib_pci_addr, &pci,
1356 : "error encountered on binding pci device");
1357 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1358 : format_vlib_pci_addr, &pci,
1359 : "error encountered on binding pci devicee");
1360 0 : return;
1361 : }
1362 : }
1363 :
1364 0 : pool_get (vim->interfaces, vif);
1365 0 : vif->dev_instance = vif - vim->interfaces;
1366 0 : vif->per_interface_next_index = ~0;
1367 0 : vif->pci_addr.as_u32 = args->addr;
1368 0 : if (args->virtio_flags & VIRTIO_FLAG_PACKED)
1369 0 : vif->is_packed = 1;
1370 :
1371 0 : if ((error =
1372 0 : vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
1373 : virtio_pci_device_ids, &h)))
1374 : {
1375 0 : args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1376 0 : args->error =
1377 0 : clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1378 : &vif->pci_addr);
1379 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1380 0 : format_vlib_pci_addr, &vif->pci_addr,
1381 : "error encountered on pci device open");
1382 0 : pool_put (vim->interfaces, vif);
1383 0 : return;
1384 : }
1385 0 : vif->pci_dev_handle = h;
1386 0 : vlib_pci_set_private_data (vm, h, vif->dev_instance);
1387 0 : vif->numa_node = vlib_pci_get_numa_node (vm, h);
1388 0 : vif->type = VIRTIO_IF_TYPE_PCI;
1389 :
1390 0 : if ((error = vlib_pci_bus_master_enable (vm, h)))
1391 : {
1392 0 : virtio_log_error (vif, "error encountered on pci bus master enable");
1393 0 : goto error;
1394 : }
1395 :
1396 : void *bar[6];
1397 0 : for (u32 i = 0; i <= 5; i++)
1398 : {
1399 :
1400 0 : if ((error = vlib_pci_map_region (vm, h, i, &bar[i])))
1401 : {
1402 0 : virtio_log_debug (vif, "no pci map region for bar %u", i);
1403 : }
1404 : else
1405 : {
1406 0 : virtio_log_debug (vif, "pci map region for bar %u at %p", i,
1407 : bar[i]);
1408 : }
1409 : }
1410 :
1411 0 : if ((error = vlib_pci_io_region (vm, h, 0)))
1412 : {
1413 0 : virtio_log_error (vif, "error encountered on pci io region");
1414 0 : goto error;
1415 : }
1416 :
1417 0 : interrupt_count = vlib_pci_get_num_msix_interrupts (vm, h);
1418 0 : if (interrupt_count > 1)
1419 : {
1420 0 : if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1421 : &virtio_pci_irq_config_handler)))
1422 : {
1423 0 : args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1424 0 : virtio_log_error (vif,
1425 : "error encountered on pci register msix handler 0");
1426 0 : goto error;
1427 : }
1428 :
1429 0 : if ((error =
1430 0 : vlib_pci_register_msix_handler (vm, h, 1, (interrupt_count - 1),
1431 : &virtio_pci_irq_queue_handler)))
1432 : {
1433 0 : args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1434 0 : virtio_log_error (vif,
1435 : "error encountered on pci register msix handler 1");
1436 0 : goto error;
1437 : }
1438 :
1439 0 : if ((error = vlib_pci_enable_msix_irq (vm, h, 0, interrupt_count)))
1440 : {
1441 0 : virtio_log_error (vif, "error encountered on pci enable msix irq");
1442 0 : goto error;
1443 : }
1444 0 : vif->support_int_mode = 1;
1445 0 : virtio_log_debug (vif, "device supports msix interrupts");
1446 : }
1447 : else
1448 : {
1449 : /*
1450 : * WARN: performance will be sub-optimal.
1451 : * Fall back to intX, if msix table-size is 1 or
1452 : * if UIO driver is being used.
1453 : */
1454 0 : if ((error =
1455 0 : vlib_pci_register_intx_handler (vm, h, &virtio_pci_irq_handler)))
1456 : {
1457 0 : virtio_log_error (vif,
1458 : "error encountered on pci register interrupt handler");
1459 0 : goto error;
1460 : }
1461 0 : vif->support_int_mode = 1;
1462 0 : virtio_log_debug (vif, "pci register interrupt handler");
1463 : }
1464 :
1465 0 : if ((error = vlib_pci_intr_enable (vm, h)))
1466 : {
1467 0 : virtio_log_error (vif, "error encountered on pci interrupt enable");
1468 0 : goto error;
1469 : }
1470 :
1471 0 : if ((error = virtio_pci_device_init (vm, vif, args, bar)))
1472 : {
1473 0 : virtio_log_error (vif, "error encountered on device init");
1474 0 : goto error;
1475 : }
1476 :
1477 : /* create interface */
1478 0 : vnet_eth_interface_registration_t eir = {};
1479 0 : eir.dev_class_index = virtio_device_class.index;
1480 0 : eir.dev_instance = vif->dev_instance;
1481 0 : eir.address = vif->mac_addr;
1482 0 : eir.cb.flag_change = virtio_pci_flag_change;
1483 0 : vif->hw_if_index = vnet_eth_register_interface (vnm, &eir);
1484 :
1485 0 : vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index);
1486 0 : vif->sw_if_index = sw->sw_if_index;
1487 0 : args->sw_if_index = sw->sw_if_index;
1488 :
1489 0 : vnet_hw_if_set_caps (vnm, vif->hw_if_index, VNET_HW_IF_CAP_INT_MODE);
1490 :
1491 0 : if (args->virtio_flags & VIRTIO_FLAG_BUFFERING)
1492 : {
1493 0 : error = virtio_set_packet_buffering (vif, args->buffering_size);
1494 0 : if (error)
1495 : {
1496 0 : args->rv = VNET_API_ERROR_INIT_FAILED;
1497 0 : virtio_log_error (vif,
1498 : "error encountered during packet buffering init");
1499 0 : goto error;
1500 : }
1501 : /*
1502 : * packet buffering flag needs to be set 1 before calling the
1503 : * virtio_pre_input_node_enable but after the successful initialization
1504 : * of buffering queues above.
1505 : * Packet buffering flag set to 0 if there will be any error during
1506 : * buffering initialization.
1507 : */
1508 0 : vif->packet_buffering = 1;
1509 0 : virtio_pre_input_node_enable (vm, vif);
1510 : }
1511 :
1512 0 : virtio_vring_set_rx_queues (vm, vif);
1513 0 : virtio_vring_set_tx_queues (vm, vif);
1514 :
1515 0 : if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
1516 : {
1517 0 : vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1518 : VNET_HW_INTERFACE_FLAG_LINK_UP);
1519 : }
1520 : else
1521 0 : vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1522 :
1523 0 : virtio_pci_offloads (vm, vif, args->gso_enabled,
1524 0 : args->checksum_offload_enabled);
1525 :
1526 0 : if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
1527 0 : (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)))
1528 : {
1529 0 : if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
1530 0 : virtio_log_warning (vif, "multiqueue is not set");
1531 : }
1532 0 : return;
1533 :
1534 0 : error:
1535 0 : virtio_pci_delete_if (vm, vif);
1536 0 : if (args->rv == 0)
1537 0 : args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1538 0 : args->error = error;
1539 : }
1540 :
1541 : int
1542 0 : virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif)
1543 : {
1544 0 : vnet_main_t *vnm = vnet_get_main ();
1545 0 : virtio_main_t *vim = &virtio_main;
1546 0 : u32 i = 0;
1547 :
1548 0 : if (vif->type != VIRTIO_IF_TYPE_PCI)
1549 0 : return VNET_API_ERROR_INVALID_INTERFACE;
1550 :
1551 0 : vlib_pci_intr_disable (vm, vif->pci_dev_handle);
1552 :
1553 0 : if (vif->virtio_pci_func)
1554 : {
1555 0 : for (i = 0; i < vif->max_queue_pairs; i++)
1556 : {
1557 0 : vif->virtio_pci_func->del_queue (vm, vif, RX_QUEUE (i));
1558 0 : vif->virtio_pci_func->del_queue (vm, vif, TX_QUEUE (i));
1559 : }
1560 :
1561 0 : if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1562 0 : vif->virtio_pci_func->del_queue (vm, vif, vif->max_queue_pairs * 2);
1563 :
1564 0 : vif->virtio_pci_func->device_reset (vm, vif);
1565 : }
1566 :
1567 0 : if (vif->hw_if_index)
1568 : {
1569 0 : vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1570 0 : ethernet_delete_interface (vnm, vif->hw_if_index);
1571 : }
1572 :
1573 0 : vlib_pci_device_close (vm, vif->pci_dev_handle);
1574 :
1575 0 : vec_foreach_index (i, vif->rxq_vrings)
1576 : {
1577 0 : vnet_virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
1578 0 : if (vring->used)
1579 : {
1580 0 : virtio_free_buffers (vm, vring);
1581 : }
1582 0 : vec_free (vring->buffers);
1583 0 : vlib_physmem_free (vm, vring->desc);
1584 : }
1585 :
1586 0 : if (vif->packet_buffering)
1587 0 : virtio_pre_input_node_disable (vm, vif);
1588 :
1589 0 : vec_foreach_index (i, vif->txq_vrings)
1590 : {
1591 0 : vnet_virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
1592 0 : if (vring->used)
1593 : {
1594 0 : virtio_free_buffers (vm, vring);
1595 : }
1596 0 : vec_free (vring->buffers);
1597 0 : gro_flow_table_free (vring->flow_table);
1598 0 : virtio_vring_buffering_free (vm, vring->buffering);
1599 0 : clib_spinlock_free (&vring->lockp);
1600 0 : vlib_physmem_free (vm, vring->desc);
1601 : }
1602 :
1603 0 : if (vif->cxq_vring != NULL)
1604 : {
1605 0 : u16 last = vif->cxq_vring->last_used_idx;
1606 0 : u16 n_left = vif->cxq_vring->used->idx - last;
1607 0 : while (n_left)
1608 : {
1609 0 : last++;
1610 0 : n_left--;
1611 : }
1612 :
1613 0 : vif->cxq_vring->last_used_idx = last;
1614 0 : vlib_physmem_free (vm, vif->cxq_vring->desc);
1615 : }
1616 :
1617 0 : vec_free (vif->rxq_vrings);
1618 0 : vec_free (vif->txq_vrings);
1619 0 : vec_free (vif->cxq_vring);
1620 :
1621 0 : clib_error_free (vif->error);
1622 0 : memset (vif, 0, sizeof (*vif));
1623 0 : pool_put (vim->interfaces, vif);
1624 :
1625 0 : return 0;
1626 : }
1627 :
1628 : int
1629 0 : virtio_pci_enable_disable_offloads (vlib_main_t * vm, virtio_if_t * vif,
1630 : int gso_enabled,
1631 : int checksum_offload_enabled,
1632 : int offloads_disabled)
1633 : {
1634 0 : if (vif->type != VIRTIO_IF_TYPE_PCI)
1635 0 : return VNET_API_ERROR_INVALID_INTERFACE;
1636 :
1637 0 : if (gso_enabled)
1638 0 : virtio_pci_offloads (vm, vif, 1, 0);
1639 0 : else if (checksum_offload_enabled)
1640 0 : virtio_pci_offloads (vm, vif, 0, 1);
1641 0 : else if (offloads_disabled)
1642 0 : virtio_pci_offloads (vm, vif, 0, 0);
1643 :
1644 0 : return 0;
1645 : }
1646 :
1647 : /*
1648 : * fd.io coding-style-patch-verification: ON
1649 : *
1650 : * Local Variables:
1651 : * eval: (c-set-style "gnu")
1652 : * End:
1653 : */
|
