Line data Source code
1 : /*
2 : * Copyright (c) 2018 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #include <vppinfra/types.h>
17 : #include <vlib/vlib.h>
18 : #include <vlib/pci/pci.h>
19 : #include <vnet/ethernet/ethernet.h>
20 : #include <vnet/plugin/plugin.h>
21 : #include <vpp/app/version.h>
22 : #include <vnet/interface/rx_queue_funcs.h>
23 : #include <vnet/interface/tx_queue_funcs.h>
24 : #include <vmxnet3/vmxnet3.h>
25 :
26 : #define PCI_VENDOR_ID_VMWARE 0x15ad
27 : #define PCI_DEVICE_ID_VMWARE_VMXNET3 0x07b0
28 :
29 : vmxnet3_main_t vmxnet3_main;
30 :
31 : static pci_device_id_t vmxnet3_pci_device_ids[] = {
32 : {
33 : .vendor_id = PCI_VENDOR_ID_VMWARE,
34 : .device_id = PCI_DEVICE_ID_VMWARE_VMXNET3},
35 : {0},
36 : };
37 :
38 : static clib_error_t *
39 0 : vmxnet3_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
40 : u32 flags)
41 : {
42 0 : vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
43 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
44 0 : vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, hi->dev_instance);
45 0 : uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
46 :
47 0 : if (vd->flags & VMXNET3_DEVICE_F_ERROR)
48 0 : return clib_error_return (0, "device is in error state");
49 :
50 0 : if (is_up)
51 : {
52 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
53 : VNET_HW_INTERFACE_FLAG_LINK_UP);
54 0 : vd->flags |= VMXNET3_DEVICE_F_ADMIN_UP;
55 : }
56 : else
57 : {
58 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
59 0 : vd->flags &= ~VMXNET3_DEVICE_F_ADMIN_UP;
60 : }
61 0 : return 0;
62 : }
63 :
64 : static clib_error_t *
65 0 : vmxnet3_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
66 : vnet_hw_if_rx_mode mode)
67 : {
68 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
69 0 : vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
70 0 : vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
71 0 : vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
72 : vmxnet3_per_thread_data_t *ptd;
73 :
74 0 : if (mode == rxq->mode)
75 0 : return 0;
76 0 : if ((mode != VNET_HW_IF_RX_MODE_POLLING) &&
77 : (mode != VNET_HW_IF_RX_MODE_INTERRUPT))
78 0 : return clib_error_return (0, "Rx mode %U not supported",
79 : format_vnet_hw_if_rx_mode, mode);
80 0 : rxq->mode = mode;
81 0 : ptd = vec_elt_at_index (vmxm->per_thread_data, rxq->thread_index);
82 0 : if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING)
83 0 : ptd->polling_q_count++;
84 : else
85 : {
86 0 : ASSERT (ptd->polling_q_count != 0);
87 0 : ptd->polling_q_count--;
88 : }
89 :
90 0 : return 0;
91 : }
92 :
93 : static void
94 0 : vmxnet3_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
95 : u32 node_index)
96 : {
97 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
98 0 : vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
99 0 : vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
100 :
101 : /* Shut off redirection */
102 0 : if (node_index == ~0)
103 : {
104 0 : vd->per_interface_next_index = node_index;
105 0 : return;
106 : }
107 :
108 0 : vd->per_interface_next_index =
109 0 : vlib_node_add_next (vlib_get_main (), vmxnet3_input_node.index,
110 : node_index);
111 : }
112 :
113 : static void
114 0 : vmxnet3_clear_hw_interface_counters (u32 instance)
115 : {
116 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
117 0 : vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, instance);
118 0 : vmxnet3_tx_queue *tx = VMXNET3_TX_START (vd);
119 0 : vmxnet3_rx_queue *rx = VMXNET3_RX_START (vd);
120 : u16 qid;
121 :
122 : /*
123 : * Set the "last_cleared_stats" to the current stats, so that
124 : * things appear to clear from a display perspective.
125 : */
126 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
127 :
128 0 : vec_foreach_index (qid, vd->txqs)
129 : {
130 0 : vmxnet3_tx_stats *txs = vec_elt_at_index (vd->tx_stats, qid);
131 0 : clib_memcpy (txs, &tx->stats, sizeof (*txs));
132 0 : tx++;
133 : }
134 0 : vec_foreach_index (qid, vd->rxqs)
135 : {
136 0 : vmxnet3_rx_stats *rxs = vec_elt_at_index (vd->rx_stats, qid);
137 0 : clib_memcpy (rxs, &rx->stats, sizeof (*rxs));
138 0 : rx++;
139 : }
140 0 : }
141 :
142 : static char *vmxnet3_tx_func_error_strings[] = {
143 : #define _(n,s) s,
144 : foreach_vmxnet3_tx_func_error
145 : #undef _
146 : };
147 :
148 : /* *INDENT-OFF* */
149 1679 : VNET_DEVICE_CLASS (vmxnet3_device_class,) =
150 : {
151 : .name = "VMXNET3 interface",
152 : .format_device = format_vmxnet3_device,
153 : .format_device_name = format_vmxnet3_device_name,
154 : .admin_up_down_function = vmxnet3_interface_admin_up_down,
155 : .clear_counters = vmxnet3_clear_hw_interface_counters,
156 : .rx_mode_change_function = vmxnet3_interface_rx_mode_change,
157 : .rx_redirect_to_node = vmxnet3_set_interface_next_node,
158 : .tx_function_n_errors = VMXNET3_TX_N_ERROR,
159 : .tx_function_error_strings = vmxnet3_tx_func_error_strings,
160 : };
161 : /* *INDENT-ON* */
162 :
163 : static u32
164 0 : vmxnet3_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
165 : {
166 0 : return 0;
167 : }
168 :
169 : static void
170 0 : vmxnet3_write_mac (vmxnet3_device_t * vd)
171 : {
172 : u32 val;
173 :
174 0 : memcpy (&val, vd->mac_addr, 4);
175 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACL, val);
176 :
177 0 : val = 0;
178 0 : memcpy (&val, vd->mac_addr + 4, 2);
179 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACH, val);
180 0 : }
181 :
182 : static clib_error_t *
183 0 : vmxnet3_provision_driver_shared (vlib_main_t * vm, vmxnet3_device_t * vd)
184 : {
185 : vmxnet3_shared *shared;
186 : u64 shared_dma;
187 : u16 qid, rid;
188 0 : vmxnet3_tx_queue *tx = VMXNET3_TX_START (vd);
189 0 : vmxnet3_rx_queue *rx = VMXNET3_RX_START (vd);
190 :
191 0 : vd->driver_shared =
192 0 : vlib_physmem_alloc_aligned_on_numa (vm, sizeof (*vd->driver_shared), 512,
193 : vd->numa_node);
194 0 : if (vd->driver_shared == 0)
195 0 : return vlib_physmem_last_error (vm);
196 :
197 0 : clib_memset (vd->driver_shared, 0, sizeof (*vd->driver_shared));
198 :
199 0 : vec_foreach_index (qid, vd->txqs)
200 : {
201 0 : vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
202 :
203 0 : tx->cfg.desc_address = vmxnet3_dma_addr (vm, vd, txq->tx_desc);
204 0 : tx->cfg.comp_address = vmxnet3_dma_addr (vm, vd, txq->tx_comp);
205 0 : tx->cfg.num_desc = txq->size;
206 0 : tx->cfg.num_comp = txq->size;
207 0 : tx++;
208 : }
209 :
210 0 : vec_foreach_index (qid, vd->rxqs)
211 : {
212 0 : vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
213 :
214 0 : for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
215 : {
216 0 : rx->cfg.desc_address[rid] = vmxnet3_dma_addr (vm, vd,
217 0 : rxq->rx_desc[rid]);
218 0 : rx->cfg.num_desc[rid] = rxq->size;
219 : }
220 0 : rx->cfg.comp_address = vmxnet3_dma_addr (vm, vd, rxq->rx_comp);
221 0 : rx->cfg.num_comp = rxq->size;
222 0 : rx->cfg.intr_index = qid;
223 0 : rx++;
224 : }
225 :
226 0 : shared = vd->driver_shared;
227 0 : shared->magic = VMXNET3_SHARED_MAGIC;
228 0 : shared->misc.version = VMXNET3_VERSION_MAGIC;
229 : if (sizeof (void *) == 4)
230 : shared->misc.guest_info = VMXNET3_GOS_BITS_32;
231 : else
232 0 : shared->misc.guest_info = VMXNET3_GOS_BITS_64;
233 0 : shared->misc.guest_info |= VMXNET3_GOS_TYPE_LINUX;
234 0 : shared->misc.version_support = VMXNET3_VERSION_SELECT;
235 0 : shared->misc.upt_features = VMXNET3_F_RXCSUM;
236 0 : if (vd->gso_enable)
237 0 : shared->misc.upt_features |= VMXNET3_F_LRO;
238 0 : if (vd->num_rx_queues > 1)
239 : {
240 0 : shared->misc.upt_features |= VMXNET3_F_RSS;
241 0 : shared->rss.version = 1;
242 0 : shared->rss.address = vmxnet3_dma_addr (vm, vd, vd->rss);
243 0 : shared->rss.length = sizeof (*vd->rss);
244 : }
245 0 : shared->misc.max_num_rx_sg = 0;
246 0 : shared->misc.upt_version_support = VMXNET3_UPT_VERSION_SELECT;
247 0 : shared->misc.queue_desc_address = vmxnet3_dma_addr (vm, vd, vd->queues);
248 0 : shared->misc.queue_desc_len = sizeof (*tx) * vd->num_tx_queues +
249 0 : sizeof (*rx) * vd->num_rx_queues;
250 0 : shared->misc.mtu = VMXNET3_MTU;
251 0 : shared->misc.num_tx_queues = vd->num_tx_queues;
252 0 : shared->misc.num_rx_queues = vd->num_rx_queues;
253 0 : shared->interrupt.num_intrs = vd->num_intrs;
254 0 : shared->interrupt.event_intr_index = vd->num_rx_queues;
255 0 : shared->interrupt.control = VMXNET3_IC_DISABLE_ALL;
256 0 : shared->rx_filter.mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST |
257 : VMXNET3_RXMODE_ALL_MULTI | VMXNET3_RXMODE_PROMISC;
258 0 : shared_dma = vmxnet3_dma_addr (vm, vd, shared);
259 :
260 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAL, shared_dma);
261 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAH, shared_dma >> 32);
262 :
263 0 : return 0;
264 : }
265 :
266 : static inline void
267 0 : vmxnet3_enable_interrupt (vmxnet3_device_t * vd)
268 : {
269 : int i;
270 0 : vmxnet3_shared *shared = vd->driver_shared;
271 :
272 0 : shared->interrupt.control &= ~VMXNET3_IC_DISABLE_ALL;
273 0 : for (i = 0; i < vd->num_intrs; i++)
274 0 : vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 0);
275 0 : }
276 :
277 : static inline void
278 0 : vmxnet3_disable_interrupt (vmxnet3_device_t * vd)
279 : {
280 : int i;
281 0 : vmxnet3_shared *shared = vd->driver_shared;
282 :
283 0 : shared->interrupt.control |= VMXNET3_IC_DISABLE_ALL;
284 0 : for (i = 0; i < vd->num_intrs; i++)
285 0 : vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 1);
286 0 : }
287 :
288 : static clib_error_t *
289 0 : vmxnet3_rxq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
290 : {
291 : vmxnet3_rxq_t *rxq;
292 : vmxnet3_rx_stats *rxs;
293 : u16 rid;
294 :
295 0 : vec_validate (vd->rx_stats, qid);
296 0 : rxs = vec_elt_at_index (vd->rx_stats, qid);
297 0 : clib_memset (rxs, 0, sizeof (*rxs));
298 :
299 0 : vec_validate_aligned (vd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
300 0 : rxq = vec_elt_at_index (vd->rxqs, qid);
301 0 : clib_memset (rxq, 0, sizeof (*rxq));
302 0 : rxq->size = qsz;
303 0 : rxq->mode = VNET_HW_IF_RX_MODE_POLLING;
304 0 : for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
305 : {
306 0 : rxq->rx_desc[rid] = vlib_physmem_alloc_aligned_on_numa
307 : (vm, qsz * sizeof (*rxq->rx_desc[rid]), 512, vd->numa_node);
308 :
309 0 : if (rxq->rx_desc[rid] == 0)
310 0 : return vlib_physmem_last_error (vm);
311 :
312 0 : clib_memset (rxq->rx_desc[rid], 0, qsz * sizeof (*rxq->rx_desc[rid]));
313 : }
314 0 : rxq->rx_comp =
315 0 : vlib_physmem_alloc_aligned_on_numa (vm, qsz * sizeof (*rxq->rx_comp), 512,
316 : vd->numa_node);
317 0 : if (rxq->rx_comp == 0)
318 0 : return vlib_physmem_last_error (vm);
319 :
320 0 : clib_memset (rxq->rx_comp, 0, qsz * sizeof (*rxq->rx_comp));
321 0 : for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
322 : {
323 : vmxnet3_rx_ring *ring;
324 :
325 0 : ring = &rxq->rx_ring[rid];
326 0 : ring->gen = VMXNET3_RXF_GEN;
327 0 : ring->rid = rid;
328 0 : vec_validate_aligned (ring->bufs, rxq->size, CLIB_CACHE_LINE_BYTES);
329 : }
330 0 : rxq->rx_comp_ring.gen = VMXNET3_RXCF_GEN;
331 :
332 0 : return 0;
333 : }
334 :
335 : static clib_error_t *
336 0 : vmxnet3_txq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
337 : {
338 : vmxnet3_txq_t *txq;
339 : vmxnet3_tx_stats *txs;
340 : u32 size;
341 :
342 0 : vec_validate_aligned (vd->txqs, qid, CLIB_CACHE_LINE_BYTES);
343 0 : txq = vec_elt_at_index (vd->txqs, qid);
344 0 : clib_memset (txq, 0, sizeof (*txq));
345 0 : clib_spinlock_init (&txq->lock);
346 :
347 0 : vec_validate (vd->tx_stats, qid);
348 0 : txs = vec_elt_at_index (vd->tx_stats, qid);
349 0 : clib_memset (txs, 0, sizeof (*txs));
350 :
351 0 : txq->size = qsz;
352 0 : txq->reg_txprod = qid * 8 + VMXNET3_REG_TXPROD;
353 :
354 0 : size = qsz * sizeof (*txq->tx_desc);
355 0 : txq->tx_desc =
356 0 : vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
357 0 : if (txq->tx_desc == 0)
358 0 : return vlib_physmem_last_error (vm);
359 :
360 0 : clib_memset (txq->tx_desc, 0, size);
361 :
362 0 : size = qsz * sizeof (*txq->tx_comp);
363 0 : txq->tx_comp =
364 0 : vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
365 0 : if (txq->tx_comp == 0)
366 0 : return vlib_physmem_last_error (vm);
367 :
368 0 : clib_memset (txq->tx_comp, 0, size);
369 0 : vec_validate_aligned (txq->tx_ring.bufs, txq->size, CLIB_CACHE_LINE_BYTES);
370 0 : txq->tx_ring.gen = VMXNET3_TXF_GEN;
371 0 : txq->tx_comp_ring.gen = VMXNET3_TXCF_GEN;
372 :
373 0 : return 0;
374 : }
375 :
376 : static const u8 vmxnet3_rss_key[VMXNET3_RSS_MAX_KEY_SZ] = {
377 : 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
378 : 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
379 : 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
380 : 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
381 : 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
382 : };
383 :
384 : static clib_error_t *
385 0 : vmxnet3_rss_init (vlib_main_t * vm, vmxnet3_device_t * vd)
386 : {
387 : vmxnet3_rss_shared *rss;
388 0 : size_t size = sizeof (*rss);
389 : u8 i;
390 :
391 0 : vd->rss = vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
392 0 : if (vd->rss == 0)
393 0 : return vlib_physmem_last_error (vm);
394 :
395 0 : clib_memset (vd->rss, 0, size);
396 0 : rss = vd->rss;
397 0 : rss->hash_type =
398 : VMXNET3_RSS_HASH_TYPE_IPV4 | VMXNET3_RSS_HASH_TYPE_TCP_IPV4 |
399 : VMXNET3_RSS_HASH_TYPE_IPV6 | VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
400 0 : rss->hash_func = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
401 0 : rss->hash_key_sz = VMXNET3_RSS_MAX_KEY_SZ;
402 0 : rss->ind_table_sz = VMXNET3_RSS_MAX_IND_TABLE_SZ;
403 0 : clib_memcpy (rss->hash_key, vmxnet3_rss_key, VMXNET3_RSS_MAX_KEY_SZ);
404 0 : for (i = 0; i < rss->ind_table_sz; i++)
405 0 : rss->ind_table[i] = i % vd->num_rx_queues;
406 :
407 0 : return 0;
408 : }
409 :
410 : static clib_error_t *
411 0 : vmxnet3_device_init (vlib_main_t * vm, vmxnet3_device_t * vd,
412 : vmxnet3_create_if_args_t * args)
413 : {
414 0 : clib_error_t *error = 0;
415 : u32 ret, i, size;
416 :
417 : /* Quiesce the device */
418 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
419 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
420 0 : if (ret != 0)
421 : {
422 0 : error = clib_error_return (0, "error on quiescing device rc (%u)", ret);
423 0 : return error;
424 : }
425 :
426 : /* Reset the device */
427 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
428 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
429 0 : if (ret != 0)
430 : {
431 0 : error = clib_error_return (0, "error on resetting device rc (%u)", ret);
432 0 : return error;
433 : }
434 :
435 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_VRRS);
436 0 : vd->version = count_leading_zeros (ret);
437 0 : vd->version = uword_bits - vd->version;
438 :
439 0 : if (vd->version == 0)
440 : {
441 0 : error = clib_error_return (0, "unsupported hardware version %u",
442 : vd->version);
443 0 : return error;
444 : }
445 :
446 : /* cap support version to 3 */
447 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_VRRS,
448 0 : 1 << (clib_min (3, vd->version) - 1));
449 :
450 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_UVRS);
451 0 : if (ret & 1)
452 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_UVRS, 1);
453 : else
454 : {
455 0 : error = clib_error_return (0, "unsupported upt version %u", ret);
456 0 : return error;
457 : }
458 :
459 : /* GSO is only supported for version >= 3 */
460 0 : if (args->enable_gso)
461 : {
462 0 : if (vd->version >= 3)
463 0 : vd->gso_enable = 1;
464 : else
465 : {
466 : error =
467 0 : clib_error_return (0,
468 : "GSO is not supported because hardware version"
469 : " is %u. It must be >= 3", vd->version);
470 0 : return error;
471 : }
472 : }
473 :
474 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
475 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
476 0 : if (ret & 1)
477 : {
478 0 : vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
479 0 : vd->link_speed = ret >> 16;
480 : }
481 : else
482 0 : vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
483 :
484 : /* Get the mac address */
485 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACL);
486 0 : clib_memcpy (vd->mac_addr, &ret, 4);
487 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACH);
488 0 : clib_memcpy (vd->mac_addr + 4, &ret, 2);
489 :
490 0 : size = sizeof (vmxnet3_rx_queue) * vd->num_rx_queues +
491 0 : sizeof (vmxnet3_tx_queue) * vd->num_tx_queues;
492 :
493 0 : vd->queues =
494 0 : vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
495 0 : if (vd->queues == 0)
496 0 : return vlib_physmem_last_error (vm);
497 :
498 0 : clib_memset (vd->queues, 0, size);
499 :
500 0 : if (vd->num_rx_queues > 1)
501 : {
502 0 : error = vmxnet3_rss_init (vm, vd);
503 0 : if (error)
504 0 : return error;
505 : }
506 :
507 0 : for (i = 0; i < vd->num_rx_queues; i++)
508 : {
509 0 : error = vmxnet3_rxq_init (vm, vd, i, args->rxq_size);
510 0 : if (error)
511 0 : return error;
512 : }
513 :
514 0 : for (i = 0; i < vd->num_tx_queues; i++)
515 : {
516 0 : error = vmxnet3_txq_init (vm, vd, i, args->txq_size);
517 0 : if (error)
518 0 : return error;
519 : }
520 :
521 0 : error = vmxnet3_provision_driver_shared (vm, vd);
522 0 : if (error)
523 0 : return error;
524 :
525 0 : vmxnet3_write_mac (vd);
526 :
527 : /* Activate device */
528 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
529 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
530 0 : if (ret != 0)
531 : {
532 : error =
533 0 : clib_error_return (0, "error on activating device rc (%u)", ret);
534 0 : return error;
535 : }
536 :
537 0 : return error;
538 : }
539 :
540 : static void
541 0 : vmxnet3_rxq_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
542 : {
543 0 : vnet_main_t *vnm = vnet_get_main ();
544 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
545 0 : uword pd = vlib_pci_get_private_data (vm, h);
546 0 : vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
547 0 : u16 qid = line;
548 0 : vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
549 :
550 0 : if (vec_len (vd->rxqs) > qid && (rxq->mode != VNET_HW_IF_RX_MODE_POLLING))
551 : {
552 0 : vmxnet3_per_thread_data_t *ptd =
553 0 : vec_elt_at_index (vmxm->per_thread_data, rxq->thread_index);
554 0 : if (ptd->polling_q_count == 0)
555 0 : vnet_hw_if_rx_queue_set_int_pending (vnm, rxq->queue_index);
556 : }
557 0 : }
558 :
559 : static void
560 0 : vmxnet3_event_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
561 : u16 line)
562 : {
563 0 : vnet_main_t *vnm = vnet_get_main ();
564 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
565 0 : uword pd = vlib_pci_get_private_data (vm, h);
566 0 : vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
567 : u32 ret;
568 :
569 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
570 0 : ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
571 0 : if (ret & 1)
572 : {
573 0 : vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
574 0 : vd->link_speed = ret >> 16;
575 0 : vnet_hw_interface_set_link_speed (
576 : vnm, vd->hw_if_index,
577 0 : (vd->link_speed == UINT32_MAX) ? UINT32_MAX : vd->link_speed * 1000);
578 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
579 : VNET_HW_INTERFACE_FLAG_LINK_UP);
580 : }
581 : else
582 : {
583 0 : vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
584 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
585 : }
586 0 : }
587 :
588 : static u8
589 0 : vmxnet3_queue_size_valid (u16 qsz)
590 : {
591 0 : if (qsz < 64 || qsz > 4096)
592 0 : return 0;
593 0 : if ((qsz % 64) != 0)
594 0 : return 0;
595 0 : return 1;
596 : }
597 :
598 : static u8
599 0 : vmxnet3_tx_queue_num_valid (u16 num)
600 : {
601 0 : vlib_thread_main_t *tm = vlib_get_thread_main ();
602 :
603 0 : if ((num > VMXNET3_TXQ_MAX) || (num > tm->n_vlib_mains))
604 0 : return 0;
605 0 : return 1;
606 : }
607 :
608 : static u8
609 0 : vmxnet3_rx_queue_num_valid (u16 num)
610 : {
611 0 : if (num > VMXNET3_RXQ_MAX)
612 0 : return 0;
613 0 : return 1;
614 : }
615 :
616 : void
617 0 : vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
618 : {
619 0 : vnet_main_t *vnm = vnet_get_main ();
620 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
621 0 : vnet_eth_interface_registration_t eir = {};
622 :
623 : vmxnet3_device_t *vd;
624 : vlib_pci_dev_handle_t h;
625 0 : vnet_hw_if_caps_change_t cc = {};
626 0 : clib_error_t *error = 0;
627 : u16 qid;
628 : u32 num_intr;
629 :
630 0 : if (args->txq_num == 0)
631 0 : args->txq_num = 1;
632 0 : if (args->rxq_num == 0)
633 0 : args->rxq_num = 1;
634 0 : if (!vmxnet3_rx_queue_num_valid (args->rxq_num))
635 : {
636 0 : args->rv = VNET_API_ERROR_INVALID_VALUE;
637 0 : args->error =
638 0 : clib_error_return (error, "number of rx queues must be <= %u",
639 : VMXNET3_RXQ_MAX);
640 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
641 : format_vlib_pci_addr, &args->addr,
642 : "number of rx queues must be <= %u", VMXNET3_RXQ_MAX);
643 0 : return;
644 : }
645 :
646 0 : if (!vmxnet3_tx_queue_num_valid (args->txq_num))
647 : {
648 0 : args->rv = VNET_API_ERROR_INVALID_VALUE;
649 0 : args->error =
650 0 : clib_error_return (error,
651 : "number of tx queues must be <= %u and <= number of "
652 : "CPU's assigned to VPP", VMXNET3_TXQ_MAX);
653 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
654 : format_vlib_pci_addr, &args->addr,
655 : "number of tx queues must be <= %u and <= number of "
656 : "CPU's assigned to VPP", VMXNET3_TXQ_MAX);
657 0 : return;
658 : }
659 0 : if (args->rxq_size == 0)
660 0 : args->rxq_size = VMXNET3_NUM_RX_DESC;
661 0 : if (args->txq_size == 0)
662 0 : args->txq_size = VMXNET3_NUM_TX_DESC;
663 :
664 0 : if (!vmxnet3_queue_size_valid (args->rxq_size) ||
665 0 : !vmxnet3_queue_size_valid (args->txq_size))
666 : {
667 0 : args->rv = VNET_API_ERROR_INVALID_VALUE;
668 0 : args->error =
669 0 : clib_error_return (error,
670 : "queue size must be <= 4096, >= 64, "
671 : "and multiples of 64");
672 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
673 : format_vlib_pci_addr, &args->addr,
674 : "queue size must be <= 4096, >= 64, and multiples of 64");
675 0 : return;
676 : }
677 :
678 : /* *INDENT-OFF* */
679 0 : pool_foreach (vd, vmxm->devices) {
680 0 : if (vd->pci_addr.as_u32 == args->addr.as_u32)
681 : {
682 0 : args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
683 0 : args->error =
684 0 : clib_error_return (error, "%U: %s", format_vlib_pci_addr,
685 : &args->addr, "pci address in use");
686 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
687 : format_vlib_pci_addr, &args->addr, "pci address in use");
688 0 : return;
689 : }
690 : }
691 : /* *INDENT-ON* */
692 :
693 0 : if (args->bind)
694 : {
695 0 : error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto",
696 0 : VMXNET3_BIND_FORCE == args->bind);
697 0 : if (error)
698 : {
699 0 : args->rv = VNET_API_ERROR_INVALID_INTERFACE;
700 0 : args->error =
701 0 : clib_error_return (error, "%U: %s", format_vlib_pci_addr,
702 : &args->addr,
703 : "error encountered on binding pci device");
704 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
705 : format_vlib_pci_addr, &args->addr,
706 : "error encountered on binding pci devicee");
707 0 : return;
708 : }
709 : }
710 :
711 0 : if ((error =
712 0 : vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
713 : {
714 0 : args->rv = VNET_API_ERROR_INVALID_INTERFACE;
715 0 : args->error =
716 0 : clib_error_return (error, "%U: %s", format_vlib_pci_addr,
717 : &args->addr,
718 : "error encountered on pci device open");
719 0 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
720 : format_vlib_pci_addr, &args->addr,
721 : "error encountered on pci device open");
722 0 : return;
723 : }
724 :
725 : /*
726 : * Do not use vmxnet3_log_error prior to this line since the macro
727 : * references vd->pci_dev_handle
728 : */
729 0 : pool_get (vmxm->devices, vd);
730 0 : vd->num_tx_queues = args->txq_num;
731 0 : vd->num_rx_queues = args->rxq_num;
732 0 : vd->dev_instance = vd - vmxm->devices;
733 0 : vd->per_interface_next_index = ~0;
734 0 : vd->pci_addr = args->addr;
735 :
736 0 : if (args->enable_elog)
737 0 : vd->flags |= VMXNET3_DEVICE_F_ELOG;
738 :
739 0 : vd->pci_dev_handle = h;
740 0 : vd->numa_node = vlib_pci_get_numa_node (vm, h);
741 0 : vd->num_intrs = vd->num_rx_queues + 1; // +1 for the event interrupt
742 :
743 0 : vlib_pci_set_private_data (vm, h, vd->dev_instance);
744 :
745 0 : if ((error = vlib_pci_bus_master_enable (vm, h)))
746 : {
747 0 : vmxnet3_log_error (vd, "error encountered on pci bus master enable");
748 0 : goto error;
749 : }
750 :
751 0 : if ((error = vlib_pci_map_region (vm, h, 0, (void **) &vd->bar[0])))
752 : {
753 0 : vmxnet3_log_error (vd, "error encountered on pci map region for bar 0");
754 0 : goto error;
755 : }
756 :
757 0 : if ((error = vlib_pci_map_region (vm, h, 1, (void **) &vd->bar[1])))
758 : {
759 0 : vmxnet3_log_error (vd, "error encountered on pci map region for bar 1");
760 0 : goto error;
761 : }
762 :
763 0 : num_intr = vlib_pci_get_num_msix_interrupts (vm, h);
764 0 : if (num_intr < vd->num_rx_queues + 1)
765 : {
766 0 : vmxnet3_log_error (vd,
767 : "No sufficient interrupt lines (%u) for rx queues",
768 : num_intr);
769 : error =
770 0 : clib_error_return (0,
771 : "No sufficient interrupt lines (%u) for rx queues",
772 : num_intr);
773 0 : goto error;
774 : }
775 0 : if ((error = vlib_pci_register_msix_handler (vm, h, 0, vd->num_rx_queues,
776 : &vmxnet3_rxq_irq_handler)))
777 : {
778 0 : vmxnet3_log_error (vd,
779 : "error encountered on pci register msix handler 0");
780 0 : goto error;
781 : }
782 :
783 0 : if ((error = vlib_pci_register_msix_handler (vm, h, vd->num_rx_queues, 1,
784 : &vmxnet3_event_irq_handler)))
785 : {
786 0 : vmxnet3_log_error (vd,
787 : "error encountered on pci register msix handler 1");
788 0 : goto error;
789 : }
790 :
791 0 : if ((error = vlib_pci_enable_msix_irq (vm, h, 0, vd->num_rx_queues + 1)))
792 : {
793 0 : vmxnet3_log_error (vd, "error encountered on pci enable msix irq");
794 0 : goto error;
795 : }
796 :
797 0 : if ((error = vlib_pci_intr_enable (vm, h)))
798 : {
799 0 : vmxnet3_log_error (vd, "error encountered on pci interrupt enable");
800 0 : goto error;
801 : }
802 :
803 0 : if ((error = vmxnet3_device_init (vm, vd, args)))
804 : {
805 0 : vmxnet3_log_error (vd, "error encountered on device init");
806 0 : goto error;
807 : }
808 :
809 : /* create interface */
810 0 : eir.dev_class_index = vmxnet3_device_class.index;
811 0 : eir.dev_instance = vd->dev_instance;
812 0 : eir.address = vd->mac_addr;
813 0 : eir.cb.flag_change = vmxnet3_flag_change;
814 0 : vd->hw_if_index = vnet_eth_register_interface (vnm, &eir);
815 :
816 0 : vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vd->hw_if_index);
817 0 : vd->sw_if_index = sw->sw_if_index;
818 0 : args->sw_if_index = sw->sw_if_index;
819 :
820 0 : cc.mask = VNET_HW_IF_CAP_INT_MODE | VNET_HW_IF_CAP_TCP_GSO |
821 : VNET_HW_IF_CAP_TX_TCP_CKSUM | VNET_HW_IF_CAP_TX_UDP_CKSUM;
822 0 : if (vd->gso_enable)
823 0 : cc.val = cc.mask;
824 : else
825 0 : cc.val = VNET_HW_IF_CAP_INT_MODE;
826 :
827 0 : vnet_hw_if_change_caps (vnm, vd->hw_if_index, &cc);
828 :
829 0 : vnet_hw_if_set_input_node (vnm, vd->hw_if_index, vmxnet3_input_node.index);
830 : /* Disable interrupts */
831 0 : vmxnet3_disable_interrupt (vd);
832 0 : vec_foreach_index (qid, vd->rxqs)
833 : {
834 0 : vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
835 : u32 qi, fi;
836 : vmxnet3_per_thread_data_t *ptd;
837 :
838 0 : qi = vnet_hw_if_register_rx_queue (vnm, vd->hw_if_index, qid,
839 : VNET_HW_IF_RXQ_THREAD_ANY);
840 0 : fi = vlib_pci_get_msix_file_index (vm, vd->pci_dev_handle, qid);
841 0 : vnet_hw_if_set_rx_queue_file_index (vnm, qi, fi);
842 0 : rxq->queue_index = qi;
843 0 : rxq->thread_index =
844 0 : vnet_hw_if_get_rx_queue_thread_index (vnm, rxq->queue_index);
845 0 : if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING)
846 : {
847 0 : ptd = vec_elt_at_index (vmxm->per_thread_data, rxq->thread_index);
848 0 : ptd->polling_q_count++;
849 : }
850 0 : rxq->buffer_pool_index =
851 0 : vnet_hw_if_get_rx_queue_numa_node (vnm, rxq->queue_index);
852 0 : vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
853 0 : vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
854 : }
855 :
856 0 : vec_foreach_index (qid, vd->txqs)
857 : {
858 0 : vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
859 0 : txq->queue_index =
860 0 : vnet_hw_if_register_tx_queue (vnm, vd->hw_if_index, qid);
861 : }
862 0 : for (u32 i = 0; i < vlib_get_n_threads (); i++)
863 : {
864 0 : u32 qi = vd->txqs[i % vd->num_tx_queues].queue_index;
865 0 : vnet_hw_if_tx_queue_assign_thread (vnm, qi, i);
866 : }
867 0 : vnet_hw_if_update_runtime_data (vnm, vd->hw_if_index);
868 :
869 0 : vd->flags |= VMXNET3_DEVICE_F_INITIALIZED;
870 0 : vmxnet3_enable_interrupt (vd);
871 :
872 0 : vnet_hw_interface_set_link_speed (
873 0 : vnm, vd->hw_if_index,
874 0 : (vd->link_speed == UINT32_MAX) ? UINT32_MAX : vd->link_speed * 1000);
875 0 : if (vd->flags & VMXNET3_DEVICE_F_LINK_UP)
876 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
877 : VNET_HW_INTERFACE_FLAG_LINK_UP);
878 : else
879 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
880 0 : return;
881 :
882 0 : error:
883 0 : vmxnet3_delete_if (vm, vd);
884 0 : args->rv = VNET_API_ERROR_INVALID_INTERFACE;
885 0 : args->error = error;
886 : }
887 :
888 : void
889 0 : vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * vd)
890 : {
891 0 : vnet_main_t *vnm = vnet_get_main ();
892 0 : vmxnet3_main_t *vmxm = &vmxnet3_main;
893 : u32 i, bi;
894 : u16 desc_idx;
895 :
896 : /* Quiesce the device */
897 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
898 :
899 : /* Reset the device */
900 0 : vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
901 :
902 0 : if (vd->hw_if_index)
903 : {
904 0 : vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
905 0 : ethernet_delete_interface (vnm, vd->hw_if_index);
906 : }
907 :
908 0 : vlib_pci_device_close (vm, vd->pci_dev_handle);
909 :
910 : /* *INDENT-OFF* */
911 0 : vec_foreach_index (i, vd->rxqs)
912 : {
913 0 : vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
914 0 : u16 mask = rxq->size - 1;
915 : u16 rid;
916 0 : vmxnet3_per_thread_data_t *ptd =
917 0 : vec_elt_at_index (vmxm->per_thread_data, rxq->thread_index);
918 :
919 0 : if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING)
920 : {
921 0 : ASSERT (ptd->polling_q_count != 0);
922 0 : ptd->polling_q_count--;
923 : }
924 0 : for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
925 : {
926 : vmxnet3_rx_ring *ring;
927 :
928 0 : ring = &rxq->rx_ring[rid];
929 0 : desc_idx = (ring->consume + 1) & mask;
930 0 : vlib_buffer_free_from_ring (vm, ring->bufs, desc_idx, rxq->size,
931 0 : ring->fill);
932 0 : vec_free (ring->bufs);
933 0 : vlib_physmem_free (vm, rxq->rx_desc[rid]);
934 : }
935 0 : vlib_physmem_free (vm, rxq->rx_comp);
936 : }
937 : /* *INDENT-ON* */
938 0 : vec_free (vd->rxqs);
939 0 : vec_free (vd->rx_stats);
940 :
941 : /* *INDENT-OFF* */
942 0 : vec_foreach_index (i, vd->txqs)
943 : {
944 0 : vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, i);
945 0 : u16 mask = txq->size - 1;
946 : u16 end_idx;
947 :
948 0 : desc_idx = txq->tx_ring.consume;
949 0 : end_idx = txq->tx_ring.produce;
950 0 : while (desc_idx != end_idx)
951 : {
952 0 : bi = txq->tx_ring.bufs[desc_idx];
953 0 : vlib_buffer_free_no_next (vm, &bi, 1);
954 0 : desc_idx++;
955 0 : desc_idx &= mask;
956 : }
957 0 : clib_spinlock_free (&txq->lock);
958 0 : vec_free (txq->tx_ring.bufs);
959 0 : vlib_physmem_free (vm, txq->tx_desc);
960 0 : vlib_physmem_free (vm, txq->tx_comp);
961 : }
962 : /* *INDENT-ON* */
963 0 : vec_free (vd->txqs);
964 0 : vec_free (vd->tx_stats);
965 :
966 0 : vlib_physmem_free (vm, vd->driver_shared);
967 0 : vlib_physmem_free (vm, vd->queues);
968 0 : vlib_physmem_free (vm, vd->rss);
969 :
970 0 : clib_error_free (vd->error);
971 0 : clib_memset (vd, 0, sizeof (*vd));
972 0 : pool_put (vmxm->devices, vd);
973 :
974 0 : }
975 :
976 : /*
977 : * fd.io coding-style-patch-verification: ON
978 : *
979 : * Local Variables:
980 : * eval: (c-set-style "gnu")
981 : * End:
982 : */
|
