Line data Source code
1 : /*
2 : * Copyright (c) 2018 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 :
16 : #ifndef __included_vmnet_vmnet_h__
17 : #define __included_vmnet_vmnet_h__
18 :
19 : #define foreach_vmxnet3_tx_func_error \
20 : _(ERROR_PACKETS, "error packets") \
21 : _(LINK_DOWN, "link down") \
22 : _(NO_FREE_SLOTS, "no free tx slots")
23 :
24 : typedef enum
25 : {
26 : #define _(f,s) VMXNET3_TX_ERROR_##f,
27 : foreach_vmxnet3_tx_func_error
28 : #undef _
29 : VMXNET3_TX_N_ERROR,
30 : } vmxnet3_tx_func_error_t;
31 :
32 : #define foreach_vmxnet3_rxmode_flags \
33 : _(0, UCAST, "unicast") \
34 : _(1, MCAST, "multicast") \
35 : _(2, BCAST, "broadcast") \
36 : _(3, ALL_MULTI, "all multicast") \
37 : _(4, PROMISC, "promiscuous")
38 :
39 : enum
40 : {
41 : #define _(a, b, c) VMXNET3_RXMODE_##b = (1 << a),
42 : foreach_vmxnet3_rxmode_flags
43 : #undef _
44 : };
45 :
46 : #define foreach_vmxnet3_show_entry \
47 : _(RX_COMP, "rx comp") \
48 : _(RX_DESC0, "rx desc 0") \
49 : _(RX_DESC1, "rx desc 1") \
50 : _(TX_COMP, "tx comp") \
51 : _(TX_DESC, "tx desc")
52 :
53 : enum
54 : {
55 : #define _(a, b) VMXNET3_SHOW_##a,
56 : foreach_vmxnet3_show_entry
57 : #undef _
58 : };
59 :
60 : #define foreach_vmxnet3_feature_flags \
61 : _(0, RXCSUM, "rx checksum") \
62 : _(1, RSS, "RSS") \
63 : _(2, RXVLAN, "rx VLAN") \
64 : _(3, LRO, "LRO")
65 :
66 : enum
67 : {
68 : #define _(a, b, c) VMXNET3_F_##b = (1 << a),
69 : foreach_vmxnet3_feature_flags
70 : #undef _
71 : };
72 :
73 : #define foreach_vmxnet3_rss_hash_type \
74 : _(0, IPV4, "ipv4") \
75 : _(1, TCP_IPV4, "tcp ipv4") \
76 : _(2, IPV6, "ipv6") \
77 : _(3, TCP_IPV6, "tcp ipv6")
78 :
79 : enum
80 : {
81 : #define _(a, b, c) VMXNET3_RSS_HASH_TYPE_##b = (1 << a),
82 : foreach_vmxnet3_rss_hash_type
83 : #undef _
84 : };
85 :
86 : #define VMXNET3_RSS_HASH_FUNC_TOEPLITZ 1
87 : #define VMXNET3_RSS_MAX_KEY_SZ 40
88 : #define VMXNET3_RSS_MAX_IND_TABLE_SZ 128
89 :
90 : #define VMXNET3_TXQ_MAX 8
91 : #define VMXNET3_RXQ_MAX 16
92 : #define VMXNET3_TX_START(vd) ((vd)->queues)
93 : #define VMXNET3_RX_START(vd) \
94 : ((vd)->queues + (vd)->num_tx_queues * sizeof (vmxnet3_tx_queue))
95 :
96 : /* BAR 0 */
97 : #define VMXNET3_REG_IMR 0x0000 /* Interrupt Mask Register */
98 : #define VMXNET3_REG_TXPROD 0x0600 /* Tx Producer Index */
99 : #define VMXNET3_REG_RXPROD 0x0800 /* Rx Producer Index for ring 1 */
100 : #define VMXNET3_REG_RXPROD2 0x0A00 /* Rx Producer Index for ring 2 */
101 :
102 :
103 : /* BAR 1 */
104 : #define VMXNET3_REG_VRRS 0x0000 /* VMXNET3 Revision Report Selection */
105 : #define VMXNET3_REG_UVRS 0x0008 /* UPT Version Report Selection */
106 : #define VMXNET3_REG_DSAL 0x0010 /* Driver Shared Address Low */
107 : #define VMXNET3_REG_DSAH 0x0018 /* Driver Shared Address High */
108 : #define VMXNET3_REG_CMD 0x0020 /* Command */
109 : #define VMXNET3_REG_MACL 0x0028 /* MAC Address Low */
110 : #define VMXNET3_REG_MACH 0x0030 /* MAC Address High */
111 : #define VMXNET3_REG_ICR 0x0038 /* Interrupt Cause Register */
112 : #define VMXNET3_REG_ECR 0x0040 /* Event Cause Register */
113 :
114 : #define VMXNET3_VLAN_LEN 4
115 : #define VMXNET3_FCS_LEN 4
116 : #define VMXNET3_MTU (1514 + VMXNET3_VLAN_LEN + VMXNET3_FCS_LEN)
117 :
118 : #define VMXNET3_RXF_BTYPE (1 << 14) /* rx body buffer type */
119 : #define VMXNET3_RXF_GEN (1 << 31) /* rx generation */
120 :
121 : #define VMXNET3_RXCF_CKSUM_MASK (0xFFFF) /* rx checksum mask */
122 : #define VMXNET3_RXCF_TUC (1 << 16) /* rx udp/tcp checksum correct */
123 : #define VMXNET3_RXCF_UDP (1 << 17) /* rx udp packet */
124 : #define VMXNET3_RXCF_TCP (1 << 18) /* rx tcp packet */
125 : #define VMXNET3_RXCF_IPC (1 << 19) /* rx ip checksum correct */
126 : #define VMXNET3_RXCF_IP6 (1 << 20) /* rx ip6 packet */
127 : #define VMXNET3_RXCF_IP4 (1 << 21) /* rx ip4 packet */
128 : #define VMXNET3_RXCF_CT (0x7F << 24) /* rx completion type 24-30, 7 bits */
129 : #define VMXNET3_RXCF_GEN (1 << 31) /* rx completion generation */
130 :
131 : #define VMXNET3_RXC_INDEX (0xFFF) /* rx completion index mask */
132 :
133 : #define foreach_vmxnet3_offload \
134 : _(0, NONE, "none") \
135 : _(2, CSUM, "checksum") \
136 : _(3, TSO, "tso")
137 :
138 : enum
139 : {
140 : #define _(a, b, c) VMXNET3_OM_##b = (a),
141 : foreach_vmxnet3_offload
142 : #undef _
143 : };
144 :
145 : /* tx desc flag 0 */
146 : #define VMXNET3_TXF_GEN (1 << 14) /* tx generation */
147 :
148 : /* tx desc flag 1 */
149 : #define VMXNET3_TXF_OM(x) ((x) << 10) /* tx offload mode */
150 : #define VMXNET3_TXF_MSSCOF(x) ((x) << 18) /* tx MSS checksum offset, flags */
151 : #define VMXNET3_TXF_EOP (1 << 12) /* tx end of packet */
152 : #define VMXNET3_TXF_CQ (1 << 13) /* tx completion request */
153 :
154 : /* tx completion flag */
155 : #define VMXNET3_TXCF_GEN (1 << 31) /* tx completion generation */
156 : #define VMXNET3_TXC_INDEX (0xFFF) /* tx completion index mask */
157 :
158 : #define VMXNET3_RX_RING_SIZE 2
159 : #define VMXNET3_INPUT_REFILL_THRESHOLD 32
160 : #define VMXNET3_NUM_TX_DESC 1024
161 : #define VMXNET3_NUM_TX_COMP VMXNET3_NUM_TX_DESC
162 : #define VMXNET3_NUM_RX_DESC 1024
163 : #define VMXNET3_NUM_RX_COMP VMXNET3_NUM_RX_DESC
164 :
165 : #define VMXNET3_VERSION_MAGIC 0x69505845
166 : #define VMXNET3_SHARED_MAGIC 0xbabefee1
167 : #define VMXNET3_VERSION_SELECT 1
168 : #define VMXNET3_UPT_VERSION_SELECT 1
169 : #define VMXNET3_MAX_INTRS 25
170 : #define VMXNET3_IC_DISABLE_ALL 0x1
171 :
172 : #define VMXNET3_GOS_BITS_32 (1 << 0)
173 : #define VMXNET3_GOS_BITS_64 (2 << 0)
174 : #define VMXNET3_GOS_TYPE_LINUX (1 << 2)
175 : #define VMXNET3_RXCL_LEN_MASK (0x3FFF) // 14 bits
176 : #define VMXNET3_RXCL_ERROR (1 << 14)
177 :
178 : #define VMXNET3_RXCI_EOP (1 << 14) /* end of packet */
179 : #define VMXNET3_RXCI_SOP (1 << 15) /* start of packet */
180 : #define VMXNET3_RXCI_CNC (1 << 30) /* Checksum not calculated */
181 :
182 : #define VMXNET3_RXCOMP_TYPE (3 << 24) /* RX completion descriptor */
183 : #define VMXNET3_RXCOMP_TYPE_LRO (4 << 24) /* RX completion descriptor for LRO */
184 :
185 : #define VMXNET3_RXECF_MSS_MASK (0xFFFF) // 16 bits
186 :
187 : #define foreach_vmxnet3_device_flags \
188 : _(0, INITIALIZED, "initialized") \
189 : _(1, ERROR, "error") \
190 : _(2, ADMIN_UP, "admin-up") \
191 : _(3, IOVA, "iova") \
192 : _(4, LINK_UP, "link-up") \
193 : _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
194 : _(6, ELOG, "elog")
195 :
196 : enum
197 : {
198 : #define _(a, b, c) VMXNET3_DEVICE_F_##b = (1 << a),
199 : foreach_vmxnet3_device_flags
200 : #undef _
201 : };
202 :
203 : #define foreach_vmxnet3_set_cmds \
204 : _(0, ACTIVATE_DEV, "activate device") \
205 : _(1, QUIESCE_DEV, "quiesce device") \
206 : _(2, RESET_DEV, "reset device") \
207 : _(3, UPDATE_RX_MODE, "update rx mode") \
208 : _(4, UPDATE_MAC_FILTERS, "update mac filters") \
209 : _(5, UPDATE_VLAN_FILTERS, "update vlan filters") \
210 : _(6, UPDATE_RSSIDT, "update rss idt") \
211 : _(7, UPDATE_IML, "update iml") \
212 : _(8, UPDATE_PMCFG, "update pm cfg") \
213 : _(9, UPDATE_FEATURE, "update feature") \
214 : _(10, STOP_EMULATION, "stop emulation") \
215 : _(11, LOAD_PLUGIN, "load plugin") \
216 : _(12, ACTIVATE_VF, "activate vf") \
217 : _(13, RESERVED3, "reserved 3") \
218 : _(14, RESERVED4, "reservced 4") \
219 : _(15, REGISTER_MEMREGS, "register mem regs")
220 :
221 : enum
222 : {
223 : #define _(a, b, c) VMXNET3_CMD_##b = (a + 0xCAFE0000),
224 : foreach_vmxnet3_set_cmds
225 : #undef _
226 : };
227 :
228 : #define foreach_vmxnet3_get_cmds \
229 : _(0, GET_QUEUE_STATUS, "get queue status") \
230 : _(1, GET_STATS, "get stats") \
231 : _(2, GET_LINK, "get link") \
232 : _(3, GET_PERM_MAC_LO, "get perm mac lo") \
233 : _(4, GET_PERM_MAC_HI, "get perm mac hi") \
234 : _(5, GET_DID_LO, "get did lo") \
235 : _(6, GET_DID_HI, "get did hi") \
236 : _(7, GET_DEV_EXTRA_INFO, "get dev extra info") \
237 : _(8, GET_CONF_INTR, "get conf intr") \
238 : _(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \
239 : _(10, GET_TXDATA_DESC_SIZE, "get txdata desc size") \
240 : _(11, RESERVED5, "reserved5")
241 :
242 : enum
243 : {
244 : #define _(a, b, c) VMXNET3_CMD_##b = (a + 0xF00D0000),
245 : foreach_vmxnet3_get_cmds
246 : #undef _
247 : };
248 :
249 : typedef CLIB_PACKED (struct
250 : {
251 : u32 version; u32 guest_info; u32 version_support;
252 : u32 upt_version_support; u64 upt_features;
253 : u64 driver_data_address; u64 queue_desc_address;
254 : u32 driver_data_len; u32 queue_desc_len;
255 : u32 mtu;
256 : u16 max_num_rx_sg; u8 num_tx_queues; u8 num_rx_queues;
257 : u32 pad[4];
258 : }) vmxnet3_misc_config;
259 :
260 : typedef CLIB_PACKED (struct
261 : {
262 : u8 mask_mode;
263 : u8 num_intrs;
264 : u8 event_intr_index;
265 : u8 moderation_level[VMXNET3_MAX_INTRS]; u32 control;
266 : u32 pad[2];
267 : }) vmxnet3_interrupt_config;
268 :
269 : typedef CLIB_PACKED (struct
270 : {
271 : u32 mode; u16 multicast_len; u16 pad;
272 : u64 multicast_address; u8 vlan_filter[512];
273 : }) vmxnet3_rx_filter_config;
274 :
275 : typedef CLIB_PACKED (struct
276 : {
277 : u32 version; u32 length;
278 : u64 address;
279 : }) vmxnet3_variable_config;
280 :
281 : typedef CLIB_PACKED (struct
282 : {
283 : u32 magic;
284 : u32 pad;
285 : vmxnet3_misc_config misc;
286 : vmxnet3_interrupt_config interrupt;
287 : vmxnet3_rx_filter_config rx_filter;
288 : vmxnet3_variable_config rss;
289 : vmxnet3_variable_config pattern;
290 : vmxnet3_variable_config plugin; u32 ecr;
291 : u32 pad1[5];
292 : }) vmxnet3_shared;
293 :
294 : typedef CLIB_PACKED (struct
295 : {
296 : u8 stopped;
297 : u8 pad[3];
298 : u32 error;
299 : }) vmxnet3_queue_status;
300 :
301 : typedef CLIB_PACKED (struct
302 : {
303 : u32 num_deferred; u32 threshold;
304 : u64 pad;
305 : }) vmxnet3_tx_queue_control;
306 :
307 : typedef CLIB_PACKED (struct
308 : {
309 : u64 desc_address;
310 : u64 data_address;
311 : u64 comp_address; u64 driver_data_address; u64 pad;
312 : u32 num_desc;
313 : u32 num_data; u32 num_comp; u32 driver_data_len;
314 : u8 intr_index;
315 : u8 pad1; u16 data_address_size; u8 pad2[4];
316 : }) vmxnet3_tx_queue_config;
317 :
318 : typedef CLIB_PACKED (struct
319 : {
320 : u64 tso_pkts;
321 : u64 tso_bytes;
322 : u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
323 : u64 mcast_bytes;
324 : u64 bcast_pkts; u64 bcast_bytes; u64 error_pkts;
325 : u64 discard_pkts;
326 : }) vmxnet3_tx_stats;
327 :
328 : typedef CLIB_PACKED (struct
329 : {
330 : vmxnet3_tx_queue_control ctrl;
331 : vmxnet3_tx_queue_config cfg;
332 : vmxnet3_queue_status status; vmxnet3_tx_stats stats;
333 : u8 pad[88];
334 : }) vmxnet3_tx_queue;
335 :
336 : typedef CLIB_PACKED (struct
337 : {
338 : u8 update_prod; u8 pad[7];
339 : u64 pad1;
340 : }) vmxnet3_rx_queue_control;
341 :
342 : typedef CLIB_PACKED (struct
343 : {
344 : u64 desc_address[2];
345 : u64 comp_address; u64 driver_data_address;
346 : u64 data_address; u32 num_desc[2];
347 : u32 num_comp;
348 : u32 driver_data_len; u8 intr_index; u8 pad1;
349 : u16 data_address_size; u8 pad2[4];
350 : }) vmxnet3_rx_queue_config;
351 :
352 : typedef CLIB_PACKED (struct
353 : {
354 : u64 lro_pkts;
355 : u64 lro_bytes;
356 : u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
357 : u64 mcast_bytes;
358 : u64 bcast_pkts; u64 bcast_bytes; u64 nobuf_pkts;
359 : u64 error_pkts;
360 : }) vmxnet3_rx_stats;
361 :
362 : typedef CLIB_PACKED (struct
363 : {
364 : vmxnet3_rx_queue_control ctrl;
365 : vmxnet3_rx_queue_config cfg;
366 : vmxnet3_queue_status status; vmxnet3_rx_stats stats;
367 : u8 pad[88];
368 : }) vmxnet3_rx_queue;
369 :
370 : /*
371 : * flags:
372 : * buffer length -- bits 0-13
373 : * buffer type -- bit 14
374 : * descriptor type -- bit 15
375 : * reserved -- bits 16-30
376 : * generation -- bit 31
377 : */
378 : typedef CLIB_PACKED (struct
379 : {
380 : u64 address;
381 : u32 flags;
382 : u32 pad;
383 : }) vmxnet3_rx_desc;
384 :
385 : /*
386 : * index:
387 : * RX desc index -- bits 0-11
388 : * ext1 -- bits 12-13
389 : * end of packet -- bit 14
390 : * start of packet -- bit 15
391 : * ring ID -- bits 16-25
392 : * RSS hash type -- bits 26-29
393 : * checksum not calculated -- bit 30
394 : * ext2 -- bit 31
395 : *
396 : * rss: RSS hash value
397 : *
398 : * len:
399 : * data length -- bits 0-13
400 : * error -- bit 14
401 : * tag is stripped -- bit 15
402 : * tag stripped -- bits 16-31
403 : *
404 : * flags:
405 : * checksum -- bits 0 - 15
406 : * tcp/udp checksum correct-- bit 16
407 : * udp packet -- bit 17
408 : * tcp packet -- bit 18
409 : * ip checksum correct -- bit 19
410 : * ipv6 -- bit 20
411 : * ipv4 -- bit 21
412 : * ip fragment -- bit 22
413 : * frame crc correct -- bit 23
414 : * completion type -- bits 24-30
415 : * generation -- bit 31
416 : */
417 : typedef CLIB_PACKED (struct
418 : {
419 : u32 index; u32 rss;
420 : u32 len;
421 : u32 flags;
422 : }) vmxnet3_rx_comp;
423 :
424 : /*
425 : * flags:
426 : * mss -- bits 0 - 15
427 : * tcp/udp checksum correct-- bit 16
428 : * udp packet -- bit 17
429 : * tcp packet -- bit 18
430 : * ip checksum correct -- bit 19
431 : * ipv6 -- bit 20
432 : * ipv4 -- bit 21
433 : * ip fragment -- bit 22
434 : * frame crc correct -- bit 23
435 : * completion type -- bits 24-30
436 : * generation -- bit 31
437 : */
438 : typedef CLIB_PACKED (struct
439 : {
440 : u32 dword1;
441 : u8 seg_cnt; u8 dup_ack_cnt; u16 ts_delta; u32 dword2;
442 : u32 flags;
443 : }) vmxnet3_rx_comp_ext;
444 :
445 : /*
446 : * index:
447 : * TX desc index -- bits 0-11
448 : * ext1 -- bits 12-31
449 : *
450 : * flags:
451 : * reserved -- bits 0-23
452 : * completion type -- bits 24-30
453 : * generation -- bit 31
454 : */
455 : typedef CLIB_PACKED (struct
456 : {
457 : u32 index;
458 : u32 pad[2];
459 : u32 flags;
460 : }) vmxnet3_tx_comp;
461 :
462 : /*
463 : * flags[0]:
464 : * length -- bits 0-13
465 : * generation -- bit 14
466 : * reserved -- bit 15
467 : * descriptor type -- bit 16
468 : * ext1 -- bit 17
469 : * MSS, checksum offset -- bits 18-31
470 : * flags[1]:
471 : * header length -- bits 0-9
472 : * offload mode -- bits 10-11
473 : * end of packet -- bit 12
474 : * completion request -- bit 13
475 : * ext2 -- bit 14
476 : * vlan tag insertion -- bit 15
477 : * tag to insert -- bits 16-31
478 : */
479 : typedef CLIB_PACKED (struct
480 : {
481 : u64 address;
482 : u32 flags[2];
483 : }) vmxnet3_tx_desc;
484 :
485 : typedef CLIB_PACKED (struct
486 : {
487 : u16 hash_type;
488 : u16 hash_func;
489 : u16 hash_key_sz;
490 : u16 ind_table_sz;
491 : u8 hash_key[VMXNET3_RSS_MAX_KEY_SZ];
492 : u8 ind_table[VMXNET3_RSS_MAX_IND_TABLE_SZ];
493 : }) vmxnet3_rss_shared;
494 :
495 : typedef struct
496 : {
497 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
498 : u32 *bufs;
499 : u32 gen;
500 : u16 fill;
501 : u16 rid;
502 : u16 produce;
503 : u16 consume;
504 : } vmxnet3_rx_ring;
505 :
506 : typedef struct
507 : {
508 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
509 : u32 gen;
510 : u16 next;
511 : } vmxnet3_rx_comp_ring;
512 :
513 : typedef struct
514 : {
515 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
516 : u32 polling_q_count;
517 : } vmxnet3_per_thread_data_t;
518 :
519 : typedef struct
520 : {
521 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
522 : u16 size;
523 : u32 mode;
524 : u8 buffer_pool_index;
525 : u32 queue_index;
526 : u32 thread_index;
527 : vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE];
528 : vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE];
529 : vmxnet3_rx_comp *rx_comp;
530 : vmxnet3_rx_comp_ring rx_comp_ring;
531 : } vmxnet3_rxq_t;
532 :
533 : typedef struct
534 : {
535 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
536 : u32 *bufs;
537 : u32 gen;
538 : u16 produce;
539 : u16 consume;
540 : } vmxnet3_tx_ring;
541 :
542 : typedef struct
543 : {
544 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
545 : u32 gen;
546 : u16 next;
547 : } vmxnet3_tx_comp_ring;
548 :
549 : typedef struct
550 : {
551 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
552 : u16 size;
553 : u32 queue_index;
554 : u32 reg_txprod;
555 : clib_spinlock_t lock;
556 :
557 : vmxnet3_tx_desc *tx_desc;
558 : vmxnet3_tx_comp *tx_comp;
559 : vmxnet3_tx_ring tx_ring;
560 : vmxnet3_tx_comp_ring tx_comp_ring;
561 : } vmxnet3_txq_t;
562 :
563 : typedef struct
564 : {
565 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
566 : u32 flags;
567 : u32 per_interface_next_index;
568 :
569 : u32 dev_instance;
570 : u32 sw_if_index;
571 : u32 hw_if_index;
572 : u32 numa_node;
573 : vlib_pci_dev_handle_t pci_dev_handle;
574 : vlib_pci_addr_t pci_addr;
575 : void *bar[2];
576 :
577 : /* queues */
578 : vmxnet3_rxq_t *rxqs;
579 : vmxnet3_txq_t *txqs;
580 :
581 : u16 num_tx_queues;
582 : u16 num_rx_queues;
583 : u16 num_intrs;
584 :
585 : u8 version;
586 : u8 mac_addr[6];
587 :
588 : clib_error_t *error;
589 :
590 : vmxnet3_shared *driver_shared;
591 : void *queues;
592 : vmxnet3_rss_shared *rss;
593 : u32 link_speed;
594 : u8 gso_enable;
595 : vmxnet3_tx_stats *tx_stats;
596 : vmxnet3_rx_stats *rx_stats;
597 : } vmxnet3_device_t;
598 :
599 : typedef struct
600 : {
601 : vmxnet3_device_t *devices;
602 : u16 msg_id_base;
603 : vlib_log_class_t log_default;
604 : vmxnet3_per_thread_data_t *per_thread_data;
605 : } vmxnet3_main_t;
606 :
607 : extern vmxnet3_main_t vmxnet3_main;
608 :
609 : typedef enum
610 : {
611 : VMXNET3_BIND_NONE = 0,
612 : VMXNET3_BIND_DEFAULT = 1,
613 : VMXNET3_BIND_FORCE = 2,
614 : } __clib_packed vmxnet3_bind_t;
615 :
616 : typedef struct
617 : {
618 : vlib_pci_addr_t addr;
619 : u32 enable_elog;
620 : u16 rxq_size;
621 : u16 rxq_num;
622 : u16 txq_size;
623 : u16 txq_num;
624 : vmxnet3_bind_t bind;
625 : u8 enable_gso;
626 : /* return */
627 : i32 rv;
628 : u32 sw_if_index;
629 : clib_error_t *error;
630 : } vmxnet3_create_if_args_t;
631 :
632 : typedef struct
633 : {
634 : u32 next_index;
635 : u32 hw_if_index;
636 : vlib_buffer_t buffer;
637 : } vmxnet3_input_trace_t;
638 :
639 : void vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args);
640 : void vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * ad);
641 :
642 : extern clib_error_t *vmxnet3_plugin_api_hookup (vlib_main_t * vm);
643 : extern vlib_node_registration_t vmxnet3_input_node;
644 : extern vnet_device_class_t vmxnet3_device_class;
645 :
646 : /* format.c */
647 : format_function_t format_vmxnet3_device;
648 : format_function_t format_vmxnet3_device_name;
649 : format_function_t format_vmxnet3_input_trace;
650 :
651 : #define vmxnet3_log_debug(dev, f, ...) \
652 : vlib_log (VLIB_LOG_LEVEL_DEBUG, vmxnet3_main.log_default, "%U: " f, \
653 : format_vlib_pci_addr, &dev->pci_addr, \
654 : ## __VA_ARGS__)
655 :
656 : #define vmxnet3_log_error(dev, f, ...) \
657 : vlib_log (VLIB_LOG_LEVEL_ERR, vmxnet3_main.log_default, "%U: " f, \
658 : format_vlib_pci_addr, &dev->pci_addr, \
659 : ## __VA_ARGS__)
660 :
661 : /* no log version, called by data plane */
662 : static_always_inline void
663 0 : vmxnet3_reg_write_inline (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
664 : {
665 0 : *(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val;
666 0 : }
667 :
668 : static_always_inline void
669 0 : vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
670 : {
671 0 : vmxnet3_log_debug (vd, "reg wr bar %u addr 0x%x val 0x%x", bar, addr, val);
672 0 : vmxnet3_reg_write_inline (vd, bar, addr, val);
673 0 : }
674 :
675 : static_always_inline u32
676 0 : vmxnet3_reg_read (vmxnet3_device_t * vd, u8 bar, u32 addr)
677 : {
678 : u32 val;
679 :
680 0 : val = *(volatile u32 *) (vd->bar[bar] + addr);
681 0 : vmxnet3_log_debug (vd, "reg rd bar %u addr 0x%x val 0x%x", bar, addr, val);
682 :
683 0 : return val;
684 : }
685 :
686 : static_always_inline uword
687 0 : vmxnet3_dma_addr (vlib_main_t * vm, vmxnet3_device_t * vd, void *p)
688 : {
689 0 : return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) :
690 0 : vlib_physmem_get_pa (vm, p);
691 : }
692 :
693 : static_always_inline void
694 0 : vmxnet3_rx_ring_advance_produce (vmxnet3_rxq_t * rxq, vmxnet3_rx_ring * ring)
695 : {
696 0 : ring->produce++;
697 0 : if (PREDICT_FALSE (ring->produce == rxq->size))
698 : {
699 0 : ring->produce = 0;
700 0 : ring->gen ^= VMXNET3_RXF_GEN;
701 : }
702 0 : }
703 :
704 : static_always_inline clib_error_t *
705 0 : vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd,
706 : vmxnet3_rxq_t * rxq)
707 : {
708 : vmxnet3_rx_desc *rxd;
709 : u16 n_refill, n_alloc;
710 : vmxnet3_rx_ring *ring;
711 : vmxnet3_rx_queue *rx;
712 :
713 0 : ring = &rxq->rx_ring[0];
714 0 : n_refill = rxq->size - ring->fill;
715 :
716 0 : if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
717 0 : return 0;
718 :
719 0 : n_alloc =
720 0 : vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce,
721 0 : rxq->size, n_refill,
722 0 : rxq->buffer_pool_index);
723 0 : if (PREDICT_FALSE (n_alloc != n_refill))
724 : {
725 0 : if (n_alloc)
726 0 : vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
727 : n_alloc);
728 0 : return clib_error_return (0, "buffer alloc failed");
729 : }
730 :
731 0 : while (n_alloc)
732 : {
733 0 : vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
734 0 : rxd = &rxq->rx_desc[0][ring->produce];
735 0 : rxd->address = vlib_buffer_get_pa (vm, b);
736 0 : rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm);
737 :
738 0 : vmxnet3_rx_ring_advance_produce (rxq, ring);
739 0 : ring->fill++;
740 0 : n_alloc--;
741 : }
742 :
743 0 : rx = VMXNET3_RX_START (vd);
744 0 : if (PREDICT_FALSE (rx->ctrl.update_prod))
745 0 : vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD, ring->produce);
746 :
747 0 : return 0;
748 : }
749 :
750 : static_always_inline clib_error_t *
751 0 : vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd,
752 : vmxnet3_rxq_t * rxq)
753 : {
754 : vmxnet3_rx_desc *rxd;
755 : u16 n_refill, n_alloc;
756 : vmxnet3_rx_ring *ring;
757 : vmxnet3_rx_queue *rx;
758 :
759 0 : ring = &rxq->rx_ring[1];
760 0 : n_refill = rxq->size - ring->fill;
761 :
762 0 : if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
763 0 : return 0;
764 :
765 0 : n_alloc =
766 0 : vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce,
767 0 : rxq->size, n_refill,
768 0 : rxq->buffer_pool_index);
769 0 : if (PREDICT_FALSE (n_alloc != n_refill))
770 : {
771 0 : if (n_alloc)
772 0 : vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
773 : n_alloc);
774 0 : return clib_error_return (0, "buffer alloc failed");
775 : }
776 :
777 0 : while (n_alloc)
778 : {
779 0 : vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
780 0 : rxd = &rxq->rx_desc[1][ring->produce];
781 0 : rxd->address = vlib_buffer_get_pa (vm, b);
782 0 : rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm) |
783 : VMXNET3_RXF_BTYPE;
784 :
785 0 : vmxnet3_rx_ring_advance_produce (rxq, ring);
786 0 : ring->fill++;
787 0 : n_alloc--;
788 : }
789 :
790 0 : rx = VMXNET3_RX_START (vd);
791 0 : if (PREDICT_FALSE (rx->ctrl.update_prod))
792 0 : vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD2, ring->produce);
793 :
794 0 : return 0;
795 : }
796 :
797 : #endif /* __included_vmnet_vmnet_h__ */
798 : /*
799 : * fd.io coding-style-patch-verification: ON
800 : *
801 : * Local Variables:
802 : * eval: (c-set-style "gnu")
803 : * End:
804 : */
|
