Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : #ifndef __VIRTIO_INLINE_H__
16 : #define __VIRTIO_INLINE_H__
17 :
18 : #define foreach_virtio_input_error \
19 : _ (BUFFER_ALLOC, "buffer alloc error") \
20 : _ (UNKNOWN, "unknown")
21 :
22 : typedef enum
23 : {
24 : #define _(f, s) VIRTIO_INPUT_ERROR_##f,
25 : foreach_virtio_input_error
26 : #undef _
27 : VIRTIO_INPUT_N_ERROR,
28 : } virtio_input_error_t;
29 :
30 : static_always_inline void
31 533412300 : virtio_refill_vring_split (vlib_main_t *vm, virtio_if_t *vif,
32 : virtio_if_type_t type, vnet_virtio_vring_t *vring,
33 : const int hdr_sz, u32 node_index)
34 : {
35 : u16 used, next, avail, n_slots, n_refill;
36 533412300 : u16 sz = vring->queue_size;
37 533412300 : u16 mask = sz - 1;
38 :
39 535260500 : more:
40 535260500 : used = vring->desc_in_use;
41 :
42 535260500 : if (sz - used < sz / 8)
43 533412300 : return;
44 :
45 : /* deliver free buffers in chunks of 64 */
46 1848510 : n_refill = clib_min (sz - used, 64);
47 :
48 1848510 : next = vring->desc_next;
49 1848510 : avail = vring->avail->idx;
50 3697020 : n_slots = vlib_buffer_alloc_to_ring_from_pool (vm, vring->buffers, next,
51 1848510 : vring->queue_size, n_refill,
52 1848510 : vring->buffer_pool_index);
53 :
54 1848510 : if (PREDICT_FALSE (n_slots != n_refill))
55 : {
56 0 : vlib_error_count (vm, node_index, VIRTIO_INPUT_ERROR_BUFFER_ALLOC,
57 0 : n_refill - n_slots);
58 0 : if (n_slots == 0)
59 0 : return;
60 : }
61 :
62 115931000 : while (n_slots)
63 : {
64 114081800 : vnet_virtio_vring_desc_t *d = &vring->desc[next];
65 : ;
66 114081800 : vlib_buffer_t *b = vlib_get_buffer (vm, vring->buffers[next]);
67 : /*
68 : * current_data may not be initialized with 0 and may contain
69 : * previous offset. Here we want to make sure, it should be 0
70 : * initialized.
71 : */
72 114081800 : b->current_data = -hdr_sz;
73 114081800 : clib_memset (vlib_buffer_get_current (b), 0, hdr_sz);
74 114081800 : d->addr = ((type == VIRTIO_IF_TYPE_PCI) ?
75 114081800 : vlib_buffer_get_current_pa (vm, b) :
76 228164600 : pointer_to_uword (vlib_buffer_get_current (b)));
77 114081800 : d->len = vlib_buffer_get_default_data_size (vm) + hdr_sz;
78 114081800 : d->flags = VRING_DESC_F_WRITE;
79 114081800 : vring->avail->ring[avail & mask] = next;
80 114081800 : avail++;
81 114081800 : next = (next + 1) & mask;
82 114081800 : n_slots--;
83 114081800 : used++;
84 : }
85 1848510 : clib_atomic_store_seq_cst (&vring->avail->idx, avail);
86 1848510 : vring->desc_next = next;
87 1848510 : vring->desc_in_use = used;
88 1848510 : if ((clib_atomic_load_seq_cst (&vring->used->flags) &
89 : VRING_USED_F_NO_NOTIFY) == 0)
90 : {
91 650559 : virtio_kick (vm, vring, vif);
92 : }
93 1848510 : goto more;
94 : }
95 :
96 : static_always_inline void
97 0 : virtio_refill_vring_packed (vlib_main_t *vm, virtio_if_t *vif,
98 : virtio_if_type_t type, vnet_virtio_vring_t *vring,
99 : const int hdr_sz, u32 node_index)
100 : {
101 0 : u16 used, next, n_slots, n_refill, flags = 0, first_desc_flags;
102 0 : u16 sz = vring->queue_size;
103 :
104 0 : more:
105 0 : used = vring->desc_in_use;
106 :
107 0 : if (sz == used)
108 0 : return;
109 :
110 : /* deliver free buffers in chunks of 64 */
111 0 : n_refill = clib_min (sz - used, 64);
112 :
113 0 : next = vring->desc_next;
114 0 : first_desc_flags = vring->packed_desc[next].flags;
115 0 : n_slots = vlib_buffer_alloc_to_ring_from_pool (
116 0 : vm, vring->buffers, next, sz, n_refill, vring->buffer_pool_index);
117 :
118 0 : if (PREDICT_FALSE (n_slots != n_refill))
119 : {
120 0 : vlib_error_count (vm, node_index, VIRTIO_INPUT_ERROR_BUFFER_ALLOC,
121 0 : n_refill - n_slots);
122 0 : if (n_slots == 0)
123 0 : return;
124 : }
125 :
126 0 : while (n_slots)
127 : {
128 0 : vnet_virtio_vring_packed_desc_t *d = &vring->packed_desc[next];
129 0 : vlib_buffer_t *b = vlib_get_buffer (vm, vring->buffers[next]);
130 : /*
131 : * current_data may not be initialized with 0 and may contain
132 : * previous offset. Here we want to make sure, it should be 0
133 : * initialized.
134 : */
135 0 : b->current_data = -hdr_sz;
136 0 : clib_memset (vlib_buffer_get_current (b), 0, hdr_sz);
137 0 : d->addr = ((type == VIRTIO_IF_TYPE_PCI) ?
138 0 : vlib_buffer_get_current_pa (vm, b) :
139 0 : pointer_to_uword (vlib_buffer_get_current (b)));
140 0 : d->len = vlib_buffer_get_default_data_size (vm) + hdr_sz;
141 :
142 0 : if (vring->avail_wrap_counter)
143 0 : flags = (VRING_DESC_F_AVAIL | VRING_DESC_F_WRITE);
144 : else
145 0 : flags = (VRING_DESC_F_USED | VRING_DESC_F_WRITE);
146 :
147 0 : d->id = next;
148 0 : if (vring->desc_next == next)
149 0 : first_desc_flags = flags;
150 : else
151 0 : d->flags = flags;
152 :
153 0 : next++;
154 0 : if (next >= sz)
155 : {
156 0 : next = 0;
157 0 : vring->avail_wrap_counter ^= 1;
158 : }
159 0 : n_slots--;
160 0 : used++;
161 : }
162 0 : CLIB_MEMORY_STORE_BARRIER ();
163 0 : vring->packed_desc[vring->desc_next].flags = first_desc_flags;
164 0 : vring->desc_next = next;
165 0 : vring->desc_in_use = used;
166 0 : CLIB_MEMORY_BARRIER ();
167 0 : if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
168 : {
169 0 : virtio_kick (vm, vring, vif);
170 : }
171 :
172 0 : goto more;
173 : }
174 :
175 : #endif
176 :
177 : /*
178 : * fd.io coding-style-patch-verification: ON
179 : *
180 : * Local Variables:
181 : * eval: (c-set-style "gnu")
182 : * End:
183 : */
|
