Line data Source code
1 :
2 : /*
3 : * Copyright (c) 2016 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : */
16 :
17 : #include <vnet/vnet.h>
18 : #include <vnet/hash/hash.h>
19 : #include <vlib/threads.h>
20 : #include <vnet/feature/feature.h>
21 :
22 : typedef struct
23 : {
24 : vnet_hash_fn_t hash_fn;
25 : uword *workers_bitmap;
26 : u32 *workers;
27 : } per_inteface_handoff_data_t;
28 :
29 : typedef struct
30 : {
31 : u32 cached_next_index;
32 : u32 num_workers;
33 : u32 first_worker_index;
34 :
35 : per_inteface_handoff_data_t *if_data;
36 :
37 : /* Worker handoff index */
38 : u32 frame_queue_index;
39 : } handoff_main_t;
40 :
41 : extern handoff_main_t handoff_main;
42 :
43 : #ifndef CLIB_MARCH_VARIANT
44 :
45 : handoff_main_t handoff_main;
46 :
47 : #endif /* CLIB_MARCH_VARIANT */
48 :
49 : typedef struct
50 : {
51 : u32 sw_if_index;
52 : u32 next_worker_index;
53 : u32 buffer_index;
54 : } worker_handoff_trace_t;
55 :
56 : #define foreach_worker_handoff_error \
57 : _(CONGESTION_DROP, "congestion drop")
58 :
59 : typedef enum
60 : {
61 : #define _(sym,str) WORKER_HANDOFF_ERROR_##sym,
62 : foreach_worker_handoff_error
63 : #undef _
64 : WORKER_HANDOFF_N_ERROR,
65 : } worker_handoff_error_t;
66 :
67 : static char *worker_handoff_error_strings[] = {
68 : #define _(sym,string) string,
69 : foreach_worker_handoff_error
70 : #undef _
71 : };
72 :
73 : /* packet trace format function */
74 : static u8 *
75 0 : format_worker_handoff_trace (u8 * s, va_list * args)
76 : {
77 0 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
78 0 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
79 0 : worker_handoff_trace_t *t = va_arg (*args, worker_handoff_trace_t *);
80 :
81 0 : s = format (s, "worker-handoff: sw_if_index %d, next_worker %d, buffer 0x%x",
82 : t->sw_if_index, t->next_worker_index, t->buffer_index);
83 0 : return s;
84 : }
85 :
86 : static void
87 0 : worker_handoff_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node,
88 : vlib_buffer_t **bufs, u16 *threads, u32 n_vectors)
89 : {
90 : worker_handoff_trace_t *t;
91 : vlib_buffer_t **b;
92 : u16 *ti;
93 :
94 0 : b = bufs;
95 0 : ti = threads;
96 :
97 0 : while (n_vectors)
98 : {
99 0 : t = vlib_add_trace (vm, node, b[0], sizeof (*t));
100 0 : t->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
101 0 : t->next_worker_index = ti[0];
102 0 : t->buffer_index = vlib_get_buffer_index (vm, b[0]);
103 :
104 0 : b += 1;
105 0 : ti += 1;
106 0 : n_vectors -= 1;
107 : }
108 0 : }
109 :
110 2300 : VLIB_NODE_FN (worker_handoff_node) (vlib_main_t * vm,
111 : vlib_node_runtime_t * node,
112 : vlib_frame_t * frame)
113 : {
114 0 : handoff_main_t *hm = &handoff_main;
115 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
116 : u32 n_enq, n_left_from, *from;
117 : u16 thread_indices[VLIB_FRAME_SIZE], *ti;
118 :
119 0 : from = vlib_frame_vector_args (frame);
120 0 : n_left_from = frame->n_vectors;
121 0 : vlib_get_buffers (vm, from, bufs, n_left_from);
122 :
123 0 : b = bufs;
124 0 : ti = thread_indices;
125 :
126 0 : while (n_left_from > 0)
127 : {
128 : per_inteface_handoff_data_t *ihd0;
129 : u32 sw_if_index0, hash, index0;
130 : void *data;
131 :
132 0 : sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
133 0 : ihd0 = vec_elt_at_index (hm->if_data, sw_if_index0);
134 :
135 : /* Compute ingress LB hash */
136 0 : data = vlib_buffer_get_current (b[0]);
137 0 : ihd0->hash_fn (&data, &hash, 1);
138 :
139 : /* if input node did not specify next index, then packet
140 : should go to ethernet-input */
141 :
142 0 : if (PREDICT_TRUE (is_pow2 (vec_len (ihd0->workers))))
143 0 : index0 = hash & (vec_len (ihd0->workers) - 1);
144 : else
145 0 : index0 = hash % vec_len (ihd0->workers);
146 :
147 0 : ti[0] = hm->first_worker_index + ihd0->workers[index0];
148 :
149 : /* next */
150 0 : n_left_from -= 1;
151 0 : ti += 1;
152 0 : b += 1;
153 : }
154 :
155 0 : if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
156 0 : worker_handoff_trace_frame (vm, node, bufs, thread_indices,
157 0 : frame->n_vectors);
158 :
159 0 : n_enq = vlib_buffer_enqueue_to_thread (vm, node, hm->frame_queue_index, from,
160 0 : thread_indices, frame->n_vectors, 1);
161 :
162 0 : if (n_enq < frame->n_vectors)
163 0 : vlib_node_increment_counter (vm, node->node_index,
164 : WORKER_HANDOFF_ERROR_CONGESTION_DROP,
165 0 : frame->n_vectors - n_enq);
166 0 : return frame->n_vectors;
167 : }
168 :
169 183788 : VLIB_REGISTER_NODE (worker_handoff_node) = {
170 : .name = "worker-handoff",
171 : .vector_size = sizeof (u32),
172 : .format_trace = format_worker_handoff_trace,
173 : .type = VLIB_NODE_TYPE_INTERNAL,
174 : .n_errors = ARRAY_LEN(worker_handoff_error_strings),
175 : .error_strings = worker_handoff_error_strings,
176 :
177 : .n_next_nodes = 1,
178 : .next_nodes = {
179 : [0] = "error-drop",
180 : },
181 : };
182 :
183 : #ifndef CLIB_MARCH_VARIANT
184 :
185 : int
186 0 : interface_handoff_enable_disable (vlib_main_t *vm, u32 sw_if_index,
187 : uword *bitmap, u8 is_sym, int is_l4,
188 : int enable_disable)
189 : {
190 0 : handoff_main_t *hm = &handoff_main;
191 : vnet_sw_interface_t *sw;
192 0 : vnet_main_t *vnm = vnet_get_main ();
193 : per_inteface_handoff_data_t *d;
194 0 : int i, rv = 0;
195 :
196 0 : if (pool_is_free_index (vnm->interface_main.sw_interfaces, sw_if_index))
197 0 : return VNET_API_ERROR_INVALID_SW_IF_INDEX;
198 :
199 0 : sw = vnet_get_sw_interface (vnm, sw_if_index);
200 0 : if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
201 0 : return VNET_API_ERROR_INVALID_SW_IF_INDEX;
202 :
203 0 : if (clib_bitmap_last_set (bitmap) >= hm->num_workers)
204 0 : return VNET_API_ERROR_INVALID_WORKER;
205 :
206 0 : if (hm->frame_queue_index == ~0)
207 : {
208 0 : vlib_node_t *n = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
209 0 : hm->frame_queue_index = vlib_frame_queue_main_init (n->index, 0);
210 : }
211 :
212 0 : vec_validate (hm->if_data, sw_if_index);
213 0 : d = vec_elt_at_index (hm->if_data, sw_if_index);
214 :
215 0 : vec_free (d->workers);
216 0 : vec_free (d->workers_bitmap);
217 :
218 0 : if (enable_disable)
219 : {
220 0 : d->workers_bitmap = bitmap;
221 0 : clib_bitmap_foreach (i, bitmap)
222 : {
223 0 : vec_add1(d->workers, i);
224 : }
225 :
226 0 : if (is_sym)
227 : {
228 0 : if (is_l4)
229 0 : return VNET_API_ERROR_UNIMPLEMENTED;
230 :
231 0 : d->hash_fn = vnet_hash_function_from_name (
232 : "handoff-eth-sym", VNET_HASH_FN_TYPE_ETHERNET);
233 : }
234 : else
235 : {
236 0 : if (is_l4)
237 0 : d->hash_fn =
238 0 : vnet_hash_default_function (VNET_HASH_FN_TYPE_ETHERNET);
239 : else
240 0 : d->hash_fn = vnet_hash_function_from_name (
241 : "handoff-eth", VNET_HASH_FN_TYPE_ETHERNET);
242 : }
243 : }
244 :
245 0 : vnet_feature_enable_disable ("device-input", "worker-handoff",
246 : sw_if_index, enable_disable, 0, 0);
247 0 : return rv;
248 : }
249 :
250 : static clib_error_t *
251 0 : set_interface_handoff_command_fn (vlib_main_t * vm,
252 : unformat_input_t * input,
253 : vlib_cli_command_t * cmd)
254 : {
255 0 : u32 sw_if_index = ~0, is_sym = 0, is_l4 = 0;
256 0 : int enable_disable = 1;
257 0 : uword *bitmap = 0;
258 0 : int rv = 0;
259 :
260 0 : while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
261 : {
262 0 : if (unformat (input, "disable"))
263 0 : enable_disable = 0;
264 0 : else if (unformat (input, "workers %U", unformat_bitmap_list, &bitmap))
265 : ;
266 0 : else if (unformat (input, "%U", unformat_vnet_sw_interface,
267 : vnet_get_main (), &sw_if_index))
268 : ;
269 0 : else if (unformat (input, "symmetrical"))
270 0 : is_sym = 1;
271 0 : else if (unformat (input, "asymmetrical"))
272 0 : is_sym = 0;
273 0 : else if (unformat (input, "l4"))
274 0 : is_l4 = 1;
275 : else
276 0 : break;
277 : }
278 :
279 0 : if (sw_if_index == ~0)
280 0 : return clib_error_return (0, "Please specify an interface...");
281 :
282 0 : if (bitmap == 0)
283 0 : return clib_error_return (0, "Please specify list of workers...");
284 :
285 0 : rv = interface_handoff_enable_disable (vm, sw_if_index, bitmap, is_sym,
286 : is_l4, enable_disable);
287 :
288 0 : switch (rv)
289 : {
290 0 : case 0:
291 0 : break;
292 :
293 0 : case VNET_API_ERROR_INVALID_SW_IF_INDEX:
294 0 : return clib_error_return (0, "Invalid interface");
295 : break;
296 :
297 0 : case VNET_API_ERROR_INVALID_WORKER:
298 0 : return clib_error_return (0, "Invalid worker(s)");
299 : break;
300 :
301 0 : case VNET_API_ERROR_UNIMPLEMENTED:
302 0 : return clib_error_return (0,
303 : "Device driver doesn't support redirection");
304 : break;
305 :
306 0 : default:
307 0 : return clib_error_return (0, "unknown return value %d", rv);
308 : }
309 :
310 0 : return 0;
311 : }
312 :
313 : /* *INDENT-OFF* */
314 285289 : VLIB_CLI_COMMAND (set_interface_handoff_command, static) = {
315 : .path = "set interface handoff",
316 : .short_help = "set interface handoff <interface-name> workers <workers-list>"
317 : " [symmetrical|asymmetrical]",
318 : .function = set_interface_handoff_command_fn,
319 : };
320 : /* *INDENT-ON* */
321 :
322 : clib_error_t *
323 575 : handoff_init (vlib_main_t * vm)
324 : {
325 575 : handoff_main_t *hm = &handoff_main;
326 575 : vlib_thread_main_t *tm = vlib_get_thread_main ();
327 : clib_error_t *error;
328 : uword *p;
329 :
330 575 : if ((error = vlib_call_init_function (vm, threads_init)))
331 0 : return error;
332 :
333 : vlib_thread_registration_t *tr;
334 : /* Only the standard vnet worker threads are supported */
335 575 : p = hash_get_mem (tm->thread_registrations_by_name, "workers");
336 575 : if (p)
337 : {
338 575 : tr = (vlib_thread_registration_t *) p[0];
339 575 : if (tr)
340 : {
341 575 : hm->num_workers = tr->count;
342 575 : hm->first_worker_index = tr->first_index;
343 : }
344 : }
345 :
346 575 : hm->frame_queue_index = ~0;
347 :
348 575 : return 0;
349 : }
350 :
351 9215 : VLIB_INIT_FUNCTION (handoff_init);
352 :
353 : #endif /* CLIB_MARCH_VARIANT */
354 : /*
355 : * fd.io coding-style-patch-verification: ON
356 : *
357 : * Local Variables:
358 : * eval: (c-set-style "gnu")
359 : * End:
360 : */
|
