Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * buffer.c: allocate/free network buffers.
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : /**
41 : * @file
42 : *
43 : * Allocate/free network buffers.
44 : */
45 :
46 : #include <vppinfra/linux/sysfs.h>
47 : #include <vlib/vlib.h>
48 : #include <vlib/unix/unix.h>
49 : #include <vlib/stats/stats.h>
50 :
51 : #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384
52 : #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192
53 :
54 : #ifdef CLIB_HAVE_VEC128
55 : /* Assumptions by vlib_buffer_free_inline: */
56 : STATIC_ASSERT_FITS_IN (vlib_buffer_t, flags, 16);
57 : STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16);
58 : STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16);
59 : #endif
60 :
61 : u16 __vlib_buffer_external_hdr_size = 0;
62 :
63 : uword
64 265031 : vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
65 : vlib_buffer_t * b_first)
66 : {
67 265031 : vlib_buffer_t *b = b_first;
68 265031 : uword l_first = b_first->current_length;
69 265031 : uword l = 0;
70 636970 : while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
71 : {
72 371939 : b = vlib_get_buffer (vm, b->next_buffer);
73 371939 : l += b->current_length;
74 : }
75 265031 : b_first->total_length_not_including_first_buffer = l;
76 265031 : b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
77 265031 : return l + l_first;
78 : }
79 :
80 : u8 *
81 696265 : format_vlib_buffer_no_chain (u8 * s, va_list * args)
82 : {
83 696265 : vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
84 696265 : u32 indent = format_get_indent (s);
85 696265 : u8 *a = 0;
86 :
87 : #define _(bit, name, v) \
88 : if (v && (b->flags & VLIB_BUFFER_##name)) \
89 : a = format (a, "%s ", v);
90 696265 : foreach_vlib_buffer_flag
91 : #undef _
92 696265 : s = format (s, "current data %d, length %d, buffer-pool %d, "
93 696265 : "ref-count %u", b->current_data, b->current_length,
94 696265 : b->buffer_pool_index, b->ref_count);
95 :
96 696265 : if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
97 165392 : s = format (s, ", totlen-nifb %d",
98 : b->total_length_not_including_first_buffer);
99 :
100 696265 : if (b->flags & VLIB_BUFFER_IS_TRACED)
101 696264 : s = format (s, ", trace handle 0x%x", b->trace_handle);
102 :
103 696265 : if (a)
104 165392 : s = format (s, "\n%U%v", format_white_space, indent, a);
105 696265 : vec_free (a);
106 :
107 696265 : return s;
108 : }
109 :
110 : u8 *
111 1 : format_vlib_buffer (u8 * s, va_list * args)
112 : {
113 1 : vlib_main_t *vm = vlib_get_main ();
114 1 : vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
115 1 : u32 indent = format_get_indent (s);
116 :
117 1 : s = format (s, "%U", format_vlib_buffer_no_chain, b);
118 :
119 2 : while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
120 : {
121 1 : u32 next_buffer = b->next_buffer;
122 1 : b = vlib_get_buffer (vm, next_buffer);
123 :
124 : s =
125 1 : format (s, "\n%Unext-buffer 0x%x, segment length %d, ref-count %u",
126 1 : format_white_space, indent, next_buffer, b->current_length,
127 1 : b->ref_count);
128 : }
129 :
130 1 : return s;
131 : }
132 :
133 : u8 *
134 1 : format_vlib_buffer_and_data (u8 * s, va_list * args)
135 : {
136 1 : vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
137 :
138 1 : s = format (s, "%U, %U",
139 : format_vlib_buffer, b,
140 : format_hex_bytes, vlib_buffer_get_current (b), 64);
141 :
142 1 : return s;
143 : }
144 :
145 : static u8 *
146 2 : format_vlib_buffer_known_state (u8 * s, va_list * args)
147 : {
148 2 : vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t);
149 : char *t;
150 :
151 2 : switch (state)
152 : {
153 0 : case VLIB_BUFFER_UNKNOWN:
154 0 : t = "unknown";
155 0 : break;
156 :
157 1 : case VLIB_BUFFER_KNOWN_ALLOCATED:
158 1 : t = "known-allocated";
159 1 : break;
160 :
161 1 : case VLIB_BUFFER_KNOWN_FREE:
162 1 : t = "known-free";
163 1 : break;
164 :
165 0 : default:
166 0 : t = "invalid";
167 0 : break;
168 : }
169 :
170 2 : return format (s, "%s", t);
171 : }
172 :
173 : u8 *
174 0 : format_vlib_buffer_contents (u8 * s, va_list * va)
175 : {
176 0 : vlib_main_t *vm = va_arg (*va, vlib_main_t *);
177 0 : vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
178 :
179 : while (1)
180 : {
181 0 : vec_add (s, vlib_buffer_get_current (b), b->current_length);
182 0 : if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
183 0 : break;
184 0 : b = vlib_get_buffer (vm, b->next_buffer);
185 : }
186 :
187 0 : return s;
188 : }
189 :
190 : static u8 *
191 5 : vlib_validate_buffer_helper (vlib_main_t * vm,
192 : u32 bi,
193 : uword follow_buffer_next, uword ** unique_hash)
194 : {
195 5 : vlib_buffer_main_t *bm = vm->buffer_main;
196 5 : vlib_buffer_t *b = vlib_get_buffer (vm, bi);
197 :
198 5 : if (vec_len (bm->buffer_pools) <= b->buffer_pool_index)
199 0 : return format (0, "unknown buffer pool 0x%x", b->buffer_pool_index);
200 :
201 5 : if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
202 0 : return format (0, "current data %d before pre-data", b->current_data);
203 :
204 5 : if (b->current_data + b->current_length >
205 5 : vlib_buffer_get_default_data_size (vm))
206 0 : return format (0, "%d-%d beyond end of buffer %d", b->current_data,
207 0 : b->current_length, vlib_buffer_get_default_data_size (vm));
208 :
209 5 : if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
210 : {
211 : vlib_buffer_known_state_t k;
212 : u8 *msg, *result;
213 :
214 2 : k = vlib_buffer_is_known (vm, b->next_buffer);
215 2 : if (k != VLIB_BUFFER_KNOWN_ALLOCATED)
216 0 : return format (0, "next 0x%x: %U",
217 : b->next_buffer, format_vlib_buffer_known_state, k);
218 :
219 2 : if (unique_hash)
220 : {
221 1 : if (hash_get (*unique_hash, b->next_buffer))
222 0 : return format (0, "duplicate buffer 0x%x", b->next_buffer);
223 :
224 1 : hash_set1 (*unique_hash, b->next_buffer);
225 : }
226 :
227 2 : msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
228 2 : if (msg)
229 : {
230 0 : result = format (0, "next 0x%x: %v", b->next_buffer, msg);
231 0 : vec_free (msg);
232 0 : return result;
233 : }
234 : }
235 :
236 5 : return 0;
237 : }
238 :
239 : u8 *
240 3 : vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
241 : {
242 3 : return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
243 : /* unique_hash */ 0);
244 : }
245 :
246 : u8 *
247 3 : vlib_validate_buffers (vlib_main_t * vm,
248 : u32 * buffers,
249 : uword next_buffer_stride,
250 : uword n_buffers,
251 : vlib_buffer_known_state_t known_state,
252 : uword follow_buffer_next)
253 : {
254 : uword i, *hash;
255 3 : u32 bi, *b = buffers;
256 : vlib_buffer_known_state_t k;
257 3 : u8 *msg = 0, *result = 0;
258 :
259 3 : hash = hash_create (0, 0);
260 5 : for (i = 0; i < n_buffers; i++)
261 : {
262 3 : bi = b[0];
263 3 : b += next_buffer_stride;
264 :
265 : /* Buffer is not unique. */
266 3 : if (hash_get (hash, bi))
267 : {
268 0 : msg = format (0, "not unique");
269 0 : goto done;
270 : }
271 :
272 3 : k = vlib_buffer_is_known (vm, bi);
273 3 : if (k != known_state)
274 : {
275 1 : msg = format (0, "is %U; expected %U",
276 : format_vlib_buffer_known_state, k,
277 : format_vlib_buffer_known_state, known_state);
278 1 : goto done;
279 : }
280 :
281 2 : msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
282 2 : if (msg)
283 0 : goto done;
284 :
285 2 : hash_set1 (hash, bi);
286 : }
287 :
288 2 : done:
289 3 : if (msg)
290 : {
291 1 : result = format (0, "0x%x: %v", bi, msg);
292 1 : vec_free (msg);
293 : }
294 3 : hash_free (hash);
295 3 : return result;
296 : }
297 :
298 : /* When debugging validate that given buffers are either known allocated
299 : or known free. */
300 : void
301 32016000 : vlib_buffer_validate_alloc_free (vlib_main_t * vm,
302 : u32 * buffers,
303 : uword n_buffers,
304 : vlib_buffer_known_state_t expected_state)
305 : {
306 32016000 : vlib_buffer_main_t *bm = vm->buffer_main;
307 : u32 *b;
308 : uword i, bi, is_free;
309 :
310 : if (CLIB_DEBUG == 0)
311 : return;
312 :
313 32016000 : is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
314 32016000 : b = buffers;
315 470470000 : for (i = 0; i < n_buffers; i++)
316 : {
317 : vlib_buffer_known_state_t known;
318 :
319 438454000 : bi = b[0];
320 438454000 : b += 1;
321 438454000 : known = vlib_buffer_is_known (vm, bi);
322 :
323 438454000 : if (known == VLIB_BUFFER_UNKNOWN &&
324 : expected_state == VLIB_BUFFER_KNOWN_FREE)
325 79637 : known = VLIB_BUFFER_KNOWN_FREE;
326 :
327 438454000 : if (known != expected_state)
328 : {
329 0 : clib_panic ("%s %U buffer 0x%x", is_free ? "freeing" : "allocating",
330 : format_vlib_buffer_known_state, known, bi);
331 : }
332 :
333 438454000 : clib_spinlock_lock (&bm->buffer_known_hash_lockp);
334 438454000 : hash_set (bm->buffer_known_hash, bi, is_free ? VLIB_BUFFER_KNOWN_FREE :
335 : VLIB_BUFFER_KNOWN_ALLOCATED);
336 438454000 : clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
337 : }
338 : }
339 :
340 : void
341 3450 : vlib_packet_template_init (vlib_main_t * vm,
342 : vlib_packet_template_t * t,
343 : void *packet_data,
344 : uword n_packet_data_bytes,
345 : uword min_n_buffers_each_alloc, char *fmt, ...)
346 : {
347 : va_list va;
348 :
349 3450 : va_start (va, fmt);
350 3450 : t->name = va_format (0, fmt, &va);
351 3450 : va_end (va);
352 :
353 3450 : vlib_worker_thread_barrier_sync (vm);
354 :
355 3450 : clib_memset (t, 0, sizeof (t[0]));
356 :
357 3450 : vec_add (t->packet_data, packet_data, n_packet_data_bytes);
358 3450 : t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
359 3450 : vlib_worker_thread_barrier_release (vm);
360 3450 : }
361 :
362 : void *
363 1009 : vlib_packet_template_get_packet (vlib_main_t * vm,
364 : vlib_packet_template_t * t, u32 * bi_result)
365 : {
366 : u32 bi;
367 : vlib_buffer_t *b;
368 :
369 1009 : if (vlib_buffer_alloc (vm, &bi, 1) != 1)
370 0 : return 0;
371 :
372 1009 : *bi_result = bi;
373 :
374 1009 : b = vlib_get_buffer (vm, bi);
375 2018 : clib_memcpy_fast (vlib_buffer_get_current (b),
376 2018 : t->packet_data, vec_len (t->packet_data));
377 1009 : b->current_length = vec_len (t->packet_data);
378 :
379 1009 : return b->data;
380 : }
381 :
382 : /* Append given data to end of buffer, possibly allocating new buffers. */
383 : int
384 7224 : vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
385 : u32 n_data_bytes)
386 : {
387 : u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
388 : vlib_buffer_t *b;
389 : void *d;
390 :
391 7224 : bi = *buffer_index;
392 7224 : if (bi == ~0 && 1 != vlib_buffer_alloc (vm, &bi, 1))
393 0 : goto out_of_buffers;
394 :
395 7224 : d = data;
396 7224 : n_left = n_data_bytes;
397 7224 : n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
398 :
399 7224 : b = vlib_get_buffer (vm, bi);
400 7224 : b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
401 :
402 : /* Get to the end of the chain before we try to append data... */
403 7225 : while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
404 1 : b = vlib_get_buffer (vm, b->next_buffer);
405 :
406 : while (1)
407 0 : {
408 : u32 n;
409 :
410 7224 : ASSERT (n_buffer_bytes >= b->current_length);
411 7224 : n_left_this_buffer =
412 7224 : n_buffer_bytes - (b->current_data + b->current_length);
413 7224 : n = clib_min (n_left_this_buffer, n_left);
414 7224 : clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d,
415 : n);
416 7224 : b->current_length += n;
417 7224 : n_left -= n;
418 7224 : if (n_left == 0)
419 7224 : break;
420 :
421 0 : d += n;
422 0 : if (1 != vlib_buffer_alloc (vm, &b->next_buffer, 1))
423 0 : goto out_of_buffers;
424 :
425 0 : b->flags |= VLIB_BUFFER_NEXT_PRESENT;
426 :
427 0 : b = vlib_get_buffer (vm, b->next_buffer);
428 : }
429 :
430 7224 : *buffer_index = bi;
431 7224 : return 0;
432 :
433 0 : out_of_buffers:
434 0 : clib_warning ("out of buffers");
435 0 : return 1;
436 : }
437 :
438 : u16
439 1 : vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
440 : vlib_buffer_t * first,
441 : vlib_buffer_t ** last, void *data,
442 : u16 data_len)
443 : {
444 1 : vlib_buffer_t *l = *last;
445 1 : u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
446 1 : u16 copied = 0;
447 1 : ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
448 2 : while (data_len)
449 : {
450 1 : u16 max = n_buffer_bytes - l->current_length - l->current_data;
451 1 : if (max == 0)
452 : {
453 1 : if (1 != vlib_buffer_alloc_from_pool (vm, &l->next_buffer, 1,
454 1 : first->buffer_pool_index))
455 0 : return copied;
456 1 : *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer);
457 1 : max = n_buffer_bytes - l->current_length - l->current_data;
458 : }
459 :
460 1 : u16 len = (data_len > max) ? max : data_len;
461 1 : clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length,
462 1 : data + copied, len);
463 1 : vlib_buffer_chain_increase_length (first, l, len);
464 1 : data_len -= len;
465 1 : copied += len;
466 : }
467 1 : return copied;
468 : }
469 :
470 : static uword
471 2875 : vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size)
472 : {
473 2875 : uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size;
474 2875 : alloc_size = CLIB_CACHE_LINE_ROUND (alloc_size);
475 :
476 : /* in case when we have even number of cachelines, we add one more for
477 : * better cache occupancy */
478 2875 : alloc_size |= CLIB_CACHE_LINE_BYTES;
479 :
480 2875 : return alloc_size;
481 : }
482 :
483 : u8
484 575 : vlib_buffer_pool_create (vlib_main_t *vm, u32 data_size, u32 physmem_map_index,
485 : char *fmt, ...)
486 : {
487 575 : vlib_buffer_main_t *bm = vm->buffer_main;
488 : vlib_buffer_pool_t *bp;
489 575 : vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
490 575 : uword start = pointer_to_uword (m->base);
491 575 : uword size = (uword) m->n_pages << m->log2_page_size;
492 575 : uword page_mask = ~pow2_mask (m->log2_page_size);
493 : u8 *p;
494 : u32 alloc_size;
495 : va_list va;
496 :
497 575 : if (vec_len (bm->buffer_pools) >= 255)
498 0 : return ~0;
499 :
500 575 : vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_CACHE_LINE_BYTES);
501 :
502 575 : if (bm->buffer_mem_size == 0)
503 : {
504 575 : bm->buffer_mem_start = start;
505 575 : bm->buffer_mem_size = size;
506 : }
507 0 : else if (start < bm->buffer_mem_start)
508 : {
509 0 : bm->buffer_mem_size += bm->buffer_mem_start - start;
510 0 : bm->buffer_mem_start = start;
511 0 : if (size > bm->buffer_mem_size)
512 0 : bm->buffer_mem_size = size;
513 : }
514 0 : else if (start > bm->buffer_mem_start)
515 : {
516 0 : uword new_size = start - bm->buffer_mem_start + size;
517 0 : if (new_size > bm->buffer_mem_size)
518 0 : bm->buffer_mem_size = new_size;
519 : }
520 :
521 575 : if ((u64) bm->buffer_mem_size >
522 : ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
523 : {
524 0 : clib_panic ("buffer memory size out of range!");
525 : }
526 :
527 575 : bp->start = start;
528 575 : bp->size = size;
529 575 : bp->index = bp - bm->buffer_pools;
530 575 : bp->buffer_template.buffer_pool_index = bp->index;
531 575 : bp->buffer_template.ref_count = 1;
532 575 : bp->physmem_map_index = physmem_map_index;
533 575 : bp->data_size = data_size;
534 575 : bp->numa_node = m->numa_node;
535 575 : bp->log2_page_size = m->log2_page_size;
536 :
537 575 : va_start (va, fmt);
538 575 : bp->name = va_format (0, fmt, &va);
539 575 : va_end (va);
540 :
541 575 : vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
542 : CLIB_CACHE_LINE_BYTES);
543 :
544 575 : alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size);
545 575 : bp->alloc_size = alloc_size;
546 :
547 : /* preallocate buffer indices memory */
548 575 : bp->buffers = clib_mem_alloc_aligned (
549 575 : round_pow2 ((size / alloc_size) * sizeof (u32), CLIB_CACHE_LINE_BYTES),
550 : CLIB_CACHE_LINE_BYTES);
551 :
552 575 : clib_spinlock_init (&bp->lock);
553 :
554 575 : p = m->base;
555 :
556 : /* start with naturally aligned address */
557 575 : p += alloc_size - (uword) p % alloc_size;
558 :
559 : /*
560 : * Waste 1 buffer (maximum) so that 0 is never a valid buffer index.
561 : * Allows various places to ASSERT (bi != 0). Much easier
562 : * than debugging downstream crashes in successor nodes.
563 : */
564 575 : if (p == m->base)
565 0 : p += alloc_size;
566 :
567 8147750 : for (; p < (u8 *) m->base + size - alloc_size; p += alloc_size)
568 : {
569 : vlib_buffer_t *b;
570 : u32 bi;
571 :
572 : /* skip if buffer spans across page boundary */
573 8147180 : if (((uword) p & page_mask) != ((uword) (p + alloc_size) & page_mask))
574 4709820 : continue;
575 :
576 3437350 : b = (vlib_buffer_t *) (p + bm->ext_hdr_size);
577 3437350 : b->template = bp->buffer_template;
578 3437350 : bi = vlib_get_buffer_index (vm, b);
579 3437350 : bp->buffers[bp->n_avail++] = bi;
580 3437350 : vlib_get_buffer (vm, bi);
581 : }
582 :
583 575 : bp->n_buffers = bp->n_avail;
584 :
585 575 : return bp->index;
586 : }
587 :
588 : static u8 *
589 202 : format_vlib_buffer_pool (u8 * s, va_list * va)
590 : {
591 202 : vlib_main_t *vm = va_arg (*va, vlib_main_t *);
592 202 : vlib_buffer_pool_t *bp = va_arg (*va, vlib_buffer_pool_t *);
593 : vlib_buffer_pool_thread_t *bpt;
594 202 : u32 cached = 0;
595 :
596 202 : if (!bp)
597 101 : return format (s, "%-20s%=6s%=6s%=6s%=11s%=6s%=8s%=8s%=8s",
598 : "Pool Name", "Index", "NUMA", "Size", "Data Size",
599 : "Total", "Avail", "Cached", "Used");
600 :
601 : /* *INDENT-OFF* */
602 215 : vec_foreach (bpt, bp->threads)
603 114 : cached += bpt->n_cached;
604 : /* *INDENT-ON* */
605 :
606 101 : s = format (s, "%-20v%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", bp->name, bp->index,
607 101 : bp->numa_node,
608 101 : bp->data_size + sizeof (vlib_buffer_t) +
609 101 : vm->buffer_main->ext_hdr_size,
610 : bp->data_size, bp->n_buffers, bp->n_avail, cached,
611 101 : bp->n_buffers - bp->n_avail - cached);
612 :
613 101 : return s;
614 : }
615 :
616 : u8 *
617 101 : format_vlib_buffer_pool_all (u8 *s, va_list *va)
618 : {
619 101 : vlib_main_t *vm = va_arg (*va, vlib_main_t *);
620 101 : vlib_buffer_main_t *bm = vm->buffer_main;
621 : vlib_buffer_pool_t *bp;
622 :
623 101 : s = format (s, "%U", format_vlib_buffer_pool, vm, 0);
624 :
625 202 : vec_foreach (bp, bm->buffer_pools)
626 101 : s = format (s, "\n%U", format_vlib_buffer_pool, vm, bp);
627 :
628 101 : return s;
629 : }
630 :
631 : static clib_error_t *
632 101 : show_buffers (vlib_main_t *vm, unformat_input_t *input,
633 : vlib_cli_command_t *cmd)
634 : {
635 101 : vlib_cli_output (vm, "%U", format_vlib_buffer_pool_all, vm);
636 101 : return 0;
637 : }
638 :
639 : /* *INDENT-OFF* */
640 285289 : VLIB_CLI_COMMAND (show_buffers_command, static) = {
641 : .path = "show buffers",
642 : .short_help = "Show packet buffer allocation",
643 : .function = show_buffers,
644 : };
645 : /* *INDENT-ON* */
646 :
647 : clib_error_t *
648 575 : vlib_buffer_num_workers_change (vlib_main_t *vm)
649 : {
650 575 : vlib_buffer_main_t *bm = vm->buffer_main;
651 : vlib_buffer_pool_t *bp;
652 :
653 1150 : vec_foreach (bp, bm->buffer_pools)
654 575 : vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
655 : CLIB_CACHE_LINE_BYTES);
656 :
657 575 : return 0;
658 : }
659 :
660 1151 : VLIB_NUM_WORKERS_CHANGE_FN (vlib_buffer_num_workers_change);
661 :
662 : static clib_error_t *
663 2300 : vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node,
664 : u32 * physmem_map_index,
665 : clib_mem_page_sz_t log2_page_size,
666 : u8 unpriv)
667 : {
668 2300 : vlib_buffer_main_t *bm = vm->buffer_main;
669 2300 : u32 buffers_per_numa = bm->buffers_per_numa;
670 : clib_error_t *error;
671 : u32 buffer_size;
672 : uword n_pages, pagesize;
673 2300 : u8 *name = 0;
674 :
675 2300 : ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
676 :
677 2300 : pagesize = clib_mem_page_bytes (log2_page_size);
678 2300 : buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size,
679 2300 : vlib_buffer_get_default_data_size
680 : (vm));
681 2300 : if (buffer_size > pagesize)
682 0 : return clib_error_return (0, "buffer size (%llu) is greater than page "
683 : "size (%llu)", buffer_size, pagesize);
684 :
685 2300 : if (buffers_per_numa == 0)
686 2300 : buffers_per_numa = unpriv ? VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV :
687 : VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA;
688 :
689 2300 : name = format (0, "buffers-numa-%d%c", numa_node, 0);
690 2300 : n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1;
691 2300 : error = vlib_physmem_shared_map_create (vm, (char *) name,
692 : n_pages * pagesize,
693 2300 : min_log2 (pagesize), numa_node,
694 : physmem_map_index);
695 2300 : vec_free (name);
696 2300 : return error;
697 : }
698 :
699 : static clib_error_t *
700 1150 : vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
701 : u8 * index)
702 : {
703 1150 : vlib_buffer_main_t *bm = vm->buffer_main;
704 : u32 physmem_map_index;
705 : clib_error_t *error;
706 :
707 1150 : if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN)
708 : {
709 1150 : error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
710 : &physmem_map_index,
711 : CLIB_MEM_PAGE_SZ_DEFAULT_HUGE,
712 : 0 /* unpriv */ );
713 1150 : if (!error)
714 0 : goto buffer_pool_create;
715 :
716 : /* If alloc failed, retry without hugepages */
717 1150 : vlib_log_warn (bm->log_default,
718 : "numa[%u] falling back to non-hugepage backed "
719 : "buffer pool (%U)", numa_node, format_clib_error, error);
720 1150 : clib_error_free (error);
721 :
722 1150 : error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
723 : &physmem_map_index,
724 : CLIB_MEM_PAGE_SZ_DEFAULT,
725 : 1 /* unpriv */ );
726 : }
727 : else
728 0 : error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
729 : &physmem_map_index,
730 : bm->log2_page_size,
731 : 0 /* unpriv */ );
732 1150 : if (error)
733 575 : return error;
734 :
735 575 : buffer_pool_create:
736 575 : *index =
737 575 : vlib_buffer_pool_create (vm, vlib_buffer_get_default_data_size (vm),
738 : physmem_map_index, "default-numa-%d", numa_node);
739 :
740 575 : if (*index == (u8) ~ 0)
741 0 : error = clib_error_return (0, "maximum number of buffer pools reached");
742 :
743 :
744 575 : return error;
745 : }
746 :
747 : void
748 1150 : vlib_buffer_main_alloc (vlib_main_t * vm)
749 : {
750 : vlib_buffer_main_t *bm;
751 :
752 1150 : if (vm->buffer_main)
753 575 : return;
754 :
755 575 : vm->buffer_main = bm = clib_mem_alloc (sizeof (bm[0]));
756 575 : clib_memset (vm->buffer_main, 0, sizeof (bm[0]));
757 575 : bm->default_data_size = VLIB_BUFFER_DEFAULT_DATA_SIZE;
758 : }
759 :
760 : static u32
761 2512 : buffer_get_cached (vlib_buffer_pool_t * bp)
762 : {
763 2512 : u32 cached = 0;
764 : vlib_buffer_pool_thread_t *bpt;
765 :
766 2512 : clib_spinlock_lock (&bp->lock);
767 :
768 : /* *INDENT-OFF* */
769 5528 : vec_foreach (bpt, bp->threads)
770 3016 : cached += bpt->n_cached;
771 : /* *INDENT-ON* */
772 :
773 2512 : clib_spinlock_unlock (&bp->lock);
774 :
775 2512 : return cached;
776 : }
777 :
778 : static vlib_buffer_pool_t *
779 3768 : buffer_get_by_index (vlib_buffer_main_t * bm, u32 index)
780 : {
781 : vlib_buffer_pool_t *bp;
782 3768 : if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index)
783 0 : return 0;
784 3768 : bp = vec_elt_at_index (bm->buffer_pools, index);
785 :
786 3768 : return bp;
787 : }
788 :
789 : static void
790 1256 : buffer_gauges_collect_used_fn (vlib_stats_collector_data_t *d)
791 : {
792 1256 : vlib_main_t *vm = vlib_get_main ();
793 : vlib_buffer_pool_t *bp =
794 1256 : buffer_get_by_index (vm->buffer_main, d->private_data);
795 1256 : if (!bp)
796 0 : return;
797 :
798 1256 : d->entry->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
799 : }
800 :
801 : static void
802 1256 : buffer_gauges_collect_available_fn (vlib_stats_collector_data_t *d)
803 : {
804 1256 : vlib_main_t *vm = vlib_get_main ();
805 : vlib_buffer_pool_t *bp =
806 1256 : buffer_get_by_index (vm->buffer_main, d->private_data);
807 1256 : if (!bp)
808 0 : return;
809 :
810 1256 : d->entry->value = bp->n_avail;
811 : }
812 :
813 : static void
814 1256 : buffer_gauges_collect_cached_fn (vlib_stats_collector_data_t *d)
815 : {
816 1256 : vlib_main_t *vm = vlib_get_main ();
817 : vlib_buffer_pool_t *bp =
818 1256 : buffer_get_by_index (vm->buffer_main, d->private_data);
819 1256 : if (!bp)
820 0 : return;
821 :
822 1256 : d->entry->value = buffer_get_cached (bp);
823 : }
824 :
825 : clib_error_t *
826 575 : vlib_buffer_main_init (struct vlib_main_t * vm)
827 : {
828 : vlib_buffer_main_t *bm;
829 : clib_error_t *err;
830 575 : clib_bitmap_t *bmp = 0, *bmp_has_memory = 0;
831 : u32 numa_node;
832 : vlib_buffer_pool_t *bp;
833 575 : u8 *name = 0, first_valid_buffer_pool_index = ~0;
834 :
835 575 : vlib_buffer_main_alloc (vm);
836 :
837 575 : bm = vm->buffer_main;
838 575 : bm->log_default = vlib_log_register_class ("buffer", 0);
839 575 : bm->ext_hdr_size = __vlib_buffer_external_hdr_size;
840 :
841 575 : clib_spinlock_init (&bm->buffer_known_hash_lockp);
842 :
843 575 : if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
844 : unformat_bitmap_list, &bmp)))
845 0 : clib_error_free (err);
846 :
847 575 : if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U",
848 : unformat_bitmap_list, &bmp_has_memory)))
849 0 : clib_error_free (err);
850 :
851 575 : if (bmp && bmp_has_memory)
852 575 : bmp = clib_bitmap_and (bmp, bmp_has_memory);
853 :
854 : /* no info from sysfs, assuming that only numa 0 exists */
855 575 : if (bmp == 0)
856 0 : bmp = clib_bitmap_set (bmp, 0, 1);
857 :
858 575 : if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES)
859 0 : clib_panic ("system have more than %u NUMA nodes",
860 : VLIB_BUFFER_MAX_NUMA_NODES);
861 :
862 : /* *INDENT-OFF* */
863 1725 : clib_bitmap_foreach (numa_node, bmp)
864 : {
865 1150 : u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
866 1150 : index[0] = ~0;
867 1150 : if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index)))
868 : {
869 575 : clib_error_report (err);
870 575 : clib_error_free (err);
871 575 : continue;
872 : }
873 :
874 575 : if (first_valid_buffer_pool_index == 0xff)
875 575 : first_valid_buffer_pool_index = index[0];
876 : }
877 : /* *INDENT-ON* */
878 :
879 575 : if (first_valid_buffer_pool_index == (u8) ~ 0)
880 : {
881 0 : err = clib_error_return (0, "failed to allocate buffer pool(s)");
882 0 : goto done;
883 : }
884 :
885 : /* *INDENT-OFF* */
886 1725 : clib_bitmap_foreach (numa_node, bmp)
887 : {
888 1150 : if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0)
889 575 : bm->default_buffer_pool_index_for_numa[numa_node] =
890 : first_valid_buffer_pool_index;
891 : }
892 : /* *INDENT-ON* */
893 :
894 1150 : vec_foreach (bp, bm->buffer_pools)
895 : {
896 575 : vlib_stats_collector_reg_t reg = { .private_data = bp - bm->buffer_pools };
897 575 : if (bp->n_buffers == 0)
898 0 : continue;
899 :
900 575 : reg.entry_index =
901 575 : vlib_stats_add_gauge ("/buffer-pools/%v/cached", bp->name);
902 575 : reg.collect_fn = buffer_gauges_collect_cached_fn;
903 575 : vlib_stats_register_collector_fn (®);
904 :
905 575 : reg.entry_index = vlib_stats_add_gauge ("/buffer-pools/%v/used", bp->name);
906 575 : reg.collect_fn = buffer_gauges_collect_used_fn;
907 575 : vlib_stats_register_collector_fn (®);
908 :
909 575 : reg.entry_index =
910 575 : vlib_stats_add_gauge ("/buffer-pools/%v/available", bp->name);
911 575 : reg.collect_fn = buffer_gauges_collect_available_fn;
912 575 : vlib_stats_register_collector_fn (®);
913 : }
914 :
915 575 : done:
916 575 : vec_free (bmp);
917 575 : vec_free (bmp_has_memory);
918 575 : vec_free (name);
919 575 : return err;
920 : }
921 :
922 : static clib_error_t *
923 575 : vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
924 : {
925 : vlib_buffer_main_t *bm;
926 :
927 575 : vlib_buffer_main_alloc (vm);
928 :
929 575 : bm = vm->buffer_main;
930 575 : bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN;
931 :
932 575 : while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
933 : {
934 0 : if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa))
935 : ;
936 0 : else if (unformat (input, "page-size %U", unformat_log2_page_size,
937 : &bm->log2_page_size))
938 : ;
939 0 : else if (unformat (input, "default data-size %u",
940 : &bm->default_data_size))
941 : ;
942 : else
943 0 : return unformat_parse_error (input);
944 : }
945 :
946 575 : unformat_free (input);
947 575 : return 0;
948 : }
949 :
950 7514 : VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers");
951 :
952 : #if VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0
953 : u32
954 : vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers)
955 : {
956 : f64 r;
957 :
958 : r = random_f64 (&vm->buffer_alloc_success_seed);
959 :
960 : /* Fail this request? */
961 : if (r > vm->buffer_alloc_success_rate)
962 : n_buffers--;
963 : /* 5% chance of returning nothing at all */
964 : if (r > vm->buffer_alloc_success_rate && r > 0.95)
965 : n_buffers = 0;
966 :
967 : return n_buffers;
968 : }
969 : #endif
970 :
971 : __clib_export int
972 0 : vlib_buffer_set_alloc_free_callback (
973 : vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
974 : vlib_buffer_alloc_free_callback_t *free_callback_fn)
975 : {
976 0 : vlib_buffer_main_t *bm = vm->buffer_main;
977 0 : if ((alloc_callback_fn && bm->alloc_callback_fn) ||
978 0 : (free_callback_fn && bm->free_callback_fn))
979 0 : return 1;
980 0 : bm->alloc_callback_fn = alloc_callback_fn;
981 0 : bm->free_callback_fn = free_callback_fn;
982 0 : return 0;
983 : }
984 :
985 : /** @endcond */
986 : /*
987 : * fd.io coding-style-patch-verification: ON
988 : *
989 : * Local Variables:
990 : * eval: (c-set-style "gnu")
991 : * End:
992 : */
|
