Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * buffer.h: VLIB buffers
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #ifndef included_vlib_buffer_h
41 : #define included_vlib_buffer_h
42 :
43 : #include <vppinfra/types.h>
44 : #include <vppinfra/cache.h>
45 : #include <vppinfra/serialize.h>
46 : #include <vppinfra/vector.h>
47 : #include <vppinfra/lock.h>
48 : #include <vlib/error.h> /* for vlib_error_t */
49 :
50 : #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 : #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 :
53 : #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
54 :
55 : /* Minimum buffer chain segment size. Does not apply to last buffer in chain.
56 : Dataplane code can safely asume that specified amount of data is not split
57 : into 2 chained buffers */
58 : #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128)
59 :
60 : /* Amount of head buffer data copied to each replica head buffer */
61 : #define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
62 :
63 : /** \file
64 : vlib buffer structure definition and a few select
65 : access methods. This structure and the buffer allocation
66 : mechanism should perhaps live in vnet, but it would take a lot
67 : of typing to make it so.
68 : */
69 :
70 : /**
71 : * Buffer Flags
72 : */
73 : #define foreach_vlib_buffer_flag \
74 : _( 0, IS_TRACED, 0) \
75 : _( 1, NEXT_PRESENT, "next-present") \
76 : _( 2, TOTAL_LENGTH_VALID, 0) \
77 : _( 3, EXT_HDR_VALID, "ext-hdr-valid")
78 :
79 : /* NOTE: only buffer generic flags should be defined here, please consider
80 : using user flags. i.e. src/vnet/buffer.h */
81 :
82 : enum
83 : {
84 : #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
85 : foreach_vlib_buffer_flag
86 : #undef _
87 : };
88 :
89 : enum
90 : {
91 : #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
92 : foreach_vlib_buffer_flag
93 : #undef _
94 : };
95 :
96 : /* User defined buffer flags. */
97 : #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
98 : #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
99 : #define VLIB_BUFFER_FLAGS_ALL (0x0f)
100 :
101 : /** \brief Compile time buffer trajectory tracing option
102 : Turn this on if you run into "bad monkey" contexts,
103 : and you want to know exactly which nodes they've visited...
104 : See vlib/main.c...
105 : */
106 : #ifndef VLIB_BUFFER_TRACE_TRAJECTORY
107 : #define VLIB_BUFFER_TRACE_TRAJECTORY 0
108 : #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
109 :
110 : #define vlib_buffer_template_fields \
111 : /** signed offset in data[], pre_data[] that we are currently \
112 : * processing. If negative current header points into predata area. */ \
113 : i16 current_data; \
114 : \
115 : /** Nbytes between current data and the end of this buffer. */ \
116 : u16 current_length; \
117 : /** buffer flags: \
118 : <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list \
119 : index, <br> VLIB_BUFFER_IS_TRACED: trace this buffer. <br> \
120 : VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer. <br> \
121 : VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says <br> \
122 : VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager \
123 : header, set to avoid adding it to a flow report <br> \
124 : VLIB_BUFFER_FLAG_USER(n): user-defined bit N \
125 : */ \
126 : u32 flags; \
127 : \
128 : /** Generic flow identifier */ \
129 : u32 flow_id; \
130 : \
131 : /** Reference count for this buffer. */ \
132 : volatile u8 ref_count; \
133 : \
134 : /** index of buffer pool this buffer belongs. */ \
135 : u8 buffer_pool_index; \
136 : \
137 : /** Error code for buffers to be enqueued to error handler. */ \
138 : vlib_error_t error; \
139 : \
140 : /** Next buffer for this linked-list of buffers. Only valid if \
141 : * VLIB_BUFFER_NEXT_PRESENT flag is set. */ \
142 : u32 next_buffer; \
143 : \
144 : /** The following fields can be in a union because once a packet enters \
145 : * the punt path, it is no longer on a feature arc */ \
146 : union \
147 : { \
148 : /** Used by feature subgraph arcs to visit enabled feature nodes */ \
149 : u32 current_config_index; \
150 : /* the reason the packet once punted */ \
151 : u32 punt_reason; \
152 : }; \
153 : \
154 : /** Opaque data used by sub-graphs for their own purposes. */ \
155 : u32 opaque[10];
156 :
157 : typedef struct
158 : {
159 : CLIB_ALIGN_MARK (align_mark, 64);
160 : vlib_buffer_template_fields
161 : } vlib_buffer_template_t;
162 :
163 : STATIC_ASSERT_SIZEOF (vlib_buffer_template_t, 64);
164 :
165 : /** VLIB buffer representation. */
166 : typedef union
167 : {
168 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
169 : struct
170 : {
171 : union
172 : {
173 : struct
174 : {
175 : vlib_buffer_template_fields
176 : };
177 : vlib_buffer_template_t template;
178 : };
179 :
180 : /* Data above is initialized or zeroed on alloc, data bellow is not
181 : * and it is app responsibility to ensure data is valid */
182 :
183 : /** start of 2nd half (2nd cacheline on systems where cacheline size is 64) */
184 : CLIB_ALIGN_MARK (second_half, 64);
185 :
186 : /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
187 : * set. */
188 : u32 trace_handle;
189 :
190 : /** Only valid for first buffer in chain. Current length plus total length
191 : * given here give total number of bytes in buffer chain. */
192 : u32 total_length_not_including_first_buffer;
193 :
194 : /**< More opaque data, see ../vnet/vnet/buffer.h */
195 : u32 opaque2[14];
196 :
197 : #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
198 : /** trace trajectory data - we use a specific cacheline for that in the
199 : * buffer when it is compiled-in */
200 : #define VLIB_BUFFER_TRACE_TRAJECTORY_MAX 31
201 : #define VLIB_BUFFER_TRACE_TRAJECTORY_SZ 64
202 : #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) (b)->trajectory_nb = 0
203 : CLIB_ALIGN_MARK (trajectory, 64);
204 : u16 trajectory_nb;
205 : u16 trajectory_trace[VLIB_BUFFER_TRACE_TRAJECTORY_MAX];
206 : #else /* VLIB_BUFFER_TRACE_TRAJECTORY */
207 : #define VLIB_BUFFER_TRACE_TRAJECTORY_SZ 0
208 : #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
209 : #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
210 :
211 : /** start of buffer headroom */
212 : CLIB_ALIGN_MARK (headroom, 64);
213 :
214 : /** Space for inserting data before buffer start. Packet rewrite string
215 : * will be rewritten backwards and may extend back before
216 : * buffer->data[0]. Must come directly before packet data. */
217 : u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE];
218 :
219 : /** Packet data */
220 : u8 data[];
221 : };
222 : #ifdef CLIB_HAVE_VEC128
223 : u8x16 as_u8x16[4];
224 : #endif
225 : #ifdef CLIB_HAVE_VEC256
226 : u8x32 as_u8x32[2];
227 : #endif
228 : #ifdef CLIB_HAVE_VEC512
229 : u8x64 as_u8x64[1];
230 : #endif
231 : } vlib_buffer_t;
232 :
233 : STATIC_ASSERT_SIZEOF (vlib_buffer_t, 128 + VLIB_BUFFER_TRACE_TRAJECTORY_SZ +
234 : VLIB_BUFFER_PRE_DATA_SIZE);
235 : STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE % CLIB_CACHE_LINE_BYTES == 0,
236 : "VLIB_BUFFER_PRE_DATA_SIZE must be divisible by cache line size");
237 :
238 : #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
239 :
240 : /** \brief Prefetch buffer metadata.
241 : The first 64 bytes of buffer contains most header information
242 :
243 : @param b - (vlib_buffer_t *) pointer to the buffer
244 : @param type - LOAD, STORE. In most cases, STORE is the right answer
245 : */
246 :
247 : #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
248 : #define vlib_prefetch_buffer_data(b,type) \
249 : CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type)
250 :
251 : always_inline void
252 : vlib_buffer_struct_is_sane (vlib_buffer_t * b)
253 : {
254 : ASSERT (sizeof (b[0]) % 64 == 0);
255 :
256 : /* Rewrite data must be before and contiguous with packet data. */
257 : ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data);
258 : }
259 :
260 : always_inline uword
261 2 : vlib_buffer_get_va (vlib_buffer_t * b)
262 : {
263 4 : return pointer_to_uword (b->data);
264 : }
265 :
266 : /** \brief Get pointer to current data to process
267 :
268 : @param b - (vlib_buffer_t *) pointer to the buffer
269 : @return - (void *) (b->data + b->current_data)
270 : */
271 :
272 : always_inline void *
273 1511431618 : vlib_buffer_get_current (vlib_buffer_t * b)
274 : {
275 : /* Check bounds. */
276 1511431618 : ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
277 1511431620 : return b->data + b->current_data;
278 : }
279 :
280 : always_inline uword
281 1 : vlib_buffer_get_current_va (vlib_buffer_t * b)
282 : {
283 1 : return vlib_buffer_get_va (b) + b->current_data;
284 : }
285 :
286 : /** \brief Advance current data pointer by the supplied (signed!) amount
287 :
288 : @param b - (vlib_buffer_t *) pointer to the buffer
289 : @param l - (word) signed increment
290 : */
291 : always_inline void
292 93300317 : vlib_buffer_advance (vlib_buffer_t * b, word l)
293 : {
294 93300317 : ASSERT (b->current_length >= l);
295 93300317 : b->current_data += l;
296 93300317 : b->current_length -= l;
297 :
298 93300317 : ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
299 : b->current_length >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE);
300 93300317 : }
301 :
302 : /** \brief Check if there is enough space in buffer to advance
303 :
304 : @param b - (vlib_buffer_t *) pointer to the buffer
305 : @param l - (word) size to check
306 : @return - 0 if there is less space than 'l' in buffer
307 : */
308 : always_inline u8
309 168178 : vlib_buffer_has_space (vlib_buffer_t * b, word l)
310 : {
311 168178 : return b->current_length >= l;
312 : }
313 :
314 : /** \brief Reset current header & length to state they were in when
315 : packet was received.
316 :
317 : @param b - (vlib_buffer_t *) pointer to the buffer
318 : */
319 :
320 : always_inline void
321 50 : vlib_buffer_reset (vlib_buffer_t * b)
322 : {
323 50 : b->current_length += clib_max (b->current_data, 0);
324 50 : b->current_data = 0;
325 50 : }
326 :
327 : /** \brief Get pointer to buffer's opaque data array
328 :
329 : @param b - (vlib_buffer_t *) pointer to the buffer
330 : @return - (void *) b->opaque
331 : */
332 : always_inline void *
333 : vlib_get_buffer_opaque (vlib_buffer_t * b)
334 : {
335 : return (void *) b->opaque;
336 : }
337 :
338 : /** \brief Get pointer to buffer's opaque2 data array
339 :
340 : @param b - (vlib_buffer_t *) pointer to the buffer
341 : @return - (void *) b->opaque2
342 : */
343 : always_inline void *
344 : vlib_get_buffer_opaque2 (vlib_buffer_t * b)
345 : {
346 : return (void *) b->opaque2;
347 : }
348 :
349 : /** \brief Get pointer to the end of buffer's data
350 : * @param b pointer to the buffer
351 : * @return pointer to tail of packet's data
352 : */
353 : always_inline u8 *
354 590045 : vlib_buffer_get_tail (vlib_buffer_t * b)
355 : {
356 590045 : return b->data + b->current_data + b->current_length;
357 : }
358 :
359 : /** \brief Append uninitialized data to buffer
360 : * @param b pointer to the buffer
361 : * @param size number of uninitialized bytes
362 : * @return pointer to beginning of uninitialized data
363 : */
364 : always_inline void *
365 1 : vlib_buffer_put_uninit (vlib_buffer_t * b, u16 size)
366 : {
367 1 : void *p = vlib_buffer_get_tail (b);
368 : /* XXX make sure there's enough space */
369 1 : b->current_length += size;
370 1 : return p;
371 : }
372 :
373 : /** \brief Prepend uninitialized data to buffer
374 : * @param b pointer to the buffer
375 : * @param size number of uninitialized bytes
376 : * @return pointer to beginning of uninitialized data
377 : */
378 : always_inline void *
379 2243181 : vlib_buffer_push_uninit (vlib_buffer_t * b, u8 size)
380 : {
381 2243181 : ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
382 2243181 : b->current_data -= size;
383 2243181 : b->current_length += size;
384 :
385 2243181 : return vlib_buffer_get_current (b);
386 : }
387 :
388 : /** \brief Make head room, typically for packet headers
389 : * @param b pointer to the buffer
390 : * @param size number of head room bytes
391 : * @return pointer to start of buffer (current data)
392 : */
393 : always_inline void *
394 1121510 : vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size)
395 : {
396 1121510 : b->current_data += size;
397 1121510 : return vlib_buffer_get_current (b);
398 : }
399 :
400 : /** \brief Construct a trace handle from thread and pool index
401 : * @param thread Thread id
402 : * @param pool_index Pool index
403 : * @return trace handle
404 : */
405 : always_inline u32
406 721940 : vlib_buffer_make_trace_handle (u32 thread, u32 pool_index)
407 : {
408 : u32 rv;
409 721940 : ASSERT (thread < 0xff);
410 721940 : ASSERT (pool_index < 0x00FFFFFF);
411 721939 : rv = (thread << 24) | (pool_index & 0x00FFFFFF);
412 721939 : return rv;
413 : }
414 :
415 : /** \brief Extract the thread id from a trace handle
416 : * @param trace_handle the trace handle
417 : * @return the thread id
418 : */
419 : always_inline u32
420 6142748 : vlib_buffer_get_trace_thread (vlib_buffer_t * b)
421 : {
422 6142748 : u32 trace_handle = b->trace_handle;
423 :
424 6142748 : return trace_handle >> 24;
425 : }
426 :
427 : /** \brief Extract the trace (pool) index from a trace handle
428 : * @param trace_handle the trace handle
429 : * @return the trace index
430 : */
431 : always_inline u32
432 6194713 : vlib_buffer_get_trace_index (vlib_buffer_t * b)
433 : {
434 6194713 : u32 trace_handle = b->trace_handle;
435 6194713 : return trace_handle & 0x00FFFFFF;
436 : }
437 :
438 : /** \brief Retrieve bytes from buffer head
439 : * @param b pointer to the buffer
440 : * @param size number of bytes to pull
441 : * @return pointer to start of buffer (current data)
442 : */
443 : always_inline void *
444 1 : vlib_buffer_pull (vlib_buffer_t * b, u8 size)
445 : {
446 1 : if (b->current_length + VLIB_BUFFER_PRE_DATA_SIZE < size)
447 0 : return 0;
448 :
449 1 : void *data = vlib_buffer_get_current (b);
450 1 : vlib_buffer_advance (b, size);
451 1 : return data;
452 : }
453 :
454 : /* Forward declaration. */
455 : struct vlib_main_t;
456 :
457 : #define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ 512
458 :
459 : typedef struct
460 : {
461 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
462 : u32 cached_buffers[VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ];
463 : u32 n_cached;
464 : } vlib_buffer_pool_thread_t;
465 :
466 : typedef struct
467 : {
468 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
469 : uword start;
470 : uword size;
471 : u8 log2_page_size;
472 : u8 index;
473 : u8 numa_node;
474 : u32 physmem_map_index;
475 : u32 data_size;
476 : u32 alloc_size;
477 : u32 n_buffers;
478 : u32 n_avail;
479 : u32 *buffers;
480 : u8 *name;
481 : clib_spinlock_t lock;
482 :
483 : /* per-thread data */
484 : vlib_buffer_pool_thread_t *threads;
485 :
486 : /* buffer metadata template */
487 : vlib_buffer_template_t buffer_template;
488 : } vlib_buffer_pool_t;
489 :
490 : #define VLIB_BUFFER_MAX_NUMA_NODES 32
491 :
492 : typedef u32 (vlib_buffer_alloc_free_callback_t) (struct vlib_main_t *vm,
493 : u8 buffer_pool_index,
494 : u32 *buffers, u32 n_buffers);
495 :
496 : typedef struct
497 : {
498 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
499 : /* Virtual memory address and size of buffer memory, used for calculating
500 : buffer index */
501 : uword buffer_mem_start;
502 : uword buffer_mem_size;
503 : vlib_buffer_pool_t *buffer_pools;
504 :
505 : vlib_buffer_alloc_free_callback_t *alloc_callback_fn;
506 : vlib_buffer_alloc_free_callback_t *free_callback_fn;
507 :
508 : u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
509 :
510 : /* config */
511 : u32 buffers_per_numa;
512 : u16 ext_hdr_size;
513 : u32 default_data_size;
514 : clib_mem_page_sz_t log2_page_size;
515 :
516 : /* Hash table mapping buffer index into number
517 : 0 => allocated but free, 1 => allocated and not-free.
518 : If buffer index is not in hash table then this buffer
519 : has never been allocated. */
520 : uword *buffer_known_hash;
521 : clib_spinlock_t buffer_known_hash_lockp;
522 :
523 : /* logging */
524 : vlib_log_class_t log_default;
525 : } vlib_buffer_main_t;
526 :
527 : clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
528 :
529 : format_function_t format_vlib_buffer_pool_all;
530 :
531 : int vlib_buffer_set_alloc_free_callback (
532 : struct vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
533 : vlib_buffer_alloc_free_callback_t *free_callback_fn);
534 :
535 : extern u16 __vlib_buffer_external_hdr_size;
536 : #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
537 : static void __clib_constructor \
538 : vnet_buffer_set_ext_hdr_size() \
539 : { \
540 : if (__vlib_buffer_external_hdr_size) \
541 : clib_error ("buffer external header space already set"); \
542 : __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
543 : }
544 :
545 : #endif /* included_vlib_buffer_h */
546 :
547 : /*
548 : * fd.io coding-style-patch-verification: ON
549 : *
550 : * Local Variables:
551 : * eval: (c-set-style "gnu")
552 : * End:
553 : */
|
