Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : * buffer.h: VLIB buffers
17 : *
18 : * Copyright (c) 2008 Eliot Dresselhaus
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining
21 : * a copy of this software and associated documentation files (the
22 : * "Software"), to deal in the Software without restriction, including
23 : * without limitation the rights to use, copy, modify, merge, publish,
24 : * distribute, sublicense, and/or sell copies of the Software, and to
25 : * permit persons to whom the Software is furnished to do so, subject to
26 : * the following conditions:
27 : *
28 : * The above copyright notice and this permission notice shall be
29 : * included in all copies or substantial portions of the Software.
30 : *
31 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 : * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 : * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 : * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 : * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 : * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 : * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 : */
39 :
40 : #ifndef included_vlib_buffer_h
41 : #define included_vlib_buffer_h
42 :
43 : #include <vppinfra/types.h>
44 : #include <vppinfra/cache.h>
45 : #include <vppinfra/serialize.h>
46 : #include <vppinfra/vector.h>
47 : #include <vppinfra/lock.h>
48 : #include <vlib/error.h> /* for vlib_error_t */
49 :
50 : #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 : #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 :
53 : #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
54 :
55 : /* Minimum buffer chain segment size. Does not apply to last buffer in chain.
56 : Dataplane code can safely asume that specified amount of data is not split
57 : into 2 chained buffers */
58 : #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128)
59 :
60 : /* Amount of head buffer data copied to each replica head buffer */
61 : #define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
62 :
63 : /** \file
64 : vlib buffer structure definition and a few select
65 : access methods. This structure and the buffer allocation
66 : mechanism should perhaps live in vnet, but it would take a lot
67 : of typing to make it so.
68 : */
69 :
70 : /**
71 : * Buffer Flags
72 : */
73 : #define foreach_vlib_buffer_flag \
74 : _( 0, IS_TRACED, 0) \
75 : _( 1, NEXT_PRESENT, "next-present") \
76 : _( 2, TOTAL_LENGTH_VALID, 0) \
77 : _( 3, EXT_HDR_VALID, "ext-hdr-valid")
78 :
79 : /* NOTE: only buffer generic flags should be defined here, please consider
80 : using user flags. i.e. src/vnet/buffer.h */
81 :
82 : enum
83 : {
84 : #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
85 : foreach_vlib_buffer_flag
86 : #undef _
87 : };
88 :
89 : enum
90 : {
91 : #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
92 : foreach_vlib_buffer_flag
93 : #undef _
94 : };
95 :
96 : /* User defined buffer flags. */
97 : #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
98 : #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
99 : #define VLIB_BUFFER_FLAGS_ALL (0x0f)
100 :
101 : /** \brief Compile time buffer trajectory tracing option
102 : Turn this on if you run into "bad monkey" contexts,
103 : and you want to know exactly which nodes they've visited...
104 : See vlib/main.c...
105 : */
106 : #ifndef VLIB_BUFFER_TRACE_TRAJECTORY
107 : #define VLIB_BUFFER_TRACE_TRAJECTORY 0
108 : #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
109 :
110 : /** VLIB buffer representation. */
111 : typedef union
112 : {
113 : struct
114 : {
115 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
116 :
117 : /** signed offset in data[], pre_data[] that we are currently
118 : * processing. If negative current header points into predata area. */
119 : i16 current_data;
120 :
121 : /** Nbytes between current data and the end of this buffer. */
122 : u16 current_length;
123 :
124 : /** buffer flags:
125 : <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
126 : <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
127 : <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
128 : <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
129 : <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
130 : set to avoid adding it to a flow report
131 : <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
132 : */
133 : u32 flags;
134 :
135 : /** Generic flow identifier */
136 : u32 flow_id;
137 :
138 : /** Reference count for this buffer. */
139 : volatile u8 ref_count;
140 :
141 : /** index of buffer pool this buffer belongs. */
142 : u8 buffer_pool_index;
143 :
144 : /** Error code for buffers to be enqueued to error handler. */
145 : vlib_error_t error;
146 :
147 : /** Next buffer for this linked-list of buffers. Only valid if
148 : * VLIB_BUFFER_NEXT_PRESENT flag is set. */
149 : u32 next_buffer;
150 :
151 : /** The following fields can be in a union because once a packet enters
152 : * the punt path, it is no longer on a feature arc */
153 : union
154 : {
155 : /** Used by feature subgraph arcs to visit enabled feature nodes */
156 : u32 current_config_index;
157 : /* the reason the packet once punted */
158 : u32 punt_reason;
159 : };
160 :
161 : /** Opaque data used by sub-graphs for their own purposes. */
162 : u32 opaque[10];
163 :
164 : /** part of buffer metadata which is initialized on alloc ends here. */
165 : STRUCT_MARK (template_end);
166 :
167 : /** start of 2nd half (2nd cacheline on systems where cacheline size is 64) */
168 : CLIB_ALIGN_MARK (second_half, 64);
169 :
170 : /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
171 : * set. */
172 : u32 trace_handle;
173 :
174 : /** Only valid for first buffer in chain. Current length plus total length
175 : * given here give total number of bytes in buffer chain. */
176 : u32 total_length_not_including_first_buffer;
177 :
178 : /**< More opaque data, see ../vnet/vnet/buffer.h */
179 : u32 opaque2[14];
180 :
181 : #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
182 : /** trace trajectory data - we use a specific cacheline for that in the
183 : * buffer when it is compiled-in */
184 : #define VLIB_BUFFER_TRACE_TRAJECTORY_MAX 31
185 : #define VLIB_BUFFER_TRACE_TRAJECTORY_SZ 64
186 : #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) (b)->trajectory_nb = 0
187 : CLIB_ALIGN_MARK (trajectory, 64);
188 : u16 trajectory_nb;
189 : u16 trajectory_trace[VLIB_BUFFER_TRACE_TRAJECTORY_MAX];
190 : #else /* VLIB_BUFFER_TRACE_TRAJECTORY */
191 : #define VLIB_BUFFER_TRACE_TRAJECTORY_SZ 0
192 : #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
193 : #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
194 :
195 : /** start of buffer headroom */
196 : CLIB_ALIGN_MARK (headroom, 64);
197 :
198 : /** Space for inserting data before buffer start. Packet rewrite string
199 : * will be rewritten backwards and may extend back before
200 : * buffer->data[0]. Must come directly before packet data. */
201 : u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE];
202 :
203 : /** Packet data */
204 : u8 data[];
205 : };
206 : #ifdef CLIB_HAVE_VEC128
207 : u8x16 as_u8x16[4];
208 : #endif
209 : #ifdef CLIB_HAVE_VEC256
210 : u8x32 as_u8x32[2];
211 : #endif
212 : #ifdef CLIB_HAVE_VEC512
213 : u8x64 as_u8x64[1];
214 : #endif
215 : } vlib_buffer_t;
216 :
217 : STATIC_ASSERT_SIZEOF (vlib_buffer_t, 128 + VLIB_BUFFER_TRACE_TRAJECTORY_SZ +
218 : VLIB_BUFFER_PRE_DATA_SIZE);
219 : STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE % CLIB_CACHE_LINE_BYTES == 0,
220 : "VLIB_BUFFER_PRE_DATA_SIZE must be divisible by cache line size");
221 :
222 : #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
223 :
224 : /** \brief Prefetch buffer metadata.
225 : The first 64 bytes of buffer contains most header information
226 :
227 : @param b - (vlib_buffer_t *) pointer to the buffer
228 : @param type - LOAD, STORE. In most cases, STORE is the right answer
229 : */
230 :
231 : #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
232 : #define vlib_prefetch_buffer_data(b,type) \
233 : CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type)
234 :
235 : always_inline void
236 : vlib_buffer_struct_is_sane (vlib_buffer_t * b)
237 : {
238 : ASSERT (sizeof (b[0]) % 64 == 0);
239 :
240 : /* Rewrite data must be before and contiguous with packet data. */
241 : ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data);
242 : }
243 :
244 : always_inline uword
245 0 : vlib_buffer_get_va (vlib_buffer_t * b)
246 : {
247 0 : return pointer_to_uword (b->data);
248 : }
249 :
250 : /** \brief Get pointer to current data to process
251 :
252 : @param b - (vlib_buffer_t *) pointer to the buffer
253 : @return - (void *) (b->data + b->current_data)
254 : */
255 :
256 : always_inline void *
257 1326179667 : vlib_buffer_get_current (vlib_buffer_t * b)
258 : {
259 : /* Check bounds. */
260 1326179667 : ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
261 1326179773 : return b->data + b->current_data;
262 : }
263 :
264 : always_inline uword
265 0 : vlib_buffer_get_current_va (vlib_buffer_t * b)
266 : {
267 0 : return vlib_buffer_get_va (b) + b->current_data;
268 : }
269 :
270 : /** \brief Advance current data pointer by the supplied (signed!) amount
271 :
272 : @param b - (vlib_buffer_t *) pointer to the buffer
273 : @param l - (word) signed increment
274 : */
275 : always_inline void
276 87668211 : vlib_buffer_advance (vlib_buffer_t * b, word l)
277 : {
278 87668211 : ASSERT (b->current_length >= l);
279 87668211 : b->current_data += l;
280 87668211 : b->current_length -= l;
281 :
282 87668211 : ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
283 : b->current_length >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE);
284 87668211 : }
285 :
286 : /** \brief Check if there is enough space in buffer to advance
287 :
288 : @param b - (vlib_buffer_t *) pointer to the buffer
289 : @param l - (word) size to check
290 : @return - 0 if there is less space than 'l' in buffer
291 : */
292 : always_inline u8
293 152856 : vlib_buffer_has_space (vlib_buffer_t * b, word l)
294 : {
295 152856 : return b->current_length >= l;
296 : }
297 :
298 : /** \brief Reset current header & length to state they were in when
299 : packet was received.
300 :
301 : @param b - (vlib_buffer_t *) pointer to the buffer
302 : */
303 :
304 : always_inline void
305 49 : vlib_buffer_reset (vlib_buffer_t * b)
306 : {
307 49 : b->current_length += clib_max (b->current_data, 0);
308 49 : b->current_data = 0;
309 49 : }
310 :
311 : /** \brief Get pointer to buffer's opaque data array
312 :
313 : @param b - (vlib_buffer_t *) pointer to the buffer
314 : @return - (void *) b->opaque
315 : */
316 : always_inline void *
317 : vlib_get_buffer_opaque (vlib_buffer_t * b)
318 : {
319 : return (void *) b->opaque;
320 : }
321 :
322 : /** \brief Get pointer to buffer's opaque2 data array
323 :
324 : @param b - (vlib_buffer_t *) pointer to the buffer
325 : @return - (void *) b->opaque2
326 : */
327 : always_inline void *
328 : vlib_get_buffer_opaque2 (vlib_buffer_t * b)
329 : {
330 : return (void *) b->opaque2;
331 : }
332 :
333 : /** \brief Get pointer to the end of buffer's data
334 : * @param b pointer to the buffer
335 : * @return pointer to tail of packet's data
336 : */
337 : always_inline u8 *
338 521719 : vlib_buffer_get_tail (vlib_buffer_t * b)
339 : {
340 521719 : return b->data + b->current_data + b->current_length;
341 : }
342 :
343 : /** \brief Append uninitialized data to buffer
344 : * @param b pointer to the buffer
345 : * @param size number of uninitialized bytes
346 : * @return pointer to beginning of uninitialized data
347 : */
348 : always_inline void *
349 0 : vlib_buffer_put_uninit (vlib_buffer_t * b, u16 size)
350 : {
351 0 : void *p = vlib_buffer_get_tail (b);
352 : /* XXX make sure there's enough space */
353 0 : b->current_length += size;
354 0 : return p;
355 : }
356 :
357 : /** \brief Prepend uninitialized data to buffer
358 : * @param b pointer to the buffer
359 : * @param size number of uninitialized bytes
360 : * @return pointer to beginning of uninitialized data
361 : */
362 : always_inline void *
363 2158134 : vlib_buffer_push_uninit (vlib_buffer_t * b, u8 size)
364 : {
365 2158134 : ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
366 2158134 : b->current_data -= size;
367 2158134 : b->current_length += size;
368 :
369 2158134 : return vlib_buffer_get_current (b);
370 : }
371 :
372 : /** \brief Make head room, typically for packet headers
373 : * @param b pointer to the buffer
374 : * @param size number of head room bytes
375 : * @return pointer to start of buffer (current data)
376 : */
377 : always_inline void *
378 1078980 : vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size)
379 : {
380 1078980 : b->current_data += size;
381 1078980 : return vlib_buffer_get_current (b);
382 : }
383 :
384 : /** \brief Construct a trace handle from thread and pool index
385 : * @param thread Thread id
386 : * @param pool_index Pool index
387 : * @return trace handle
388 : */
389 : always_inline u32
390 668202 : vlib_buffer_make_trace_handle (u32 thread, u32 pool_index)
391 : {
392 : u32 rv;
393 668202 : ASSERT (thread < 0xff);
394 668202 : ASSERT (pool_index < 0x00FFFFFF);
395 668204 : rv = (thread << 24) | (pool_index & 0x00FFFFFF);
396 668204 : return rv;
397 : }
398 :
399 : /** \brief Extract the thread id from a trace handle
400 : * @param trace_handle the trace handle
401 : * @return the thread id
402 : */
403 : always_inline u32
404 5708074 : vlib_buffer_get_trace_thread (vlib_buffer_t * b)
405 : {
406 5708074 : u32 trace_handle = b->trace_handle;
407 :
408 5708074 : return trace_handle >> 24;
409 : }
410 :
411 : /** \brief Extract the trace (pool) index from a trace handle
412 : * @param trace_handle the trace handle
413 : * @return the trace index
414 : */
415 : always_inline u32
416 5760218 : vlib_buffer_get_trace_index (vlib_buffer_t * b)
417 : {
418 5760218 : u32 trace_handle = b->trace_handle;
419 5760218 : return trace_handle & 0x00FFFFFF;
420 : }
421 :
422 : /** \brief Retrieve bytes from buffer head
423 : * @param b pointer to the buffer
424 : * @param size number of bytes to pull
425 : * @return pointer to start of buffer (current data)
426 : */
427 : always_inline void *
428 0 : vlib_buffer_pull (vlib_buffer_t * b, u8 size)
429 : {
430 0 : if (b->current_length + VLIB_BUFFER_PRE_DATA_SIZE < size)
431 0 : return 0;
432 :
433 0 : void *data = vlib_buffer_get_current (b);
434 0 : vlib_buffer_advance (b, size);
435 0 : return data;
436 : }
437 :
438 : /* Forward declaration. */
439 : struct vlib_main_t;
440 :
441 : #define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ 512
442 :
443 : typedef struct
444 : {
445 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
446 : u32 cached_buffers[VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ];
447 : u32 n_cached;
448 : } vlib_buffer_pool_thread_t;
449 :
450 : typedef struct
451 : {
452 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
453 : uword start;
454 : uword size;
455 : uword log2_page_size;
456 : u8 index;
457 : u32 numa_node;
458 : u32 physmem_map_index;
459 : u32 data_size;
460 : u32 n_buffers;
461 : u32 n_avail;
462 : u32 *buffers;
463 : u8 *name;
464 : clib_spinlock_t lock;
465 :
466 : /* per-thread data */
467 : vlib_buffer_pool_thread_t *threads;
468 :
469 : /* buffer metadata template */
470 : vlib_buffer_t buffer_template;
471 : } vlib_buffer_pool_t;
472 :
473 : #define VLIB_BUFFER_MAX_NUMA_NODES 32
474 :
475 : typedef u32 (vlib_buffer_alloc_free_callback_t) (struct vlib_main_t *vm,
476 : u8 buffer_pool_index,
477 : u32 *buffers, u32 n_buffers);
478 :
479 : typedef struct
480 : {
481 : CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
482 : /* Virtual memory address and size of buffer memory, used for calculating
483 : buffer index */
484 : uword buffer_mem_start;
485 : uword buffer_mem_size;
486 : vlib_buffer_pool_t *buffer_pools;
487 :
488 : vlib_buffer_alloc_free_callback_t *alloc_callback_fn;
489 : vlib_buffer_alloc_free_callback_t *free_callback_fn;
490 :
491 : u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
492 :
493 : /* config */
494 : u32 buffers_per_numa;
495 : u16 ext_hdr_size;
496 : u32 default_data_size;
497 : clib_mem_page_sz_t log2_page_size;
498 :
499 : /* Hash table mapping buffer index into number
500 : 0 => allocated but free, 1 => allocated and not-free.
501 : If buffer index is not in hash table then this buffer
502 : has never been allocated. */
503 : uword *buffer_known_hash;
504 : clib_spinlock_t buffer_known_hash_lockp;
505 :
506 : /* logging */
507 : vlib_log_class_t log_default;
508 : } vlib_buffer_main_t;
509 :
510 : clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
511 :
512 : format_function_t format_vlib_buffer_pool_all;
513 :
514 : int vlib_buffer_set_alloc_free_callback (
515 : struct vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
516 : vlib_buffer_alloc_free_callback_t *free_callback_fn);
517 :
518 : extern u16 __vlib_buffer_external_hdr_size;
519 : #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
520 : static void __clib_constructor \
521 : vnet_buffer_set_ext_hdr_size() \
522 : { \
523 : if (__vlib_buffer_external_hdr_size) \
524 : clib_error ("buffer external header space already set"); \
525 : __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
526 : }
527 :
528 : #endif /* included_vlib_buffer_h */
529 :
530 : /*
531 : * fd.io coding-style-patch-verification: ON
532 : *
533 : * Local Variables:
534 : * eval: (c-set-style "gnu")
535 : * End:
536 : */
|
