Line data Source code
1 : /*
2 : * Copyright (c) 2015 Cisco and/or its affiliates.
3 : * Licensed under the Apache License, Version 2.0 (the "License");
4 : * you may not use this file except in compliance with the License.
5 : * You may obtain a copy of the License at:
6 : *
7 : * http://www.apache.org/licenses/LICENSE-2.0
8 : *
9 : * Unless required by applicable law or agreed to in writing, software
10 : * distributed under the License is distributed on an "AS IS" BASIS,
11 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : * See the License for the specific language governing permissions and
13 : * limitations under the License.
14 : */
15 : /*
16 : Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 :
18 : Permission is hereby granted, free of charge, to any person obtaining
19 : a copy of this software and associated documentation files (the
20 : "Software"), to deal in the Software without restriction, including
21 : without limitation the rights to use, copy, modify, merge, publish,
22 : distribute, sublicense, and/or sell copies of the Software, and to
23 : permit persons to whom the Software is furnished to do so, subject to
24 : the following conditions:
25 :
26 : The above copyright notice and this permission notice shall be
27 : included in all copies or substantial portions of the Software.
28 :
29 : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 : EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 : MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 : NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33 : LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34 : OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35 : WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 : */
37 :
38 : #ifndef _included_clib_mem_h
39 : #define _included_clib_mem_h
40 :
41 : #include <stdarg.h>
42 : #include <unistd.h>
43 : #include <sys/mman.h>
44 :
45 : #include <vppinfra/clib.h> /* uword, etc */
46 : #include <vppinfra/clib_error.h>
47 :
48 : #include <vppinfra/os.h>
49 : #include <vppinfra/string.h> /* memcpy, clib_memset */
50 : #ifdef CLIB_SANITIZE_ADDR
51 : #include <sanitizer/asan_interface.h>
52 : #endif
53 :
54 : #define CLIB_MAX_MHEAPS 256
55 : #define CLIB_MAX_NUMAS 16
56 : #define CLIB_MEM_VM_MAP_FAILED ((void *) ~0)
57 : #define CLIB_MEM_ERROR (-1)
58 : #define CLIB_MEM_LOG2_MIN_ALIGN (3)
59 : #define CLIB_MEM_MIN_ALIGN (1 << CLIB_MEM_LOG2_MIN_ALIGN)
60 :
61 : typedef enum
62 : {
63 : CLIB_MEM_PAGE_SZ_UNKNOWN = 0,
64 : CLIB_MEM_PAGE_SZ_DEFAULT = 1,
65 : CLIB_MEM_PAGE_SZ_DEFAULT_HUGE = 2,
66 : CLIB_MEM_PAGE_SZ_4K = 12,
67 : CLIB_MEM_PAGE_SZ_16K = 14,
68 : CLIB_MEM_PAGE_SZ_64K = 16,
69 : CLIB_MEM_PAGE_SZ_1M = 20,
70 : CLIB_MEM_PAGE_SZ_2M = 21,
71 : CLIB_MEM_PAGE_SZ_16M = 24,
72 : CLIB_MEM_PAGE_SZ_32M = 25,
73 : CLIB_MEM_PAGE_SZ_512M = 29,
74 : CLIB_MEM_PAGE_SZ_1G = 30,
75 : CLIB_MEM_PAGE_SZ_16G = 34,
76 : } clib_mem_page_sz_t;
77 :
78 : typedef struct _clib_mem_vm_map_hdr
79 : {
80 : /* base address */
81 : uword base_addr;
82 :
83 : /* number of pages */
84 : uword num_pages;
85 :
86 : /* page size (log2) */
87 : clib_mem_page_sz_t log2_page_sz;
88 :
89 : /* file descriptor, -1 if memory is not shared */
90 : int fd;
91 :
92 : /* allocation mame */
93 : #define CLIB_VM_MAP_HDR_NAME_MAX_LEN 64
94 : char name[CLIB_VM_MAP_HDR_NAME_MAX_LEN];
95 :
96 : /* linked list */
97 : struct _clib_mem_vm_map_hdr *prev, *next;
98 : } clib_mem_vm_map_hdr_t;
99 :
100 : #define foreach_clib_mem_heap_flag \
101 : _ (0, LOCKED, "locked") \
102 : _ (1, UNMAP_ON_DESTROY, "unmap-on-destroy") \
103 : _ (2, TRACED, "traced")
104 :
105 : typedef enum
106 : {
107 : #define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i),
108 : foreach_clib_mem_heap_flag
109 : #undef _
110 : } clib_mem_heap_flag_t;
111 :
112 : typedef struct
113 : {
114 : /* base address */
115 : void *base;
116 :
117 : /* dlmalloc mspace */
118 : void *mspace;
119 :
120 : /* heap size */
121 : uword size;
122 :
123 : /* page size (log2) */
124 : clib_mem_page_sz_t log2_page_sz:8;
125 :
126 : /* flags */
127 : clib_mem_heap_flag_t flags:8;
128 :
129 : /* name - _MUST_ be last */
130 : char name[0];
131 : } clib_mem_heap_t;
132 :
133 : typedef struct
134 : {
135 : /* log2 system page size */
136 : clib_mem_page_sz_t log2_page_sz;
137 :
138 : /* log2 default hugepage size */
139 : clib_mem_page_sz_t log2_default_hugepage_sz;
140 :
141 : /* log2 system default hugepage size */
142 : clib_mem_page_sz_t log2_sys_default_hugepage_sz;
143 :
144 : /* bitmap of available numa nodes */
145 : u32 numa_node_bitmap;
146 :
147 : /* per CPU heaps */
148 : void *per_cpu_mheaps[CLIB_MAX_MHEAPS];
149 :
150 : /* per NUMA heaps */
151 : void *per_numa_mheaps[CLIB_MAX_NUMAS];
152 :
153 : /* memory maps */
154 : clib_mem_vm_map_hdr_t *first_map, *last_map;
155 :
156 : /* map lock */
157 : u8 map_lock;
158 :
159 : /* last error */
160 : clib_error_t *error;
161 : } clib_mem_main_t;
162 :
163 : extern clib_mem_main_t clib_mem_main;
164 :
165 : /* Unspecified NUMA socket */
166 : #define VEC_NUMA_UNSPECIFIED (0xFF)
167 :
168 : static_always_inline void
169 284607610 : clib_mem_poison (const void volatile *p, uword s)
170 : {
171 : #ifdef CLIB_SANITIZE_ADDR
172 : ASAN_POISON_MEMORY_REGION (p, s);
173 : #endif
174 284607610 : }
175 :
176 : static_always_inline void
177 819295068 : clib_mem_unpoison (const void volatile *p, uword s)
178 : {
179 : #ifdef CLIB_SANITIZE_ADDR
180 : ASAN_UNPOISON_MEMORY_REGION (p, s);
181 : #endif
182 819295068 : }
183 :
184 : always_inline clib_mem_heap_t *
185 175486802 : clib_mem_get_per_cpu_heap (void)
186 : {
187 175486802 : int cpu = os_get_thread_index ();
188 175486802 : return clib_mem_main.per_cpu_mheaps[cpu];
189 : }
190 :
191 : always_inline void *
192 3866817 : clib_mem_set_per_cpu_heap (void *new_heap)
193 : {
194 3866817 : int cpu = os_get_thread_index ();
195 3866816 : void *old = clib_mem_main.per_cpu_mheaps[cpu];
196 3866816 : clib_mem_main.per_cpu_mheaps[cpu] = new_heap;
197 3866816 : return old;
198 : }
199 :
200 : always_inline void *
201 : clib_mem_get_per_numa_heap (u32 numa_id)
202 : {
203 : ASSERT (numa_id < ARRAY_LEN (clib_mem_main.per_numa_mheaps));
204 : return clib_mem_main.per_numa_mheaps[numa_id];
205 : }
206 :
207 : always_inline void *
208 575 : clib_mem_set_per_numa_heap (void *new_heap)
209 : {
210 575 : int numa = os_get_numa_index ();
211 575 : void *old = clib_mem_main.per_numa_mheaps[numa];
212 575 : clib_mem_main.per_numa_mheaps[numa] = new_heap;
213 575 : return old;
214 : }
215 :
216 : always_inline void
217 74 : clib_mem_set_thread_index (void)
218 : {
219 : /*
220 : * Find an unused slot in the per-cpu-mheaps array,
221 : * and grab it for this thread. We need to be able to
222 : * push/pop the thread heap without affecting other thread(s).
223 : */
224 : int i;
225 74 : if (__os_thread_index != 0)
226 0 : return;
227 181 : for (i = 0; i < ARRAY_LEN (clib_mem_main.per_cpu_mheaps); i++)
228 181 : if (clib_atomic_bool_cmp_and_swap (&clib_mem_main.per_cpu_mheaps[i],
229 : 0, clib_mem_main.per_cpu_mheaps[0]))
230 : {
231 74 : os_set_thread_index (i);
232 74 : break;
233 : }
234 74 : ASSERT (__os_thread_index > 0);
235 : }
236 :
237 : /* Memory allocator which calls os_out_of_memory() when it fails */
238 : void *clib_mem_alloc (uword size);
239 : void *clib_mem_alloc_aligned (uword size, uword align);
240 : void *clib_mem_alloc_or_null (uword size);
241 : void *clib_mem_alloc_aligned_or_null (uword size, uword align);
242 : void *clib_mem_realloc (void *p, uword new_size);
243 : void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
244 : uword clib_mem_is_heap_object (void *p);
245 : void clib_mem_free (void *p);
246 :
247 : void *clib_mem_heap_alloc (void *heap, uword size);
248 : void *clib_mem_heap_alloc_aligned (void *heap, uword size, uword align);
249 : void *clib_mem_heap_alloc_or_null (void *heap, uword size);
250 : void *clib_mem_heap_alloc_aligned_or_null (void *heap, uword size,
251 : uword align);
252 : void *clib_mem_heap_realloc (void *heap, void *p, uword new_size);
253 : void *clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size,
254 : uword align);
255 : uword clib_mem_heap_is_heap_object (void *heap, void *p);
256 : void clib_mem_heap_free (void *heap, void *p);
257 :
258 : uword clib_mem_size (void *p);
259 : void clib_mem_free_s (void *p);
260 :
261 : /* Memory allocator which panics when it fails.
262 : Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
263 : #define clib_mem_alloc_aligned_no_fail(size,align) \
264 : ({ \
265 : uword _clib_mem_alloc_size = (size); \
266 : void * _clib_mem_alloc_p; \
267 : _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \
268 : if (! _clib_mem_alloc_p) \
269 : clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \
270 : _clib_mem_alloc_p; \
271 : })
272 :
273 : #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1)
274 :
275 : /* Alias to stack allocator for naming consistency. */
276 : #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
277 :
278 : always_inline clib_mem_heap_t *
279 11800 : clib_mem_get_heap (void)
280 : {
281 11800 : return clib_mem_get_per_cpu_heap ();
282 : }
283 :
284 : always_inline clib_mem_heap_t *
285 3866817 : clib_mem_set_heap (clib_mem_heap_t * heap)
286 : {
287 3866817 : return clib_mem_set_per_cpu_heap (heap);
288 : }
289 :
290 : void clib_mem_destroy_heap (clib_mem_heap_t * heap);
291 : clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked,
292 : char *fmt, ...);
293 :
294 : void clib_mem_main_init ();
295 : void *clib_mem_init (void *base, uword size);
296 : void *clib_mem_init_with_page_size (uword memory_size,
297 : clib_mem_page_sz_t log2_page_sz);
298 : void *clib_mem_init_thread_safe (void *memory, uword memory_size);
299 :
300 : void clib_mem_exit (void);
301 :
302 : void clib_mem_trace (int enable);
303 :
304 : int clib_mem_is_traced (void);
305 :
306 : typedef struct
307 : {
308 : /* Total number of objects allocated. */
309 : uword object_count;
310 :
311 : /* Total allocated bytes. Bytes used and free.
312 : used + free = total */
313 : uword bytes_total, bytes_used, bytes_free;
314 :
315 : /* Number of bytes used by mheap data structure overhead
316 : (e.g. free lists, mheap header). */
317 : uword bytes_overhead;
318 :
319 : /* Amount of free space returned to operating system. */
320 : uword bytes_free_reclaimed;
321 :
322 : /* For malloc which puts small objects in sbrk region and
323 : large objects in mmap'ed regions. */
324 : uword bytes_used_sbrk;
325 : uword bytes_used_mmap;
326 :
327 : /* Max. number of bytes in this heap. */
328 : uword bytes_max;
329 : } clib_mem_usage_t;
330 :
331 : void clib_mem_get_heap_usage (clib_mem_heap_t * heap,
332 : clib_mem_usage_t * usage);
333 :
334 : void *clib_mem_get_heap_base (clib_mem_heap_t * heap);
335 : uword clib_mem_get_heap_size (clib_mem_heap_t * heap);
336 : uword clib_mem_get_heap_free_space (clib_mem_heap_t * heap);
337 :
338 : u8 *format_clib_mem_usage (u8 * s, va_list * args);
339 : u8 *format_clib_mem_heap (u8 * s, va_list * va);
340 : u8 *format_clib_mem_page_stats (u8 * s, va_list * va);
341 :
342 : /* Allocate virtual address space. */
343 : always_inline void *
344 0 : clib_mem_vm_alloc (uword size)
345 : {
346 : void *mmap_addr;
347 0 : uword flags = MAP_PRIVATE;
348 :
349 : #ifdef MAP_ANONYMOUS
350 0 : flags |= MAP_ANONYMOUS;
351 : #endif
352 :
353 0 : mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
354 0 : if (mmap_addr == (void *) -1)
355 0 : mmap_addr = 0;
356 : else
357 0 : clib_mem_unpoison (mmap_addr, size);
358 :
359 0 : return mmap_addr;
360 : }
361 :
362 : always_inline void
363 0 : clib_mem_vm_free (void *addr, uword size)
364 : {
365 0 : munmap (addr, size);
366 0 : }
367 :
368 : void *clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz,
369 : uword size, int fd, uword offset, char *name);
370 :
371 : void *clib_mem_vm_map (void *start, uword size,
372 : clib_mem_page_sz_t log2_page_size, char *fmt, ...);
373 : void *clib_mem_vm_map_stack (uword size, clib_mem_page_sz_t log2_page_size,
374 : char *fmt, ...);
375 : void *clib_mem_vm_map_shared (void *start, uword size, int fd, uword offset,
376 : char *fmt, ...);
377 : int clib_mem_vm_unmap (void *base);
378 : clib_mem_vm_map_hdr_t *clib_mem_vm_get_next_map_hdr (clib_mem_vm_map_hdr_t *
379 : hdr);
380 :
381 : static_always_inline clib_mem_page_sz_t
382 9458061 : clib_mem_get_log2_page_size (void)
383 : {
384 9458061 : return clib_mem_main.log2_page_sz;
385 : }
386 :
387 : static_always_inline uword
388 93139 : clib_mem_get_page_size (void)
389 : {
390 93139 : return 1ULL << clib_mem_main.log2_page_sz;
391 : }
392 :
393 : static_always_inline void
394 0 : clib_mem_set_log2_default_hugepage_size (clib_mem_page_sz_t log2_page_sz)
395 : {
396 0 : clib_mem_main.log2_default_hugepage_sz = log2_page_sz;
397 0 : }
398 :
399 : static_always_inline clib_mem_page_sz_t
400 1150 : clib_mem_get_log2_default_hugepage_size ()
401 : {
402 1150 : return clib_mem_main.log2_default_hugepage_sz;
403 : }
404 :
405 : static_always_inline uword
406 575 : clib_mem_get_default_hugepage_size (void)
407 : {
408 575 : return 1ULL << clib_mem_main.log2_default_hugepage_sz;
409 : }
410 :
411 : int clib_mem_vm_create_fd (clib_mem_page_sz_t log2_page_size, char *fmt, ...);
412 : uword clib_mem_get_fd_page_size (int fd);
413 : clib_mem_page_sz_t clib_mem_get_fd_log2_page_size (int fd);
414 : uword clib_mem_vm_reserve (uword start, uword size,
415 : clib_mem_page_sz_t log2_page_sz);
416 : u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size,
417 : int n_pages);
418 : void clib_mem_destroy (void);
419 : int clib_mem_set_numa_affinity (u8 numa_node, int force);
420 : int clib_mem_set_default_numa_affinity ();
421 : void clib_mem_vm_randomize_va (uword * requested_va,
422 : clib_mem_page_sz_t log2_page_size);
423 : void mheap_trace (clib_mem_heap_t * v, int enable);
424 : uword clib_mem_trace_enable_disable (uword enable);
425 : void clib_mem_trace (int enable);
426 :
427 : always_inline uword
428 : clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size)
429 : {
430 : ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
431 :
432 : if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT)
433 : log2_page_size = clib_mem_get_log2_page_size ();
434 : else if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
435 : log2_page_size = clib_mem_get_log2_default_hugepage_size ();
436 :
437 : return round_pow2 (size, 1ULL << log2_page_size);
438 : }
439 :
440 : typedef struct
441 : {
442 : clib_mem_page_sz_t log2_page_sz;
443 : uword total;
444 : uword mapped;
445 : uword not_mapped;
446 : uword per_numa[CLIB_MAX_NUMAS];
447 : uword unknown;
448 : } clib_mem_page_stats_t;
449 :
450 : void clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
451 : uword n_pages, clib_mem_page_stats_t * stats);
452 :
453 : static_always_inline int
454 0 : vlib_mem_get_next_numa_node (int numa)
455 : {
456 0 : clib_mem_main_t *mm = &clib_mem_main;
457 0 : u32 bitmap = mm->numa_node_bitmap;
458 :
459 0 : if (numa >= 0)
460 0 : bitmap &= ~pow2_mask (numa + 1);
461 0 : if (bitmap == 0)
462 0 : return -1;
463 :
464 0 : return count_trailing_zeros (bitmap);
465 : }
466 :
467 : static_always_inline clib_mem_page_sz_t
468 8957 : clib_mem_log2_page_size_validate (clib_mem_page_sz_t log2_page_size)
469 : {
470 8957 : if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT)
471 3484 : return clib_mem_get_log2_page_size ();
472 5473 : if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
473 1150 : return clib_mem_get_log2_default_hugepage_size ();
474 4323 : return log2_page_size;
475 : }
476 :
477 : static_always_inline uword
478 4341 : clib_mem_page_bytes (clib_mem_page_sz_t log2_page_size)
479 : {
480 4341 : return 1ULL << clib_mem_log2_page_size_validate (log2_page_size);
481 : }
482 :
483 : static_always_inline clib_error_t *
484 0 : clib_mem_get_last_error (void)
485 : {
486 0 : return clib_mem_main.error;
487 : }
488 :
489 : /* bulk allocator */
490 :
491 : typedef void *clib_mem_bulk_handle_t;
492 : clib_mem_bulk_handle_t clib_mem_bulk_init (u32 elt_sz, u32 align,
493 : u32 min_elts_per_chunk);
494 : void clib_mem_bulk_destroy (clib_mem_bulk_handle_t h);
495 : void *clib_mem_bulk_alloc (clib_mem_bulk_handle_t h);
496 : void clib_mem_bulk_free (clib_mem_bulk_handle_t h, void *p);
497 : u8 *format_clib_mem_bulk (u8 *s, va_list *args);
498 :
499 : #include <vppinfra/error.h> /* clib_panic */
500 :
501 : #endif /* _included_clib_mem_h */
502 :
503 : /*
504 : * fd.io coding-style-patch-verification: ON
505 : *
506 : * Local Variables:
507 : * eval: (c-set-style "gnu")
508 : * End:
509 : */
|
