Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2015-2019 Cisco and/or its affiliates.
       3             :  * Licensed under the Apache License, Version 2.0 (the "License");
       4             :  * you may not use this file except in compliance with the License.
       5             :  * You may obtain a copy of the License at:
       6             :  *
       7             :  *     http://www.apache.org/licenses/LICENSE-2.0
       8             :  *
       9             :  * Unless required by applicable law or agreed to in writing, software
      10             :  * distributed under the License is distributed on an "AS IS" BASIS,
      11             :  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             :  * See the License for the specific language governing permissions and
      13             :  * limitations under the License.
      14             :  */
      15             : #include <svm/ssvm.h>
      16             : #include <svm/svm_common.h>
      17             : 
      18             : typedef int (*init_fn) (ssvm_private_t *);
      19             : typedef void (*delete_fn) (ssvm_private_t *);
      20             : 
      21             : static init_fn server_init_fns[SSVM_N_SEGMENT_TYPES] =
      22             :   { ssvm_server_init_shm, ssvm_server_init_memfd, ssvm_server_init_private };
      23             : static init_fn client_init_fns[SSVM_N_SEGMENT_TYPES] =
      24             :   { ssvm_client_init_shm, ssvm_client_init_memfd, ssvm_client_init_private };
      25             : static delete_fn delete_fns[SSVM_N_SEGMENT_TYPES] =
      26             :   { ssvm_delete_shm, ssvm_delete_memfd, ssvm_delete_private };
      27             : 
      28             : int
      29          15 : ssvm_server_init_shm (ssvm_private_t * ssvm)
      30             : {
      31             :   int ssvm_fd;
      32          15 :   u8 junk = 0, *ssvm_filename;
      33             :   ssvm_shared_header_t *sh;
      34          15 :   uword page_size, requested_va = 0;
      35             :   void *oldheap;
      36             : 
      37          15 :   if (ssvm->ssvm_size == 0)
      38           0 :     return SSVM_API_ERROR_NO_SIZE;
      39             : 
      40             :   if (CLIB_DEBUG > 1)
      41             :     clib_warning ("[%d] creating segment '%s'", getpid (), ssvm->name);
      42             : 
      43          15 :   ASSERT (vec_c_string_is_terminated (ssvm->name));
      44          15 :   ssvm_filename = format (0, "/dev/shm/%s%c", ssvm->name, 0);
      45          15 :   unlink ((char *) ssvm_filename);
      46          15 :   vec_free (ssvm_filename);
      47             : 
      48          15 :   ssvm_fd = shm_open ((char *) ssvm->name, O_RDWR | O_CREAT | O_EXCL, 0777);
      49          15 :   if (ssvm_fd < 0)
      50             :     {
      51           0 :       clib_unix_warning ("create segment '%s'", ssvm->name);
      52           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
      53             :     }
      54             : 
      55          15 :   if (fchmod (ssvm_fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP) < 0)
      56           0 :     clib_unix_warning ("ssvm segment chmod");
      57          15 :   if (svm_get_root_rp ())
      58             :     {
      59             :       /* TODO: is this really needed? */
      60          15 :       svm_main_region_t *smr = svm_get_root_rp ()->data_base;
      61          15 :       if (fchown (ssvm_fd, smr->uid, smr->gid) < 0)
      62           0 :         clib_unix_warning ("ssvm segment chown");
      63             :     }
      64             : 
      65          15 :   if (lseek (ssvm_fd, ssvm->ssvm_size, SEEK_SET) < 0)
      66             :     {
      67           0 :       clib_unix_warning ("lseek");
      68           0 :       close (ssvm_fd);
      69           0 :       return SSVM_API_ERROR_SET_SIZE;
      70             :     }
      71             : 
      72          15 :   if (write (ssvm_fd, &junk, 1) != 1)
      73             :     {
      74           0 :       clib_unix_warning ("set ssvm size");
      75           0 :       close (ssvm_fd);
      76           0 :       return SSVM_API_ERROR_SET_SIZE;
      77             :     }
      78             : 
      79          15 :   page_size = clib_mem_get_fd_page_size (ssvm_fd);
      80          15 :   if (ssvm->requested_va)
      81             :     {
      82           0 :       requested_va = ssvm->requested_va;
      83           0 :       clib_mem_vm_randomize_va (&requested_va, min_log2 (page_size));
      84             :     }
      85             : 
      86          15 :   sh = clib_mem_vm_map_shared (uword_to_pointer (requested_va, void *),
      87             :                                ssvm->ssvm_size, ssvm_fd, 0,
      88          15 :                                (char *) ssvm->name);
      89          15 :   if (sh == CLIB_MEM_VM_MAP_FAILED)
      90             :     {
      91           0 :       clib_unix_warning ("mmap");
      92           0 :       close (ssvm_fd);
      93           0 :       return SSVM_API_ERROR_MMAP;
      94             :     }
      95             : 
      96          15 :   close (ssvm_fd);
      97             : 
      98          15 :   clib_mem_unpoison (sh, sizeof (*sh));
      99          15 :   sh->server_pid = ssvm->my_pid;
     100          15 :   sh->ssvm_size = ssvm->ssvm_size;
     101          15 :   sh->ssvm_va = pointer_to_uword (sh);
     102          15 :   sh->type = SSVM_SEGMENT_SHM;
     103          30 :   sh->heap = clib_mem_create_heap (((u8 *) sh) + page_size,
     104          15 :                                    ssvm->ssvm_size - page_size,
     105             :                                    1 /* locked */ , "ssvm server shm");
     106             : 
     107          15 :   oldheap = ssvm_push_heap (sh);
     108          15 :   sh->name = format (0, "%s", ssvm->name, 0);
     109          15 :   ssvm_pop_heap (oldheap);
     110             : 
     111          15 :   ssvm->sh = sh;
     112          15 :   ssvm->my_pid = getpid ();
     113          15 :   ssvm->is_server = 1;
     114             : 
     115             :   /* The application has to set set sh->ready... */
     116          15 :   return 0;
     117             : }
     118             : 
     119             : int
     120           0 : ssvm_client_init_shm (ssvm_private_t * ssvm)
     121             : {
     122             :   struct stat stat;
     123           0 :   int ssvm_fd = -1;
     124             :   ssvm_shared_header_t *sh;
     125             : 
     126           0 :   ASSERT (vec_c_string_is_terminated (ssvm->name));
     127           0 :   ssvm->is_server = 0;
     128             : 
     129           0 :   while (ssvm->attach_timeout-- > 0)
     130             :     {
     131           0 :       if (ssvm_fd < 0)
     132           0 :         ssvm_fd = shm_open ((char *) ssvm->name, O_RDWR, 0777);
     133           0 :       if (ssvm_fd < 0)
     134             :         {
     135           0 :           sleep (1);
     136           0 :           continue;
     137             :         }
     138           0 :       if (fstat (ssvm_fd, &stat) < 0)
     139             :         {
     140           0 :           sleep (1);
     141           0 :           continue;
     142             :         }
     143             : 
     144           0 :       if (stat.st_size > 0)
     145           0 :         goto map_it;
     146             :     }
     147           0 :   clib_warning ("client timeout");
     148           0 :   return SSVM_API_ERROR_CLIENT_TIMEOUT;
     149             : 
     150           0 : map_it:
     151           0 :   sh = (void *) mmap (0, MMAP_PAGESIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
     152             :                       ssvm_fd, 0);
     153           0 :   if (sh == MAP_FAILED)
     154             :     {
     155           0 :       clib_unix_warning ("client research mmap");
     156           0 :       close (ssvm_fd);
     157           0 :       return SSVM_API_ERROR_MMAP;
     158             :     }
     159             : 
     160           0 :   while (ssvm->attach_timeout-- > 0)
     161             :     {
     162           0 :       if (sh->ready)
     163           0 :         goto re_map_it;
     164             :     }
     165           0 :   close (ssvm_fd);
     166           0 :   munmap (sh, MMAP_PAGESIZE);
     167           0 :   clib_warning ("client timeout 2");
     168           0 :   return SSVM_API_ERROR_CLIENT_TIMEOUT;
     169             : 
     170           0 : re_map_it:
     171           0 :   ssvm->requested_va = sh->ssvm_va;
     172           0 :   ssvm->ssvm_size = sh->ssvm_size;
     173           0 :   munmap (sh, MMAP_PAGESIZE);
     174             : 
     175           0 :   sh = ssvm->sh = (void *) mmap ((void *) ssvm->requested_va, ssvm->ssvm_size,
     176             :                                  PROT_READ | PROT_WRITE,
     177             :                                  MAP_SHARED | MAP_FIXED, ssvm_fd, 0);
     178             : 
     179           0 :   if (sh == MAP_FAILED)
     180             :     {
     181           0 :       clib_unix_warning ("client final mmap");
     182           0 :       close (ssvm_fd);
     183           0 :       return SSVM_API_ERROR_MMAP;
     184             :     }
     185           0 :   sh->client_pid = getpid ();
     186           0 :   close (ssvm_fd);
     187           0 :   return 0;
     188             : }
     189             : 
     190             : void
     191          15 : ssvm_delete_shm (ssvm_private_t * ssvm)
     192             : {
     193             :   u8 *fn;
     194             : 
     195          15 :   fn = format (0, "/dev/shm/%s%c", ssvm->name, 0);
     196             : 
     197             :   if (CLIB_DEBUG > 1)
     198             :     clib_warning ("[%d] unlinking ssvm (%s) backing file '%s'", getpid (),
     199             :                   ssvm->name, fn);
     200             : 
     201             :   /* Throw away the backing file */
     202          15 :   if (unlink ((char *) fn) < 0)
     203           0 :     clib_unix_warning ("unlink segment '%s'", ssvm->name);
     204             : 
     205          15 :   vec_free (fn);
     206          15 :   vec_free (ssvm->name);
     207             : 
     208          15 :   if (ssvm->is_server)
     209          15 :     clib_mem_vm_unmap (ssvm->sh);
     210             :   else
     211           0 :     munmap ((void *) ssvm->sh, ssvm->ssvm_size);
     212          15 : }
     213             : 
     214             : /**
     215             :  * Initialize memfd segment server
     216             :  */
     217             : int
     218         185 : ssvm_server_init_memfd (ssvm_private_t * memfd)
     219             : {
     220             :   uword page_size, n_pages;
     221             :   ssvm_shared_header_t *sh;
     222             :   int log2_page_size;
     223             :   void *oldheap;
     224             : 
     225         185 :   if (memfd->ssvm_size == 0)
     226           0 :     return SSVM_API_ERROR_NO_SIZE;
     227             : 
     228         185 :   ASSERT (vec_c_string_is_terminated (memfd->name));
     229             : 
     230         185 :   if (memfd->huge_page)
     231           0 :     memfd->fd = clib_mem_vm_create_fd (CLIB_MEM_PAGE_SZ_DEFAULT_HUGE,
     232           0 :                                        (char *) memfd->name);
     233             :   else
     234         185 :     memfd->fd =
     235         185 :       clib_mem_vm_create_fd (CLIB_MEM_PAGE_SZ_DEFAULT, (char *) memfd->name);
     236             : 
     237         185 :   if (memfd->fd == CLIB_MEM_ERROR)
     238             :     {
     239           0 :       clib_unix_warning ("failed to create memfd");
     240           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
     241             :     }
     242             : 
     243         185 :   log2_page_size = clib_mem_get_fd_log2_page_size (memfd->fd);
     244         185 :   if (log2_page_size == 0)
     245             :     {
     246           0 :       clib_unix_warning ("cannot determine page size");
     247           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
     248             :     }
     249             : 
     250         185 :   n_pages = ((memfd->ssvm_size - 1) >> log2_page_size) + 1;
     251             : 
     252         185 :   if ((ftruncate (memfd->fd, n_pages << log2_page_size)) == -1)
     253             :     {
     254           0 :       clib_unix_warning ("memfd ftruncate failure");
     255           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
     256             :     }
     257             : 
     258         185 :   sh = clib_mem_vm_map_shared (uword_to_pointer (memfd->requested_va, void *),
     259             :                                memfd->ssvm_size, memfd->fd, 0,
     260         185 :                                (char *) memfd->name);
     261         185 :   if (sh == CLIB_MEM_VM_MAP_FAILED)
     262             :     {
     263           0 :       clib_unix_warning ("memfd map (fd %d)", memfd->fd);
     264           0 :       close (memfd->fd);
     265           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
     266             :     }
     267             : 
     268         185 :   memfd->sh = sh;
     269         185 :   memfd->my_pid = getpid ();
     270         185 :   memfd->is_server = 1;
     271             : 
     272         185 :   sh->server_pid = memfd->my_pid;
     273         185 :   sh->ssvm_size = memfd->ssvm_size;
     274         185 :   sh->ssvm_va = pointer_to_uword (sh);
     275         185 :   sh->type = SSVM_SEGMENT_MEMFD;
     276             : 
     277         185 :   page_size = clib_mem_get_page_size ();
     278         370 :   sh->heap = clib_mem_create_heap (((u8 *) sh) + page_size,
     279         185 :                                    memfd->ssvm_size - page_size,
     280             :                                    1 /* locked */ , "ssvm server memfd");
     281         185 :   oldheap = ssvm_push_heap (sh);
     282         185 :   sh->name = format (0, "%s", memfd->name, 0);
     283         185 :   ssvm_pop_heap (oldheap);
     284             : 
     285             :   /* The application has to set set sh->ready... */
     286         185 :   return 0;
     287             : }
     288             : 
     289             : /**
     290             :  * Initialize memfd segment client
     291             :  *
     292             :  * Subtly different than svm_client_init. The caller needs to acquire
     293             :  * a usable file descriptor for the memfd segment e.g. via
     294             :  * vppinfra/socket.c:default_socket_recvmsg
     295             :  */
     296             : int
     297         150 : ssvm_client_init_memfd (ssvm_private_t * memfd)
     298             : {
     299         150 :   int mmap_flags = MAP_SHARED;
     300             :   ssvm_shared_header_t *sh;
     301             :   uword page_size;
     302             : 
     303         150 :   memfd->is_server = 0;
     304             : 
     305         150 :   page_size = clib_mem_get_fd_page_size (memfd->fd);
     306         150 :   if (!page_size)
     307             :     {
     308           0 :       clib_unix_warning ("page size unknown");
     309           0 :       return SSVM_API_ERROR_MMAP;
     310             :     }
     311             : 
     312             :   /*
     313             :    * Map the segment once, to look at the shared header
     314             :    */
     315         150 :   sh = (void *) mmap (0, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
     316             :                       memfd->fd, 0);
     317             : 
     318         150 :   if (sh == MAP_FAILED)
     319             :     {
     320           0 :       clib_unix_warning ("client research mmap (fd %d)", memfd->fd);
     321           0 :       close (memfd->fd);
     322           0 :       return SSVM_API_ERROR_MMAP;
     323             :     }
     324             : 
     325         150 :   memfd->requested_va = sh->ssvm_va;
     326         150 :   memfd->ssvm_size = sh->ssvm_size;
     327         150 :   munmap (sh, page_size);
     328             : 
     329         150 :   if (memfd->requested_va)
     330          29 :     mmap_flags |= MAP_FIXED;
     331             : 
     332             :   /*
     333             :    * Remap the segment at the 'right' address
     334             :    */
     335         150 :   sh = (void *) mmap (uword_to_pointer (memfd->requested_va, void *),
     336             :                       memfd->ssvm_size,
     337             :                       PROT_READ | PROT_WRITE, mmap_flags, memfd->fd, 0);
     338             : 
     339         150 :   if (sh == MAP_FAILED)
     340             :     {
     341           0 :       clib_unix_warning ("client final mmap");
     342           0 :       close (memfd->fd);
     343           0 :       return SSVM_API_ERROR_MMAP;
     344             :     }
     345             : 
     346         150 :   sh->client_pid = getpid ();
     347         150 :   memfd->sh = sh;
     348         150 :   return 0;
     349             : }
     350             : 
     351             : void
     352          95 : ssvm_delete_memfd (ssvm_private_t * memfd)
     353             : {
     354          95 :   vec_free (memfd->name);
     355          95 :   if (memfd->is_server)
     356          62 :     clib_mem_vm_unmap (memfd->sh);
     357             :   else
     358          33 :     munmap (memfd->sh, memfd->ssvm_size);
     359          95 :   close (memfd->fd);
     360          95 : }
     361             : 
     362             : /**
     363             :  * Initialize segment in a private heap
     364             :  */
     365             : int
     366         211 : ssvm_server_init_private (ssvm_private_t * ssvm)
     367             : {
     368         211 :   uword page_size, log2_page_size, rnd_size = 0;
     369             :   ssvm_shared_header_t *sh;
     370             :   clib_mem_heap_t *heap, *oldheap;
     371             : 
     372         211 :   log2_page_size = clib_mem_get_log2_page_size ();
     373         211 :   if (log2_page_size == 0)
     374             :     {
     375           0 :       clib_unix_warning ("cannot determine page size");
     376           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
     377             :     }
     378             : 
     379         211 :   page_size = 1ULL << log2_page_size;
     380         211 :   rnd_size = clib_max (ssvm->ssvm_size + (page_size - 1), ssvm->ssvm_size);
     381         211 :   rnd_size &= ~(page_size - 1);
     382             : 
     383         211 :   sh = clib_mem_vm_map (0, rnd_size + page_size, log2_page_size,
     384         211 :                         (char *) ssvm->name);
     385         211 :   if (sh == CLIB_MEM_VM_MAP_FAILED)
     386             :     {
     387           0 :       clib_unix_warning ("private map failed");
     388           0 :       return SSVM_API_ERROR_CREATE_FAILURE;
     389             :     }
     390             : 
     391         211 :   heap = clib_mem_create_heap ((u8 *) sh + page_size, rnd_size,
     392             :                                1 /* locked */ , "ssvm server private");
     393         211 :   if (heap == 0)
     394             :     {
     395           0 :       clib_unix_warning ("heap alloc");
     396           0 :       return -1;
     397             :     }
     398             : 
     399         211 :   rnd_size = clib_mem_get_heap_free_space (heap);
     400             : 
     401         211 :   ssvm->ssvm_size = rnd_size;
     402         211 :   ssvm->is_server = 1;
     403         211 :   ssvm->my_pid = getpid ();
     404         211 :   ssvm->requested_va = ~0;
     405             : 
     406             :   /* First page in allocated memory is set aside for the shared header */
     407         211 :   ssvm->sh = sh;
     408             : 
     409         211 :   clib_memset (sh, 0, sizeof (*sh));
     410         211 :   sh->heap = heap;
     411         211 :   sh->ssvm_size = rnd_size;
     412         211 :   sh->ssvm_va = pointer_to_uword (sh);
     413         211 :   sh->type = SSVM_SEGMENT_PRIVATE;
     414             : 
     415         211 :   oldheap = ssvm_push_heap (sh);
     416         211 :   sh->name = format (0, "%s", ssvm->name, 0);
     417         211 :   ssvm_pop_heap (oldheap);
     418             : 
     419         211 :   return 0;
     420             : }
     421             : 
     422             : int
     423           0 : ssvm_client_init_private (ssvm_private_t * ssvm)
     424             : {
     425           0 :   clib_warning ("BUG: this should not be called!");
     426           0 :   return -1;
     427             : }
     428             : 
     429             : void
     430          60 : ssvm_delete_private (ssvm_private_t * ssvm)
     431             : {
     432          60 :   vec_free (ssvm->name);
     433          60 :   clib_mem_destroy_heap (ssvm->sh->heap);
     434          60 :   clib_mem_vm_unmap (ssvm->sh);
     435          60 : }
     436             : 
     437             : int
     438         368 : ssvm_server_init (ssvm_private_t * ssvm, ssvm_segment_type_t type)
     439             : {
     440         368 :   return (server_init_fns[type]) (ssvm);
     441             : }
     442             : 
     443             : int
     444         121 : ssvm_client_init (ssvm_private_t * ssvm, ssvm_segment_type_t type)
     445             : {
     446         121 :   return (client_init_fns[type]) (ssvm);
     447             : }
     448             : 
     449             : void
     450         141 : ssvm_delete (ssvm_private_t * ssvm)
     451             : {
     452         141 :   delete_fns[ssvm->sh->type] (ssvm);
     453         141 : }
     454             : 
     455             : ssvm_segment_type_t
     456         210 : ssvm_type (const ssvm_private_t * ssvm)
     457             : {
     458         210 :   return ssvm->sh->type;
     459             : }
     460             : 
     461             : u8 *
     462           0 : ssvm_name (const ssvm_private_t * ssvm)
     463             : {
     464           0 :   return ssvm->sh->name;
     465             : }
     466             : 
     467             : /*
     468             :  * fd.io coding-style-patch-verification: ON
     469             :  *
     470             :  * Local Variables:
     471             :  * eval: (c-set-style "gnu")
     472             :  * End:
     473             :  */