Back to index

glibc  2.9
dl-tls.c
Go to the documentation of this file.
00001 /* Thread-local storage handling in the ELF dynamic linker.  Generic version.
00002    Copyright (C) 2002,2003,2004,2005,2006,2008 Free Software Foundation, Inc.
00003    This file is part of the GNU C Library.
00004 
00005    The GNU C Library is free software; you can redistribute it and/or
00006    modify it under the terms of the GNU Lesser General Public
00007    License as published by the Free Software Foundation; either
00008    version 2.1 of the License, or (at your option) any later version.
00009 
00010    The GNU C Library is distributed in the hope that it will be useful,
00011    but WITHOUT ANY WARRANTY; without even the implied warranty of
00012    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00013    Lesser General Public License for more details.
00014 
00015    You should have received a copy of the GNU Lesser General Public
00016    License along with the GNU C Library; if not, write to the Free
00017    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
00018    02111-1307 USA.  */
00019 
00020 #include <assert.h>
00021 #include <errno.h>
00022 #include <libintl.h>
00023 #include <signal.h>
00024 #include <stdlib.h>
00025 #include <unistd.h>
00026 #include <sys/param.h>
00027 
00028 #include <tls.h>
00029 #include <dl-tls.h>
00030 #include <ldsodefs.h>
00031 
00032 /* Amount of excess space to allocate in the static TLS area
00033    to allow dynamic loading of modules defining IE-model TLS data.  */
00034 #define TLS_STATIC_SURPLUS  64 + DL_NNS * 100
00035 
00036 /* Value used for dtv entries for which the allocation is delayed.  */
00037 #define TLS_DTV_UNALLOCATED ((void *) -1l)
00038 
00039 
00040 /* Out-of-memory handler.  */
00041 #ifdef SHARED
00042 static void
00043 __attribute__ ((__noreturn__))
00044 oom (void)
00045 {
00046   _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
00047 }
00048 #endif
00049 
00050 
00051 size_t
00052 internal_function
00053 _dl_next_tls_modid (void)
00054 {
00055   size_t result;
00056 
00057   if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
00058     {
00059       size_t disp = 0;
00060       struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
00061 
00062       /* Note that this branch will never be executed during program
00063         start since there are no gaps at that time.  Therefore it
00064         does not matter that the dl_tls_dtv_slotinfo is not allocated
00065         yet when the function is called for the first times.
00066 
00067         NB: the offset +1 is due to the fact that DTV[0] is used
00068         for something else.  */
00069       result = GL(dl_tls_static_nelem) + 1;
00070       if (result <= GL(dl_tls_max_dtv_idx))
00071        do
00072          {
00073            while (result - disp < runp->len)
00074              {
00075               if (runp->slotinfo[result - disp].map == NULL)
00076                 break;
00077 
00078               ++result;
00079               assert (result <= GL(dl_tls_max_dtv_idx) + 1);
00080              }
00081 
00082            if (result - disp < runp->len)
00083              break;
00084 
00085            disp += runp->len;
00086          }
00087        while ((runp = runp->next) != NULL);
00088 
00089       if (result > GL(dl_tls_max_dtv_idx))
00090        {
00091          /* The new index must indeed be exactly one higher than the
00092             previous high.  */
00093          assert (result == GL(dl_tls_max_dtv_idx) + 1);
00094          /* There is no gap anymore.  */
00095          GL(dl_tls_dtv_gaps) = false;
00096 
00097          goto nogaps;
00098        }
00099     }
00100   else
00101     {
00102       /* No gaps, allocate a new entry.  */
00103     nogaps:
00104 
00105       result = ++GL(dl_tls_max_dtv_idx);
00106     }
00107 
00108   return result;
00109 }
00110 
00111 
00112 #ifdef SHARED
00113 void
00114 internal_function
00115 _dl_determine_tlsoffset (void)
00116 {
00117   size_t max_align = TLS_TCB_ALIGN;
00118   size_t freetop = 0;
00119   size_t freebottom = 0;
00120 
00121   /* The first element of the dtv slot info list is allocated.  */
00122   assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
00123   /* There is at this point only one element in the
00124      dl_tls_dtv_slotinfo_list list.  */
00125   assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
00126 
00127   struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
00128 
00129   /* Determining the offset of the various parts of the static TLS
00130      block has several dependencies.  In addition we have to work
00131      around bugs in some toolchains.
00132 
00133      Each TLS block from the objects available at link time has a size
00134      and an alignment requirement.  The GNU ld computes the alignment
00135      requirements for the data at the positions *in the file*, though.
00136      I.e, it is not simply possible to allocate a block with the size
00137      of the TLS program header entry.  The data is layed out assuming
00138      that the first byte of the TLS block fulfills
00139 
00140        p_vaddr mod p_align == &TLS_BLOCK mod p_align
00141 
00142      This means we have to add artificial padding at the beginning of
00143      the TLS block.  These bytes are never used for the TLS data in
00144      this module but the first byte allocated must be aligned
00145      according to mod p_align == 0 so that the first byte of the TLS
00146      block is aligned according to p_vaddr mod p_align.  This is ugly
00147      and the linker can help by computing the offsets in the TLS block
00148      assuming the first byte of the TLS block is aligned according to
00149      p_align.
00150 
00151      The extra space which might be allocated before the first byte of
00152      the TLS block need not go unused.  The code below tries to use
00153      that memory for the next TLS block.  This can work if the total
00154      memory requirement for the next TLS block is smaller than the
00155      gap.  */
00156 
00157 #if TLS_TCB_AT_TP
00158   /* We simply start with zero.  */
00159   size_t offset = 0;
00160 
00161   for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
00162     {
00163       assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
00164 
00165       size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
00166                        & (slotinfo[cnt].map->l_tls_align - 1));
00167       size_t off;
00168       max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
00169 
00170       if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
00171        {
00172          off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
00173                       - firstbyte, slotinfo[cnt].map->l_tls_align)
00174               + firstbyte;
00175          if (off <= freebottom)
00176            {
00177              freetop = off;
00178 
00179              /* XXX For some architectures we perhaps should store the
00180                negative offset.  */
00181              slotinfo[cnt].map->l_tls_offset = off;
00182              continue;
00183            }
00184        }
00185 
00186       off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
00187                    slotinfo[cnt].map->l_tls_align) + firstbyte;
00188       if (off > offset + slotinfo[cnt].map->l_tls_blocksize
00189               + (freebottom - freetop))
00190        {
00191          freetop = offset;
00192          freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
00193        }
00194       offset = off;
00195 
00196       /* XXX For some architectures we perhaps should store the
00197         negative offset.  */
00198       slotinfo[cnt].map->l_tls_offset = off;
00199     }
00200 
00201   GL(dl_tls_static_used) = offset;
00202   GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align)
00203                          + TLS_TCB_SIZE);
00204 #elif TLS_DTV_AT_TP
00205   /* The TLS blocks start right after the TCB.  */
00206   size_t offset = TLS_TCB_SIZE;
00207 
00208   for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
00209     {
00210       assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
00211 
00212       size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
00213                        & (slotinfo[cnt].map->l_tls_align - 1));
00214       size_t off;
00215       max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
00216 
00217       if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
00218        {
00219          off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
00220          if (off - freebottom < firstbyte)
00221            off += slotinfo[cnt].map->l_tls_align;
00222          if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
00223            {
00224              slotinfo[cnt].map->l_tls_offset = off - firstbyte;
00225              freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
00226                          - firstbyte);
00227              continue;
00228            }
00229        }
00230 
00231       off = roundup (offset, slotinfo[cnt].map->l_tls_align);
00232       if (off - offset < firstbyte)
00233        off += slotinfo[cnt].map->l_tls_align;
00234 
00235       slotinfo[cnt].map->l_tls_offset = off - firstbyte;
00236       if (off - firstbyte - offset > freetop - freebottom)
00237        {
00238          freebottom = offset;
00239          freetop = off - firstbyte;
00240        }
00241 
00242       offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
00243     }
00244 
00245   GL(dl_tls_static_used) = offset;
00246   GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS,
00247                                 TLS_TCB_ALIGN);
00248 #else
00249 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
00250 #endif
00251 
00252   /* The alignment requirement for the static TLS block.  */
00253   GL(dl_tls_static_align) = max_align;
00254 }
00255 
00256 
00257 /* This is called only when the data structure setup was skipped at startup,
00258    when there was no need for it then.  Now we have dynamically loaded
00259    something needing TLS, or libpthread needs it.  */
00260 int
00261 internal_function
00262 _dl_tls_setup (void)
00263 {
00264   assert (GL(dl_tls_dtv_slotinfo_list) == NULL);
00265   assert (GL(dl_tls_max_dtv_idx) == 0);
00266 
00267   const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS;
00268 
00269   GL(dl_tls_dtv_slotinfo_list)
00270     = calloc (1, (sizeof (struct dtv_slotinfo_list)
00271                 + nelem * sizeof (struct dtv_slotinfo)));
00272   if (GL(dl_tls_dtv_slotinfo_list) == NULL)
00273     return -1;
00274 
00275   GL(dl_tls_dtv_slotinfo_list)->len = nelem;
00276 
00277   /* Number of elements in the static TLS block.  It can't be zero
00278      because of various assumptions.  The one element is null.  */
00279   GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1;
00280 
00281   /* This initializes more variables for us.  */
00282   _dl_determine_tlsoffset ();
00283 
00284   return 0;
00285 }
00286 rtld_hidden_def (_dl_tls_setup)
00287 #endif
00288 
00289 static void *
00290 internal_function
00291 allocate_dtv (void *result)
00292 {
00293   dtv_t *dtv;
00294   size_t dtv_length;
00295 
00296   /* We allocate a few more elements in the dtv than are needed for the
00297      initial set of modules.  This should avoid in most cases expansions
00298      of the dtv.  */
00299   dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
00300   dtv = calloc (dtv_length + 2, sizeof (dtv_t));
00301   if (dtv != NULL)
00302     {
00303       /* This is the initial length of the dtv.  */
00304       dtv[0].counter = dtv_length;
00305 
00306       /* The rest of the dtv (including the generation counter) is
00307         Initialize with zero to indicate nothing there.  */
00308 
00309       /* Add the dtv to the thread data structures.  */
00310       INSTALL_DTV (result, dtv);
00311     }
00312   else
00313     result = NULL;
00314 
00315   return result;
00316 }
00317 
00318 
00319 /* Get size and alignment requirements of the static TLS block.  */
00320 void
00321 internal_function
00322 _dl_get_tls_static_info (size_t *sizep, size_t *alignp)
00323 {
00324   *sizep = GL(dl_tls_static_size);
00325   *alignp = GL(dl_tls_static_align);
00326 }
00327 
00328 
00329 void *
00330 internal_function
00331 _dl_allocate_tls_storage (void)
00332 {
00333   void *result;
00334   size_t size = GL(dl_tls_static_size);
00335 
00336 #if TLS_DTV_AT_TP
00337   /* Memory layout is:
00338      [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
00339                        ^ This should be returned.  */
00340   size += (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1)
00341          & ~(GL(dl_tls_static_align) - 1);
00342 #endif
00343 
00344   /* Allocate a correctly aligned chunk of memory.  */
00345   result = __libc_memalign (GL(dl_tls_static_align), size);
00346   if (__builtin_expect (result != NULL, 1))
00347     {
00348       /* Allocate the DTV.  */
00349       void *allocated = result;
00350 
00351 #if TLS_TCB_AT_TP
00352       /* The TCB follows the TLS blocks.  */
00353       result = (char *) result + size - TLS_TCB_SIZE;
00354 
00355       /* Clear the TCB data structure.  We can't ask the caller (i.e.
00356         libpthread) to do it, because we will initialize the DTV et al.  */
00357       memset (result, '\0', TLS_TCB_SIZE);
00358 #elif TLS_DTV_AT_TP
00359       result = (char *) result + size - GL(dl_tls_static_size);
00360 
00361       /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before it.
00362         We can't ask the caller (i.e. libpthread) to do it, because we will
00363         initialize the DTV et al.  */
00364       memset ((char *) result - TLS_PRE_TCB_SIZE, '\0',
00365              TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
00366 #endif
00367 
00368       result = allocate_dtv (result);
00369       if (result == NULL)
00370        free (allocated);
00371     }
00372 
00373   return result;
00374 }
00375 
00376 
00377 void *
00378 internal_function
00379 _dl_allocate_tls_init (void *result)
00380 {
00381   if (result == NULL)
00382     /* The memory allocation failed.  */
00383     return NULL;
00384 
00385   dtv_t *dtv = GET_DTV (result);
00386   struct dtv_slotinfo_list *listp;
00387   size_t total = 0;
00388   size_t maxgen = 0;
00389 
00390   /* We have to prepare the dtv for all currently loaded modules using
00391      TLS.  For those which are dynamically loaded we add the values
00392      indicating deferred allocation.  */
00393   listp = GL(dl_tls_dtv_slotinfo_list);
00394   while (1)
00395     {
00396       size_t cnt;
00397 
00398       for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
00399        {
00400          struct link_map *map;
00401          void *dest;
00402 
00403          /* Check for the total number of used slots.  */
00404          if (total + cnt > GL(dl_tls_max_dtv_idx))
00405            break;
00406 
00407          map = listp->slotinfo[cnt].map;
00408          if (map == NULL)
00409            /* Unused entry.  */
00410            continue;
00411 
00412          /* Keep track of the maximum generation number.  This might
00413             not be the generation counter.  */
00414          maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
00415 
00416          if (map->l_tls_offset == NO_TLS_OFFSET
00417              || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
00418            {
00419              /* For dynamically loaded modules we simply store
00420                the value indicating deferred allocation.  */
00421              dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
00422              dtv[map->l_tls_modid].pointer.is_static = false;
00423              continue;
00424            }
00425 
00426          assert (map->l_tls_modid == cnt);
00427          assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
00428 #if TLS_TCB_AT_TP
00429          assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
00430          dest = (char *) result - map->l_tls_offset;
00431 #elif TLS_DTV_AT_TP
00432          dest = (char *) result + map->l_tls_offset;
00433 #else
00434 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
00435 #endif
00436 
00437          /* Copy the initialization image and clear the BSS part.  */
00438          dtv[map->l_tls_modid].pointer.val = dest;
00439          dtv[map->l_tls_modid].pointer.is_static = true;
00440          memset (__mempcpy (dest, map->l_tls_initimage,
00441                           map->l_tls_initimage_size), '\0',
00442                 map->l_tls_blocksize - map->l_tls_initimage_size);
00443        }
00444 
00445       total += cnt;
00446       if (total >= GL(dl_tls_max_dtv_idx))
00447        break;
00448 
00449       listp = listp->next;
00450       assert (listp != NULL);
00451     }
00452 
00453   /* The DTV version is up-to-date now.  */
00454   dtv[0].counter = maxgen;
00455 
00456   return result;
00457 }
00458 rtld_hidden_def (_dl_allocate_tls_init)
00459 
00460 void *
00461 internal_function
00462 _dl_allocate_tls (void *mem)
00463 {
00464   return _dl_allocate_tls_init (mem == NULL
00465                             ? _dl_allocate_tls_storage ()
00466                             : allocate_dtv (mem));
00467 }
00468 rtld_hidden_def (_dl_allocate_tls)
00469 
00470 
00471 void
00472 internal_function
00473 _dl_deallocate_tls (void *tcb, bool dealloc_tcb)
00474 {
00475   dtv_t *dtv = GET_DTV (tcb);
00476 
00477   /* We need to free the memory allocated for non-static TLS.  */
00478   for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
00479     if (! dtv[1 + cnt].pointer.is_static
00480        && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED)
00481       free (dtv[1 + cnt].pointer.val);
00482 
00483   /* The array starts with dtv[-1].  */
00484 #ifdef SHARED
00485   if (dtv != GL(dl_initial_dtv))
00486 #endif
00487     free (dtv - 1);
00488 
00489   if (dealloc_tcb)
00490     {
00491 #if TLS_TCB_AT_TP
00492       /* The TCB follows the TLS blocks.  Back up to free the whole block.  */
00493       tcb -= GL(dl_tls_static_size) - TLS_TCB_SIZE;
00494 #elif TLS_DTV_AT_TP
00495       /* Back up the TLS_PRE_TCB_SIZE bytes.  */
00496       tcb -= (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1)
00497             & ~(GL(dl_tls_static_align) - 1);
00498 #endif
00499       free (tcb);
00500     }
00501 }
00502 rtld_hidden_def (_dl_deallocate_tls)
00503 
00504 
00505 #ifdef SHARED
00506 /* The __tls_get_addr function has two basic forms which differ in the
00507    arguments.  The IA-64 form takes two parameters, the module ID and
00508    offset.  The form used, among others, on IA-32 takes a reference to
00509    a special structure which contain the same information.  The second
00510    form seems to be more often used (in the moment) so we default to
00511    it.  Users of the IA-64 form have to provide adequate definitions
00512    of the following macros.  */
00513 # ifndef GET_ADDR_ARGS
00514 #  define GET_ADDR_ARGS tls_index *ti
00515 # endif
00516 # ifndef GET_ADDR_MODULE
00517 #  define GET_ADDR_MODULE ti->ti_module
00518 # endif
00519 # ifndef GET_ADDR_OFFSET
00520 #  define GET_ADDR_OFFSET ti->ti_offset
00521 # endif
00522 
00523 
00524 static void *
00525 allocate_and_init (struct link_map *map)
00526 {
00527   void *newp;
00528 
00529   newp = __libc_memalign (map->l_tls_align, map->l_tls_blocksize);
00530   if (newp == NULL)
00531     oom ();
00532 
00533   /* Initialize the memory.  */
00534   memset (__mempcpy (newp, map->l_tls_initimage, map->l_tls_initimage_size),
00535          '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
00536 
00537   return newp;
00538 }
00539 
00540 
00541 struct link_map *
00542 _dl_update_slotinfo (unsigned long int req_modid)
00543 {
00544   struct link_map *the_map = NULL;
00545   dtv_t *dtv = THREAD_DTV ();
00546 
00547   /* The global dl_tls_dtv_slotinfo array contains for each module
00548      index the generation counter current when the entry was created.
00549      This array never shrinks so that all module indices which were
00550      valid at some time can be used to access it.  Before the first
00551      use of a new module index in this function the array was extended
00552      appropriately.  Access also does not have to be guarded against
00553      modifications of the array.  It is assumed that pointer-size
00554      values can be read atomically even in SMP environments.  It is
00555      possible that other threads at the same time dynamically load
00556      code and therefore add to the slotinfo list.  This is a problem
00557      since we must not pick up any information about incomplete work.
00558      The solution to this is to ignore all dtv slots which were
00559      created after the one we are currently interested.  We know that
00560      dynamic loading for this module is completed and this is the last
00561      load operation we know finished.  */
00562   unsigned long int idx = req_modid;
00563   struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
00564 
00565   while (idx >= listp->len)
00566     {
00567       idx -= listp->len;
00568       listp = listp->next;
00569     }
00570 
00571   if (dtv[0].counter < listp->slotinfo[idx].gen)
00572     {
00573       /* The generation counter for the slot is higher than what the
00574         current dtv implements.  We have to update the whole dtv but
00575         only those entries with a generation counter <= the one for
00576         the entry we need.  */
00577       size_t new_gen = listp->slotinfo[idx].gen;
00578       size_t total = 0;
00579 
00580       /* We have to look through the entire dtv slotinfo list.  */
00581       listp =  GL(dl_tls_dtv_slotinfo_list);
00582       do
00583        {
00584          for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
00585            {
00586              size_t gen = listp->slotinfo[cnt].gen;
00587 
00588              if (gen > new_gen)
00589               /* This is a slot for a generation younger than the
00590                  one we are handling now.  It might be incompletely
00591                  set up so ignore it.  */
00592               continue;
00593 
00594              /* If the entry is older than the current dtv layout we
00595                know we don't have to handle it.  */
00596              if (gen <= dtv[0].counter)
00597               continue;
00598 
00599              /* If there is no map this means the entry is empty.  */
00600              struct link_map *map = listp->slotinfo[cnt].map;
00601              if (map == NULL)
00602               {
00603                 /* If this modid was used at some point the memory
00604                    might still be allocated.  */
00605                 if (! dtv[total + cnt].pointer.is_static
00606                     && dtv[total + cnt].pointer.val != TLS_DTV_UNALLOCATED)
00607                   {
00608                     free (dtv[total + cnt].pointer.val);
00609                     dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
00610                   }
00611 
00612                 continue;
00613               }
00614 
00615              /* Check whether the current dtv array is large enough.  */
00616              size_t modid = map->l_tls_modid;
00617              assert (total + cnt == modid);
00618              if (dtv[-1].counter < modid)
00619               {
00620                 /* Reallocate the dtv.  */
00621                 dtv_t *newp;
00622                 size_t newsize = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
00623                 size_t oldsize = dtv[-1].counter;
00624 
00625                 assert (map->l_tls_modid <= newsize);
00626 
00627                 if (dtv == GL(dl_initial_dtv))
00628                   {
00629                     /* This is the initial dtv that was allocated
00630                       during rtld startup using the dl-minimal.c
00631                       malloc instead of the real malloc.  We can't
00632                       free it, we have to abandon the old storage.  */
00633 
00634                     newp = malloc ((2 + newsize) * sizeof (dtv_t));
00635                     if (newp == NULL)
00636                      oom ();
00637                     memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
00638                   }
00639                 else
00640                   {
00641                     newp = realloc (&dtv[-1],
00642                                   (2 + newsize) * sizeof (dtv_t));
00643                     if (newp == NULL)
00644                      oom ();
00645                   }
00646 
00647                 newp[0].counter = newsize;
00648 
00649                 /* Clear the newly allocated part.  */
00650                 memset (newp + 2 + oldsize, '\0',
00651                        (newsize - oldsize) * sizeof (dtv_t));
00652 
00653                 /* Point dtv to the generation counter.  */
00654                 dtv = &newp[1];
00655 
00656                 /* Install this new dtv in the thread data
00657                    structures.  */
00658                 INSTALL_NEW_DTV (dtv);
00659               }
00660 
00661              /* If there is currently memory allocate for this
00662                dtv entry free it.  */
00663              /* XXX Ideally we will at some point create a memory
00664                pool.  */
00665              if (! dtv[modid].pointer.is_static
00666                 && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED)
00667               /* Note that free is called for NULL is well.  We
00668                  deallocate even if it is this dtv entry we are
00669                  supposed to load.  The reason is that we call
00670                  memalign and not malloc.  */
00671               free (dtv[modid].pointer.val);
00672 
00673              /* This module is loaded dynamically- We defer memory
00674                allocation.  */
00675              dtv[modid].pointer.is_static = false;
00676              dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
00677 
00678              if (modid == req_modid)
00679               the_map = map;
00680            }
00681 
00682          total += listp->len;
00683        }
00684       while ((listp = listp->next) != NULL);
00685 
00686       /* This will be the new maximum generation counter.  */
00687       dtv[0].counter = new_gen;
00688     }
00689 
00690   return the_map;
00691 }
00692 
00693 
00694 static void *
00695 __attribute_noinline__
00696 tls_get_addr_tail (dtv_t *dtv, struct link_map *the_map, size_t module)
00697 {
00698   /* The allocation was deferred.  Do it now.  */
00699   if (the_map == NULL)
00700     {
00701       /* Find the link map for this module.  */
00702       size_t idx = module;
00703       struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
00704 
00705       while (idx >= listp->len)
00706        {
00707          idx -= listp->len;
00708          listp = listp->next;
00709        }
00710 
00711       the_map = listp->slotinfo[idx].map;
00712     }
00713 
00714  again:
00715   /* Make sure that, if a dlopen running in parallel forces the
00716      variable into static storage, we'll wait until the address in the
00717      static TLS block is set up, and use that.  If we're undecided
00718      yet, make sure we make the decision holding the lock as well.  */
00719   if (__builtin_expect (the_map->l_tls_offset
00720                      != FORCED_DYNAMIC_TLS_OFFSET, 0))
00721     {
00722       __rtld_lock_lock_recursive (GL(dl_load_lock));
00723       if (__builtin_expect (the_map->l_tls_offset == NO_TLS_OFFSET, 1))
00724        {
00725          the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
00726          __rtld_lock_unlock_recursive (GL(dl_load_lock));
00727        }
00728       else
00729        {
00730          __rtld_lock_unlock_recursive (GL(dl_load_lock));
00731          if (__builtin_expect (the_map->l_tls_offset
00732                             != FORCED_DYNAMIC_TLS_OFFSET, 1))
00733            {
00734              void *p = dtv[module].pointer.val;
00735              if (__builtin_expect (p == TLS_DTV_UNALLOCATED, 0))
00736               goto again;
00737 
00738              return p;
00739            }
00740        }
00741     }
00742   void *p = dtv[module].pointer.val = allocate_and_init (the_map);
00743   dtv[module].pointer.is_static = false;
00744 
00745   return p;
00746 }
00747 
00748 
00749 /* The generic dynamic and local dynamic model cannot be used in
00750    statically linked applications.  */
00751 void *
00752 __tls_get_addr (GET_ADDR_ARGS)
00753 {
00754   dtv_t *dtv = THREAD_DTV ();
00755   struct link_map *the_map = NULL;
00756   void *p;
00757 
00758   if (__builtin_expect (dtv[0].counter != GL(dl_tls_generation), 0))
00759     {
00760       the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
00761       dtv = THREAD_DTV ();
00762     }
00763 
00764   p = dtv[GET_ADDR_MODULE].pointer.val;
00765 
00766   if (__builtin_expect (p == TLS_DTV_UNALLOCATED, 0))
00767     p = tls_get_addr_tail (dtv, the_map, GET_ADDR_MODULE);
00768 
00769   return (char *) p + GET_ADDR_OFFSET;
00770 }
00771 #endif
00772 
00773 
00774 /* Look up the module's TLS block as for __tls_get_addr,
00775    but never touch anything.  Return null if it's not allocated yet.  */
00776 void *
00777 _dl_tls_get_addr_soft (struct link_map *l)
00778 {
00779   if (__builtin_expect (l->l_tls_modid == 0, 0))
00780     /* This module has no TLS segment.  */
00781     return NULL;
00782 
00783   dtv_t *dtv = THREAD_DTV ();
00784   if (__builtin_expect (dtv[0].counter != GL(dl_tls_generation), 0))
00785     {
00786       /* This thread's DTV is not completely current,
00787         but it might already cover this module.  */
00788 
00789       if (l->l_tls_modid >= dtv[-1].counter)
00790        /* Nope.  */
00791        return NULL;
00792 
00793       size_t idx = l->l_tls_modid;
00794       struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
00795       while (idx >= listp->len)
00796        {
00797          idx -= listp->len;
00798          listp = listp->next;
00799        }
00800 
00801       /* We've reached the slot for this module.
00802         If its generation counter is higher than the DTV's,
00803         this thread does not know about this module yet.  */
00804       if (dtv[0].counter < listp->slotinfo[idx].gen)
00805        return NULL;
00806     }
00807 
00808   void *data = dtv[l->l_tls_modid].pointer.val;
00809   if (__builtin_expect (data == TLS_DTV_UNALLOCATED, 0))
00810     /* The DTV is current, but this thread has not yet needed
00811        to allocate this module's segment.  */
00812     data = NULL;
00813 
00814   return data;
00815 }
00816 
00817 
00818 void
00819 _dl_add_to_slotinfo (struct link_map *l)
00820 {
00821   /* Now that we know the object is loaded successfully add
00822      modules containing TLS data to the dtv info table.  We
00823      might have to increase its size.  */
00824   struct dtv_slotinfo_list *listp;
00825   struct dtv_slotinfo_list *prevp;
00826   size_t idx = l->l_tls_modid;
00827 
00828   /* Find the place in the dtv slotinfo list.  */
00829   listp = GL(dl_tls_dtv_slotinfo_list);
00830   prevp = NULL;             /* Needed to shut up gcc.  */
00831   do
00832     {
00833       /* Does it fit in the array of this list element?  */
00834       if (idx < listp->len)
00835        break;
00836       idx -= listp->len;
00837       prevp = listp;
00838       listp = listp->next;
00839     }
00840   while (listp != NULL);
00841 
00842   if (listp == NULL)
00843     {
00844       /* When we come here it means we have to add a new element
00845         to the slotinfo list.  And the new module must be in
00846         the first slot.  */
00847       assert (idx == 0);
00848 
00849       listp = prevp->next = (struct dtv_slotinfo_list *)
00850        malloc (sizeof (struct dtv_slotinfo_list)
00851               + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
00852       if (listp == NULL)
00853        {
00854          /* We ran out of memory.  We will simply fail this
00855             call but don't undo anything we did so far.  The
00856             application will crash or be terminated anyway very
00857             soon.  */
00858 
00859          /* We have to do this since some entries in the dtv
00860             slotinfo array might already point to this
00861             generation.  */
00862          ++GL(dl_tls_generation);
00863 
00864          _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
00865 cannot create TLS data structures"));
00866        }
00867 
00868       listp->len = TLS_SLOTINFO_SURPLUS;
00869       listp->next = NULL;
00870       memset (listp->slotinfo, '\0',
00871              TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
00872     }
00873 
00874   /* Add the information into the slotinfo data structure.  */
00875   listp->slotinfo[idx].map = l;
00876   listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
00877 }