Line data Source code
1 : /*
2 : * Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
3 : *
4 : * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
5 : * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
6 : *
7 : * Permission is hereby granted to use or copy this program
8 : * for any purpose, provided the above notices are retained on all copies.
9 : * Permission to modify the code and to distribute modified code is granted,
10 : * provided the above notices are retained, and a notice that the code was
11 : * modified is included with the above copyright notice.
12 : */
13 :
14 : #include "private/gc_priv.h"
15 :
16 : #if defined(THREAD_LOCAL_ALLOC)
17 :
18 : #ifndef THREADS
19 : # error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS"
20 : #endif
21 :
22 : #include "private/thread_local_alloc.h"
23 :
24 : #include <stdlib.h>
25 :
26 : #if defined(USE_COMPILER_TLS)
27 : __thread
28 : #elif defined(USE_WIN32_COMPILER_TLS)
29 : __declspec(thread)
30 : #endif
31 : GC_key_t GC_thread_key;
32 :
33 : static GC_bool keys_initialized;
34 :
35 : #ifdef ENABLE_DISCLAIM
36 : GC_INNER ptr_t * GC_finalized_objfreelist = NULL;
37 : /* This variable is declared here to prevent linking of */
38 : /* fnlz_mlc module unless the client uses the latter one. */
39 : #endif
40 :
41 : /* Return a single nonempty freelist fl to the global one pointed to */
42 : /* by gfl. */
43 :
44 0 : static void return_single_freelist(void *fl, void **gfl)
45 : {
46 : void *q, **qptr;
47 :
48 0 : if (*gfl == 0) {
49 0 : *gfl = fl;
50 : } else {
51 : GC_ASSERT(GC_size(fl) == GC_size(*gfl));
52 : /* Concatenate: */
53 0 : qptr = &(obj_link(fl));
54 0 : while ((word)(q = *qptr) >= HBLKSIZE)
55 0 : qptr = &(obj_link(q));
56 : GC_ASSERT(0 == q);
57 0 : *qptr = *gfl;
58 0 : *gfl = fl;
59 : }
60 0 : }
61 :
62 : /* Recover the contents of the freelist array fl into the global one gfl.*/
63 : /* We hold the allocator lock. */
64 604 : static void return_freelists(void **fl, void **gfl)
65 : {
66 : int i;
67 :
68 15100 : for (i = 1; i < TINY_FREELISTS; ++i) {
69 14496 : if ((word)(fl[i]) >= HBLKSIZE) {
70 0 : return_single_freelist(fl[i], gfl+i);
71 : }
72 : /* Clear fl[i], since the thread structure may hang around. */
73 : /* Do it in a way that is likely to trap if we access it. */
74 14496 : fl[i] = (ptr_t)HBLKSIZE;
75 : }
76 : /* The 0 granule freelist really contains 1 granule objects. */
77 : # ifdef GC_GCJ_SUPPORT
78 604 : if (fl[0] == ERROR_FL) return;
79 : # endif
80 453 : if ((word)(fl[0]) >= HBLKSIZE) {
81 0 : return_single_freelist(fl[0], gfl+1);
82 : }
83 : }
84 :
85 : /* Each thread structure must be initialized. */
86 : /* This call must be made from the new thread. */
87 662 : GC_INNER void GC_init_thread_local(GC_tlfs p)
88 : {
89 : int i;
90 :
91 : GC_ASSERT(I_HOLD_LOCK());
92 662 : if (!EXPECT(keys_initialized, TRUE)) {
93 : GC_ASSERT((word)&GC_thread_key % sizeof(word) == 0);
94 : if (0 != GC_key_create(&GC_thread_key, 0)) {
95 : ABORT("Failed to create key for local allocator");
96 : }
97 163 : keys_initialized = TRUE;
98 : }
99 662 : if (0 != GC_setspecific(GC_thread_key, p)) {
100 : ABORT("Failed to set thread specific allocation pointers");
101 : }
102 16550 : for (i = 1; i < TINY_FREELISTS; ++i) {
103 15888 : p -> ptrfree_freelists[i] = (void *)(word)1;
104 15888 : p -> normal_freelists[i] = (void *)(word)1;
105 : # ifdef GC_GCJ_SUPPORT
106 15888 : p -> gcj_freelists[i] = (void *)(word)1;
107 : # endif
108 : # ifdef ENABLE_DISCLAIM
109 15888 : p -> finalized_freelists[i] = (void *)(word)1;
110 : # endif
111 : }
112 : /* Set up the size 0 free lists. */
113 : /* We now handle most of them like regular free lists, to ensure */
114 : /* That explicit deallocation works. However, allocation of a */
115 : /* size 0 "gcj" object is always an error. */
116 662 : p -> ptrfree_freelists[0] = (void *)(word)1;
117 662 : p -> normal_freelists[0] = (void *)(word)1;
118 : # ifdef GC_GCJ_SUPPORT
119 662 : p -> gcj_freelists[0] = ERROR_FL;
120 : # endif
121 : # ifdef ENABLE_DISCLAIM
122 662 : p -> finalized_freelists[0] = (void *)(word)1;
123 : # endif
124 662 : }
125 :
126 : /* We hold the allocator lock. */
127 151 : GC_INNER void GC_destroy_thread_local(GC_tlfs p)
128 : {
129 : /* We currently only do this from the thread itself or from */
130 : /* the fork handler for a child process. */
131 151 : return_freelists(p -> ptrfree_freelists, GC_aobjfreelist);
132 151 : return_freelists(p -> normal_freelists, GC_objfreelist);
133 : # ifdef GC_GCJ_SUPPORT
134 151 : return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist);
135 : # endif
136 : # ifdef ENABLE_DISCLAIM
137 151 : return_freelists(p -> finalized_freelists,
138 : (void **)GC_finalized_objfreelist);
139 : # endif
140 151 : }
141 :
142 : #ifdef GC_ASSERTIONS
143 : /* Defined in pthread_support.c or win32_threads.c. */
144 : GC_bool GC_is_thread_tsd_valid(void *tsd);
145 : #endif
146 :
147 1437418 : GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc(size_t bytes)
148 : {
149 1437418 : size_t granules = ROUNDED_UP_GRANULES(bytes);
150 : void *tsd;
151 : void *result;
152 : void **tiny_fl;
153 :
154 : # if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
155 1437418 : GC_key_t k = GC_thread_key;
156 1437418 : if (EXPECT(0 == k, FALSE)) {
157 : /* We haven't yet run GC_init_parallel. That means */
158 : /* we also aren't locking, so this is fairly cheap. */
159 0 : return GC_core_malloc(bytes);
160 : }
161 1437418 : tsd = GC_getspecific(k);
162 : # else
163 : tsd = GC_getspecific(GC_thread_key);
164 : # endif
165 : # if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
166 : if (EXPECT(0 == tsd, FALSE)) {
167 : return GC_core_malloc(bytes);
168 : }
169 : # endif
170 : GC_ASSERT(GC_is_initialized);
171 :
172 : GC_ASSERT(GC_is_thread_tsd_valid(tsd));
173 :
174 1437418 : tiny_fl = ((GC_tlfs)tsd) -> normal_freelists;
175 1437418 : GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
176 : NORMAL, GC_core_malloc(bytes), obj_link(result)=0);
177 : # ifdef LOG_ALLOCS
178 : GC_log_printf("GC_malloc(%lu) returned %p, recent GC #%lu\n",
179 : (unsigned long)bytes, result, (unsigned long)GC_gc_no);
180 : # endif
181 1437418 : return result;
182 : }
183 :
184 580363 : GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_atomic(size_t bytes)
185 : {
186 580363 : size_t granules = ROUNDED_UP_GRANULES(bytes);
187 : void *tsd;
188 : void *result;
189 : void **tiny_fl;
190 :
191 : # if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
192 580363 : GC_key_t k = GC_thread_key;
193 580363 : if (EXPECT(0 == k, FALSE)) {
194 : /* We haven't yet run GC_init_parallel. That means */
195 : /* we also aren't locking, so this is fairly cheap. */
196 0 : return GC_core_malloc_atomic(bytes);
197 : }
198 580363 : tsd = GC_getspecific(k);
199 : # else
200 : tsd = GC_getspecific(GC_thread_key);
201 : # endif
202 : # if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
203 : if (EXPECT(0 == tsd, FALSE)) {
204 : return GC_core_malloc_atomic(bytes);
205 : }
206 : # endif
207 : GC_ASSERT(GC_is_initialized);
208 580363 : tiny_fl = ((GC_tlfs)tsd) -> ptrfree_freelists;
209 580363 : GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES, PTRFREE,
210 : GC_core_malloc_atomic(bytes), (void)0 /* no init */);
211 580363 : return result;
212 : }
213 :
214 : #ifdef GC_GCJ_SUPPORT
215 :
216 : # include "atomic_ops.h" /* for AO_compiler_barrier() */
217 :
218 : # include "include/gc_gcj.h"
219 :
220 : /* Gcj-style allocation without locks is extremely tricky. The */
221 : /* fundamental issue is that we may end up marking a free list, which */
222 : /* has freelist links instead of "vtable" pointers. That is usually */
223 : /* OK, since the next object on the free list will be cleared, and */
224 : /* will thus be interpreted as containing a zero descriptor. That's */
225 : /* fine if the object has not yet been initialized. But there are */
226 : /* interesting potential races. */
227 : /* In the case of incremental collection, this seems hopeless, since */
228 : /* the marker may run asynchronously, and may pick up the pointer to */
229 : /* the next freelist entry (which it thinks is a vtable pointer), get */
230 : /* suspended for a while, and then see an allocated object instead */
231 : /* of the vtable. This may be avoidable with either a handshake with */
232 : /* the collector or, probably more easily, by moving the free list */
233 : /* links to the second word of each object. The latter isn't a */
234 : /* universal win, since on architecture like Itanium, nonzero offsets */
235 : /* are not necessarily free. And there may be cache fill order issues. */
236 : /* For now, we punt with incremental GC. This probably means that */
237 : /* incremental GC should be enabled before we fork a second thread. */
238 : /* Unlike the other thread local allocation calls, we assume that the */
239 : /* collector has been explicitly initialized. */
240 0 : GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t bytes,
241 : void * ptr_to_struct_containing_descr)
242 : {
243 0 : if (EXPECT(GC_incremental, FALSE)) {
244 0 : return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr);
245 : } else {
246 0 : size_t granules = ROUNDED_UP_GRANULES(bytes);
247 : void *result;
248 0 : void **tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))
249 0 : -> gcj_freelists;
250 : GC_ASSERT(GC_gcj_malloc_initialized);
251 0 : GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
252 : GC_gcj_kind,
253 : GC_core_gcj_malloc(bytes,
254 : ptr_to_struct_containing_descr),
255 : {AO_compiler_barrier();
256 : *(void **)result = ptr_to_struct_containing_descr;});
257 : /* This forces the initialization of the "method ptr". */
258 : /* This is necessary to ensure some very subtle properties */
259 : /* required if a GC is run in the middle of such an allocation. */
260 : /* Here we implicitly also assume atomicity for the free list. */
261 : /* and method pointer assignments. */
262 : /* We must update the freelist before we store the pointer. */
263 : /* Otherwise a GC at this point would see a corrupted */
264 : /* free list. */
265 : /* A real memory barrier is not needed, since the */
266 : /* action of stopping this thread will cause prior writes */
267 : /* to complete. */
268 : /* We assert that any concurrent marker will stop us. */
269 : /* Thus it is impossible for a mark procedure to see the */
270 : /* allocation of the next object, but to see this object */
271 : /* still containing a free list pointer. Otherwise the */
272 : /* marker, by misinterpreting the freelist link as a vtable */
273 : /* pointer, might find a random "mark descriptor" in the next */
274 : /* object. */
275 0 : return result;
276 : }
277 : }
278 :
279 : #endif /* GC_GCJ_SUPPORT */
280 :
281 : /* The thread support layer must arrange to mark thread-local */
282 : /* free lists explicitly, since the link field is often */
283 : /* invisible to the marker. It knows how to find all threads; */
284 : /* we take care of an individual thread freelist structure. */
285 487 : GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p)
286 : {
287 : ptr_t q;
288 : int j;
289 :
290 12662 : for (j = 0; j < TINY_FREELISTS; ++j) {
291 12175 : q = p -> ptrfree_freelists[j];
292 12175 : if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
293 12175 : q = p -> normal_freelists[j];
294 12175 : if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
295 : # ifdef GC_GCJ_SUPPORT
296 12175 : if (j > 0) {
297 11688 : q = p -> gcj_freelists[j];
298 11688 : if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
299 : }
300 : # endif /* GC_GCJ_SUPPORT */
301 : # ifdef ENABLE_DISCLAIM
302 12175 : q = p -> finalized_freelists[j];
303 12175 : if ((word)q > HBLKSIZE)
304 0 : GC_set_fl_marks(q);
305 : # endif
306 : }
307 487 : }
308 :
309 : #if defined(GC_ASSERTIONS)
310 : /* Check that all thread-local free-lists in p are completely marked. */
311 : void GC_check_tls_for(GC_tlfs p)
312 : {
313 : int j;
314 :
315 : for (j = 1; j < TINY_FREELISTS; ++j) {
316 : GC_check_fl_marks(&p->ptrfree_freelists[j]);
317 : GC_check_fl_marks(&p->normal_freelists[j]);
318 : # ifdef GC_GCJ_SUPPORT
319 : GC_check_fl_marks(&p->gcj_freelists[j]);
320 : # endif
321 : # ifdef ENABLE_DISCLAIM
322 : GC_check_fl_marks(&p->finalized_freelists[j]);
323 : # endif
324 : }
325 : }
326 : #endif /* GC_ASSERTIONS */
327 :
328 : #endif /* THREAD_LOCAL_ALLOC */
|
