Line data Source code
1 : /*
2 : * Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
3 : *
4 : * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
5 : * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
6 : *
7 : * Permission is hereby granted to use or copy this program
8 : * for any purpose, provided the above notices are retained on all copies.
9 : * Permission to modify the code and to distribute modified code is granted,
10 : * provided the above notices are retained, and a notice that the code was
11 : * modified is included with the above copyright notice.
12 : */
13 :
14 : #include "private/gc_priv.h"
15 :
16 : #if defined(THREAD_LOCAL_ALLOC)
17 :
18 : #ifndef THREADS
19 : # error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS"
20 : #endif
21 :
22 : #include "private/thread_local_alloc.h"
23 :
24 : #include <stdlib.h>
25 :
26 : #if defined(USE_COMPILER_TLS)
27 : __thread
28 : #elif defined(USE_WIN32_COMPILER_TLS)
29 : __declspec(thread)
30 : #endif
31 : GC_key_t GC_thread_key;
32 :
33 : static GC_bool keys_initialized;
34 :
35 : /* Return a single nonempty freelist fl to the global one pointed to */
36 : /* by gfl. */
37 :
38 0 : static void return_single_freelist(void *fl, void **gfl)
39 : {
40 : void *q, **qptr;
41 :
42 0 : if (*gfl == 0) {
43 0 : *gfl = fl;
44 : } else {
45 : GC_ASSERT(GC_size(fl) == GC_size(*gfl));
46 : /* Concatenate: */
47 0 : qptr = &(obj_link(fl));
48 0 : while ((word)(q = *qptr) >= HBLKSIZE)
49 0 : qptr = &(obj_link(q));
50 : GC_ASSERT(0 == q);
51 0 : *qptr = *gfl;
52 0 : *gfl = fl;
53 : }
54 0 : }
55 :
56 : /* Recover the contents of the freelist array fl into the global one gfl.*/
57 : /* We hold the allocator lock. */
58 453 : static void return_freelists(void **fl, void **gfl)
59 : {
60 : int i;
61 :
62 11325 : for (i = 1; i < TINY_FREELISTS; ++i) {
63 10872 : if ((word)(fl[i]) >= HBLKSIZE) {
64 0 : return_single_freelist(fl[i], gfl+i);
65 : }
66 : /* Clear fl[i], since the thread structure may hang around. */
67 : /* Do it in a way that is likely to trap if we access it. */
68 10872 : fl[i] = (ptr_t)HBLKSIZE;
69 : }
70 : /* The 0 granule freelist really contains 1 granule objects. */
71 : # ifdef GC_GCJ_SUPPORT
72 453 : if (fl[0] == ERROR_FL) return;
73 : # endif
74 302 : if ((word)(fl[0]) >= HBLKSIZE) {
75 0 : return_single_freelist(fl[0], gfl+1);
76 : }
77 : }
78 :
79 : /* Each thread structure must be initialized. */
80 : /* This call must be made from the new thread. */
81 662 : GC_INNER void GC_init_thread_local(GC_tlfs p)
82 : {
83 : int i;
84 :
85 : GC_ASSERT(I_HOLD_LOCK());
86 662 : if (!keys_initialized) {
87 : if (0 != GC_key_create(&GC_thread_key, 0)) {
88 : ABORT("Failed to create key for local allocator");
89 : }
90 163 : keys_initialized = TRUE;
91 : }
92 662 : if (0 != GC_setspecific(GC_thread_key, p)) {
93 : ABORT("Failed to set thread specific allocation pointers");
94 : }
95 16550 : for (i = 1; i < TINY_FREELISTS; ++i) {
96 15888 : p -> ptrfree_freelists[i] = (void *)(word)1;
97 15888 : p -> normal_freelists[i] = (void *)(word)1;
98 : # ifdef GC_GCJ_SUPPORT
99 15888 : p -> gcj_freelists[i] = (void *)(word)1;
100 : # endif
101 : }
102 : /* Set up the size 0 free lists. */
103 : /* We now handle most of them like regular free lists, to ensure */
104 : /* That explicit deallocation works. However, allocation of a */
105 : /* size 0 "gcj" object is always an error. */
106 662 : p -> ptrfree_freelists[0] = (void *)(word)1;
107 662 : p -> normal_freelists[0] = (void *)(word)1;
108 : # ifdef GC_GCJ_SUPPORT
109 662 : p -> gcj_freelists[0] = ERROR_FL;
110 : # endif
111 662 : }
112 :
113 : /* We hold the allocator lock. */
114 151 : GC_INNER void GC_destroy_thread_local(GC_tlfs p)
115 : {
116 : /* We currently only do this from the thread itself or from */
117 : /* the fork handler for a child process. */
118 : # ifndef HANDLE_FORK
119 : GC_ASSERT(GC_getspecific(GC_thread_key) == (void *)p);
120 : # endif
121 151 : return_freelists(p -> ptrfree_freelists, GC_aobjfreelist);
122 151 : return_freelists(p -> normal_freelists, GC_objfreelist);
123 : # ifdef GC_GCJ_SUPPORT
124 151 : return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist);
125 : # endif
126 151 : }
127 :
128 : #ifdef GC_ASSERTIONS
129 : /* Defined in pthread_support.c or win32_threads.c. */
130 : GC_bool GC_is_thread_tsd_valid(void *tsd);
131 : #endif
132 :
133 1437417 : GC_API void * GC_CALL GC_malloc(size_t bytes)
134 : {
135 1437417 : size_t granules = ROUNDED_UP_GRANULES(bytes);
136 : void *tsd;
137 : void *result;
138 : void **tiny_fl;
139 :
140 : # if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
141 1437417 : GC_key_t k = GC_thread_key;
142 1437417 : if (EXPECT(0 == k, FALSE)) {
143 : /* We haven't yet run GC_init_parallel. That means */
144 : /* we also aren't locking, so this is fairly cheap. */
145 0 : return GC_core_malloc(bytes);
146 : }
147 1437417 : tsd = GC_getspecific(k);
148 : # else
149 : tsd = GC_getspecific(GC_thread_key);
150 : # endif
151 : # if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
152 : if (EXPECT(0 == tsd, FALSE)) {
153 : return GC_core_malloc(bytes);
154 : }
155 : # endif
156 : GC_ASSERT(GC_is_initialized);
157 :
158 : GC_ASSERT(GC_is_thread_tsd_valid(tsd));
159 :
160 1437417 : tiny_fl = ((GC_tlfs)tsd) -> normal_freelists;
161 1437417 : GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
162 : NORMAL, GC_core_malloc(bytes), obj_link(result)=0);
163 : # ifdef LOG_ALLOCS
164 : GC_err_printf("GC_malloc(%u) = %p : %u\n",
165 : (unsigned)bytes, result, (unsigned)GC_gc_no);
166 : # endif
167 1437417 : return result;
168 : }
169 :
170 580358 : GC_API void * GC_CALL GC_malloc_atomic(size_t bytes)
171 : {
172 580358 : size_t granules = ROUNDED_UP_GRANULES(bytes);
173 : void *tsd;
174 : void *result;
175 : void **tiny_fl;
176 :
177 : # if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
178 580358 : GC_key_t k = GC_thread_key;
179 580358 : if (EXPECT(0 == k, FALSE)) {
180 : /* We haven't yet run GC_init_parallel. That means */
181 : /* we also aren't locking, so this is fairly cheap. */
182 0 : return GC_core_malloc_atomic(bytes);
183 : }
184 580358 : tsd = GC_getspecific(k);
185 : # else
186 : tsd = GC_getspecific(GC_thread_key);
187 : # endif
188 : # if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
189 : if (EXPECT(0 == tsd, FALSE)) {
190 : return GC_core_malloc_atomic(bytes);
191 : }
192 : # endif
193 : GC_ASSERT(GC_is_initialized);
194 580358 : tiny_fl = ((GC_tlfs)tsd) -> ptrfree_freelists;
195 580358 : GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES, PTRFREE,
196 : GC_core_malloc_atomic(bytes), (void)0 /* no init */);
197 580358 : return result;
198 : }
199 :
200 : #ifdef GC_GCJ_SUPPORT
201 :
202 : # include "atomic_ops.h" /* for AO_compiler_barrier() */
203 :
204 : # include "include/gc_gcj.h"
205 :
206 : /* Gcj-style allocation without locks is extremely tricky. The */
207 : /* fundamental issue is that we may end up marking a free list, which */
208 : /* has freelist links instead of "vtable" pointers. That is usually */
209 : /* OK, since the next object on the free list will be cleared, and */
210 : /* will thus be interpreted as containing a zero descriptor. That's */
211 : /* fine if the object has not yet been initialized. But there are */
212 : /* interesting potential races. */
213 : /* In the case of incremental collection, this seems hopeless, since */
214 : /* the marker may run asynchronously, and may pick up the pointer to */
215 : /* the next freelist entry (which it thinks is a vtable pointer), get */
216 : /* suspended for a while, and then see an allocated object instead */
217 : /* of the vtable. This may be avoidable with either a handshake with */
218 : /* the collector or, probably more easily, by moving the free list */
219 : /* links to the second word of each object. The latter isn't a */
220 : /* universal win, since on architecture like Itanium, nonzero offsets */
221 : /* are not necessarily free. And there may be cache fill order issues. */
222 : /* For now, we punt with incremental GC. This probably means that */
223 : /* incremental GC should be enabled before we fork a second thread. */
224 : /* Unlike the other thread local allocation calls, we assume that the */
225 : /* collector has been explicitly initialized. */
226 0 : GC_API void * GC_CALL GC_gcj_malloc(size_t bytes,
227 : void * ptr_to_struct_containing_descr)
228 : {
229 0 : if (GC_EXPECT(GC_incremental, 0)) {
230 0 : return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr);
231 : } else {
232 0 : size_t granules = ROUNDED_UP_GRANULES(bytes);
233 : void *result;
234 0 : void **tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))
235 0 : -> gcj_freelists;
236 : GC_ASSERT(GC_gcj_malloc_initialized);
237 0 : GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
238 : GC_gcj_kind,
239 : GC_core_gcj_malloc(bytes,
240 : ptr_to_struct_containing_descr),
241 : {AO_compiler_barrier();
242 : *(void **)result = ptr_to_struct_containing_descr;});
243 : /* This forces the initialization of the "method ptr". */
244 : /* This is necessary to ensure some very subtle properties */
245 : /* required if a GC is run in the middle of such an allocation. */
246 : /* Here we implicitly also assume atomicity for the free list. */
247 : /* and method pointer assignments. */
248 : /* We must update the freelist before we store the pointer. */
249 : /* Otherwise a GC at this point would see a corrupted */
250 : /* free list. */
251 : /* A real memory barrier is not needed, since the */
252 : /* action of stopping this thread will cause prior writes */
253 : /* to complete. */
254 : /* We assert that any concurrent marker will stop us. */
255 : /* Thus it is impossible for a mark procedure to see the */
256 : /* allocation of the next object, but to see this object */
257 : /* still containing a free list pointer. Otherwise the */
258 : /* marker, by misinterpreting the freelist link as a vtable */
259 : /* pointer, might find a random "mark descriptor" in the next */
260 : /* object. */
261 0 : return result;
262 : }
263 : }
264 :
265 : #endif /* GC_GCJ_SUPPORT */
266 :
267 : /* The thread support layer must arrange to mark thread-local */
268 : /* free lists explicitly, since the link field is often */
269 : /* invisible to the marker. It knows how to find all threads; */
270 : /* we take care of an individual thread freelist structure. */
271 543 : GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p)
272 : {
273 : ptr_t q;
274 : int j;
275 :
276 14118 : for (j = 0; j < TINY_FREELISTS; ++j) {
277 13575 : q = p -> ptrfree_freelists[j];
278 13575 : if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
279 13575 : q = p -> normal_freelists[j];
280 13575 : if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
281 : # ifdef GC_GCJ_SUPPORT
282 13575 : if (j > 0) {
283 13032 : q = p -> gcj_freelists[j];
284 13032 : if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
285 : }
286 : # endif /* GC_GCJ_SUPPORT */
287 : }
288 543 : }
289 :
290 : #if defined(GC_ASSERTIONS)
291 : /* Check that all thread-local free-lists in p are completely marked. */
292 : void GC_check_tls_for(GC_tlfs p)
293 : {
294 : int j;
295 :
296 : for (j = 1; j < TINY_FREELISTS; ++j) {
297 : GC_check_fl_marks(&p->ptrfree_freelists[j]);
298 : GC_check_fl_marks(&p->normal_freelists[j]);
299 : # ifdef GC_GCJ_SUPPORT
300 : GC_check_fl_marks(&p->gcj_freelists[j]);
301 : # endif
302 : }
303 : }
304 : #endif /* GC_ASSERTIONS */
305 :
306 : #endif /* THREAD_LOCAL_ALLOC */
|
