Ruby 3.2.3p157 (2024-01-18 revision 52bb2ac0a6971d0391efa2275f7a66bff319087c)
transient_heap.c
1/**********************************************************************
2
3 transient_heap.c - implement transient_heap.
4
5 Copyright (C) 2018 Koichi Sasada
6
7**********************************************************************/
8
9#include "debug_counter.h"
10#include "gc.h"
11#include "internal.h"
12#include "internal/array.h"
13#include "internal/gc.h"
14#include "internal/hash.h"
15#include "internal/sanitizers.h"
16#include "internal/static_assert.h"
17#include "internal/struct.h"
18#include "internal/variable.h"
19#include "ruby/debug.h"
20#include "ruby/ruby.h"
21#include "ruby_assert.h"
22#include "transient_heap.h"
23#include "vm_debug.h"
24#include "vm_sync.h"
25
26#if USE_TRANSIENT_HEAP /* USE_TRANSIENT_HEAP */
27/*
28 * 1: enable assertions
29 * 2: enable verify all transient heaps
30 */
31#ifndef TRANSIENT_HEAP_CHECK_MODE
32#define TRANSIENT_HEAP_CHECK_MODE 0
33#endif
34#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
35
36/*
37 * 1: show events
38 * 2: show dump at events
39 * 3: show all operations
40 */
41#define TRANSIENT_HEAP_DEBUG 0
42
43/* For Debug: Provide blocks infinitely.
44 * This mode generates blocks unlimitedly
45 * and prohibit access free'ed blocks to check invalid access.
46 */
47#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
48
49#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
50#include <sys/mman.h>
51#include <errno.h>
52#endif
53
54/* For Debug: Prohibit promoting to malloc space.
55 */
56#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
57
58/* size configuration */
59#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
60
61 /* K M */
62#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
63#ifndef TRANSIENT_HEAP_TOTAL_SIZE
64#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
65#endif
66#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
67#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
68#define TRANSIENT_HEAP_USABLE_SIZE (TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header))
69
70#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
71#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
72
73#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
74#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
75
76enum transient_heap_status {
77 transient_heap_none,
78 transient_heap_marking,
79 transient_heap_escaping
80};
81
82struct transient_heap_block {
83 struct transient_heap_block_header {
84 int16_t index;
85 int16_t last_marked_index;
86 int16_t objects;
87 struct transient_heap_block *next_block;
88 } info;
89 char buff[TRANSIENT_HEAP_USABLE_SIZE];
90};
91
92struct transient_heap {
93 struct transient_heap_block *using_blocks;
94 struct transient_heap_block *marked_blocks;
95 struct transient_heap_block *free_blocks;
96 int total_objects;
97 int total_marked_objects;
98 int total_blocks;
99 enum transient_heap_status status;
100
101 VALUE *promoted_objects;
102 int promoted_objects_size;
103 int promoted_objects_index;
104
105 struct transient_heap_block *arena;
106 int arena_index; /* increment only */
107};
108
109struct transient_alloc_header {
110 uint16_t magic;
111 uint16_t size;
112 int16_t next_marked_index;
113 int16_t dummy;
114 VALUE obj;
115};
116
117static struct transient_heap global_transient_heap;
118
119static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
120static const void *transient_heap_ptr(VALUE obj, int error);
121static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
122
123#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
124
125static void
126transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
127{
128 int i=0, n=0;
129
130 while (i<block->info.index) {
131 void *ptr = &block->buff[i];
132 struct transient_alloc_header *header = ptr;
133 fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, ptr, header->size, header->next_marked_index, rb_obj_info(header->obj));
134 i += header->size;
135 n++;
136 }
137}
138
139static void
140transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
141{
142 while (block) {
143 fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
144 type_str, (void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (void *)block->info.next_block);
145
146 transient_heap_block_dump(theap, block);
147 block = block->info.next_block;
148 }
149}
150
151static void
152transient_heap_dump(struct transient_heap* theap)
153{
154 fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
155 transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
156 transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
157 transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
158}
159
160/* Debug: dump all transient_heap blocks */
161void
162rb_transient_heap_dump(void)
163{
164 transient_heap_dump(&global_transient_heap);
165}
166
167#if TRANSIENT_HEAP_CHECK_MODE >= 2
168ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void transient_heap_ptr_check(struct transient_heap *theap, VALUE obj));
169static void
170transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
171{
172 if (!UNDEF_P(obj)) {
173 const void *ptr = transient_heap_ptr(obj, FALSE);
174 TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
175 }
176}
177
178ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static int transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block));
179static int
180transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
181{
182 int i=0, n=0;
183 struct transient_alloc_header *header;
184
185 while (i<block->info.index) {
186 header = (void *)&block->buff[i];
187 TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
188 transient_heap_ptr_check(theap, header->obj);
189 n ++;
190 i += header->size;
191 }
192 TH_ASSERT(block->info.objects == n);
193
194 return n;
195}
196
197static int
198transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
199{
200 int n = 0;
201 struct transient_heap_block *block = blocks;
202 while (block) {
203 n += transient_heap_block_verify(theap, block);
204 *block_num_ptr += 1;
205 block = block->info.next_block;
206 }
207
208 return n;
209}
210#endif
211
212static void
213transient_heap_verify(struct transient_heap *theap)
214{
215#if TRANSIENT_HEAP_CHECK_MODE >= 2
216 int n=0, block_num=0;
217
218 n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
219 n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
220
221 TH_ASSERT(n == theap->total_objects);
222 TH_ASSERT(n >= theap->total_marked_objects);
223 TH_ASSERT(block_num == theap->total_blocks);
224#endif
225}
226
227/* Debug: check assertions for all transient_heap blocks */
228void
229rb_transient_heap_verify(void)
230{
231 transient_heap_verify(&global_transient_heap);
232}
233
234static struct transient_heap*
235transient_heap_get(void)
236{
237 struct transient_heap* theap = &global_transient_heap;
238 transient_heap_verify(theap);
239 return theap;
240}
241
242static void
243reset_block(struct transient_heap_block *block)
244{
245 __msan_allocated_memory(block, sizeof block);
246 block->info.index = 0;
247 block->info.objects = 0;
248 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
249 block->info.next_block = NULL;
250 __asan_poison_memory_region(&block->buff, sizeof block->buff);
251}
252
253static void
254connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
255{
256 block->info.next_block = theap->free_blocks;
257 theap->free_blocks = block;
258}
259
260static void
261connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
262{
263 block->info.next_block = theap->using_blocks;
264 theap->using_blocks = block;
265}
266
267#if 0
268static void
269connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
270{
271 block->info.next_block = theap->marked_blocks;
272 theap->marked_blocks = block;
273}
274#endif
275
276static void
277append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
278{
279 if (theap->marked_blocks) {
280 struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
281 while (block) {
282 last_block = block;
283 block = block->info.next_block;
284 }
285
286 TH_ASSERT(last_block->info.next_block == NULL);
287 last_block->info.next_block = append_blocks;
288 }
289 else {
290 theap->marked_blocks = append_blocks;
291 }
292}
293
294static struct transient_heap_block *
295transient_heap_block_alloc(struct transient_heap* theap)
296{
297 struct transient_heap_block *block;
298#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
299 block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
300 MAP_PRIVATE | MAP_ANONYMOUS,
301 -1, 0);
302 if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
303#else
304 if (theap->arena == NULL) {
305 theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE);
306 if (theap->arena == NULL) {
307 rb_bug("transient_heap_block_alloc: failed\n");
308 }
309 }
310
311 TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM);
312 block = &theap->arena[theap->arena_index++];
313 TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
314#endif
315 reset_block(block);
316
317 TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
318 if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (void *)block);
319 return block;
320}
321
322
323static struct transient_heap_block *
324transient_heap_allocatable_block(struct transient_heap* theap)
325{
326 struct transient_heap_block *block;
327
328#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
329 block = transient_heap_block_alloc(theap);
330 theap->total_blocks++;
331#else
332 /* get one block from free_blocks */
333 block = theap->free_blocks;
334 if (block) {
335 theap->free_blocks = block->info.next_block;
336 block->info.next_block = NULL;
337 theap->total_blocks++;
338 }
339#endif
340
341 return block;
342}
343
344static struct transient_alloc_header *
345transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
346{
347 struct transient_heap_block *block = theap->using_blocks;
348
349 while (block) {
350 TH_ASSERT(block->info.index <= (int16_t)TRANSIENT_HEAP_USABLE_SIZE);
351
352 if (TRANSIENT_HEAP_USABLE_SIZE - block->info.index >= size) {
353 struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
354 block->info.index += size;
355 block->info.objects++;
356 return header;
357 }
358 else {
359 block = transient_heap_allocatable_block(theap);
360 if (block) connect_to_using_blocks(theap, block);
361 }
362 }
363
364 return NULL;
365}
366
367void *
368rb_transient_heap_alloc(VALUE obj, size_t req_size)
369{
370 // only on single main ractor
371 if (ruby_single_main_ractor == NULL) return NULL;
372
373 void *ret;
374 struct transient_heap* theap = transient_heap_get();
375 size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
376
377 TH_ASSERT(RB_TYPE_P(obj, T_ARRAY) ||
378 RB_TYPE_P(obj, T_OBJECT) ||
379 RB_TYPE_P(obj, T_STRUCT) ||
380 RB_TYPE_P(obj, T_HASH)); /* supported types */
381
382 if (size > TRANSIENT_HEAP_ALLOC_MAX) {
383 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
384 ret = NULL;
385 }
386#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
387 else if (RB_OBJ_PROMOTED_RAW(obj)) {
388 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
389 ret = NULL;
390 }
391#else
392 else if (RBASIC_CLASS(obj) == 0) {
393 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
394 ret = NULL;
395 }
396#endif
397 else {
398 struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
399 if (header) {
400 void *ptr;
401
402 /* header is poisoned to prevent buffer overflow, should
403 * unpoison first... */
404 asan_unpoison_memory_region(header, sizeof *header, true);
405
406 header->size = size;
407 header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;
408 header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
409 header->obj = obj; /* TODO: can we eliminate it? */
410
411 /* header is fixed; shall poison again */
412 asan_poison_memory_region(header, sizeof *header);
413 ptr = header + 1;
414
415 theap->total_objects++; /* statistics */
416
417#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
418 if (RB_OBJ_PROMOTED_RAW(obj)) {
419 transient_heap_promote_add(theap, obj);
420 }
421#endif
422 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (void *)header, ptr, (int)size, rb_obj_info(obj));
423
424 RB_DEBUG_COUNTER_INC(theap_alloc);
425
426 /* ptr is set up; OK to unpoison. */
427 asan_unpoison_memory_region(ptr, size - sizeof *header, true);
428 ret = ptr;
429 }
430 else {
431 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
432 RB_DEBUG_COUNTER_INC(theap_alloc_fail);
433 ret = NULL;
434 }
435 }
436
437 return ret;
438}
439
440void
441Init_TransientHeap(void)
442{
443 int i, block_num;
444 struct transient_heap* theap = transient_heap_get();
445
446#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
447 block_num = 0;
448#else
449 TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);
450 block_num = TRANSIENT_HEAP_BLOCK_NUM;
451#endif
452 for (i=0; i<block_num; i++) {
453 connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
454 }
455 theap->using_blocks = transient_heap_allocatable_block(theap);
456
457 theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;
458 theap->promoted_objects_index = 0;
459 /* should not use ALLOC_N to be free from GC */
460 theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
461 STATIC_ASSERT(
462 integer_overflow,
463 sizeof(VALUE) <= SIZE_MAX / TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE);
464 if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
465}
466
467static struct transient_heap_block *
468blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
469{
470 struct transient_heap_block *block = blocks;
471
472 while (block) {
473 if (block->buff <= (char *)header && (char *)header < block->buff + TRANSIENT_HEAP_USABLE_SIZE) {
474 return block;
475 }
476 block = block->info.next_block;
477 }
478
479 return NULL;
480}
481
482static struct transient_heap_block *
483alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
484{
485 struct transient_heap_block *block;
486
487 if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
488 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
489 return block;
490 }
491 else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
492 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
493 return block;
494 }
495 else {
496 return NULL;
497 }
498}
499
500static struct transient_alloc_header *
501ptr_to_alloc_header(const void *ptr)
502{
503 struct transient_alloc_header *header = (void *)ptr;
504 header -= 1;
505 return header;
506}
507
508static int
509transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
510{
511 if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
512 return TRUE;
513 }
514 else {
515 return FALSE;
516 }
517}
518
519
520int
521rb_transient_heap_managed_ptr_p(const void *ptr)
522{
523 return transient_header_managed_ptr_p(transient_heap_get(), ptr);
524}
525
526static struct transient_heap_block *
527alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
528{
529 struct transient_heap_block *block;
530#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
531 block = alloc_header_to_block_verbose(theap, header);
532 if (block == NULL) {
533 transient_heap_dump(theap);
534 rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
535 }
536#else
537 block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
538 TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
539#endif
540 return block;
541}
542
543void
544rb_transient_heap_mark(VALUE obj, const void *ptr)
545{
546 ASSERT_vm_locking();
547
548 struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
549 asan_unpoison_memory_region(header, sizeof *header, false);
550 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header, %s (%p)", rb_obj_info(obj), ptr);
551 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
552
553#if TRANSIENT_HEAP_CHECK_MODE > 0
554 {
555 struct transient_heap* theap = transient_heap_get();
556 TH_ASSERT(theap->status == transient_heap_marking);
557 TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
558
559 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
560 transient_heap_dump(theap);
561 rb_bug("rb_transient_heap_mark: magic is broken");
562 }
563 else if (header->obj != obj) {
564 // transient_heap_dump(theap);
565 rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
566 rb_obj_info(header->obj), rb_obj_info(obj));
567 }
568 }
569#endif
570
571 if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {
572 /* already marked */
573 return;
574 }
575 else {
576 struct transient_heap* theap = transient_heap_get();
577 struct transient_heap_block *block = alloc_header_to_block(theap, header);
578 __asan_unpoison_memory_region(&block->info, sizeof block->info);
579 header->next_marked_index = block->info.last_marked_index;
580 block->info.last_marked_index = (int)((char *)header - block->buff);
581 theap->total_marked_objects++;
582
583 transient_heap_verify(theap);
584 }
585}
586
587ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error));
588static const void *
589transient_heap_ptr(VALUE obj, int error)
590{
591 const void *ptr = NULL;
592
593 switch (BUILTIN_TYPE(obj)) {
594 case T_ARRAY:
595 if (RARRAY_TRANSIENT_P(obj)) {
596 TH_ASSERT(!ARY_EMBED_P(obj));
597 ptr = RARRAY(obj)->as.heap.ptr;
598 }
599 break;
600 case T_OBJECT:
601 if (ROBJ_TRANSIENT_P(obj)) {
602 RUBY_ASSERT(!rb_shape_obj_too_complex(obj));
603 ptr = ROBJECT_IVPTR(obj);
604 }
605 break;
606 case T_STRUCT:
607 if (RSTRUCT_TRANSIENT_P(obj)) {
608 ptr = rb_struct_const_heap_ptr(obj);
609 }
610 break;
611 case T_HASH:
612 if (RHASH_TRANSIENT_P(obj)) {
613 TH_ASSERT(RHASH_AR_TABLE_P(obj));
614 ptr = (VALUE *)(RHASH(obj)->as.ar);
615 }
616 else {
617 ptr = NULL;
618 }
619 break;
620 default:
621 if (error) {
622 rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
623 }
624 }
625
626 return ptr;
627}
628
629static void
630transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
631{
632 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
633
634 if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) {
635 /* duplicate check */
636 int i;
637 for (i=0; i<theap->promoted_objects_index; i++) {
638 if (theap->promoted_objects[i] == obj) return;
639 }
640 }
641
642 if (theap->promoted_objects_size <= theap->promoted_objects_index) {
643 theap->promoted_objects_size *= 2;
644 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
645 if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) {
646 /* realloc failure due to integer overflow */
647 theap->promoted_objects = NULL;
648 }
649 else {
650 theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
651 }
652 if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
653 }
654 theap->promoted_objects[theap->promoted_objects_index++] = obj;
655}
656
657void
658rb_transient_heap_promote(VALUE obj)
659{
660 ASSERT_vm_locking();
661
662 if (transient_heap_ptr(obj, FALSE)) {
663 struct transient_heap* theap = transient_heap_get();
664 transient_heap_promote_add(theap, obj);
665 }
666 else {
667 /* ignore */
668 }
669}
670
671static struct transient_alloc_header *
672alloc_header(struct transient_heap_block* block, int index)
673{
674 return (void *)&block->buff[index];
675}
676
677static void
678transient_heap_reset(void)
679{
680 ASSERT_vm_locking();
681
682 struct transient_heap* theap = transient_heap_get();
683 struct transient_heap_block* block;
684
685 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
686
687 block = theap->marked_blocks;
688 while (block) {
689 struct transient_heap_block *next_block = block->info.next_block;
690 theap->total_objects -= block->info.objects;
691#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
692 if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
693 rb_bug("madvise err:%d", errno);
694 }
695 if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
696 rb_bug("mprotect err:%d", errno);
697 }
698#else
699 reset_block(block);
700 connect_to_free_blocks(theap, block);
701#endif
702 theap->total_blocks--;
703 block = next_block;
704 }
705
706 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
707
708 theap->marked_blocks = NULL;
709 theap->total_marked_objects = 0;
710}
711
712static void
713transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
714{
715 int marked_index = block->info.last_marked_index;
716 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
717
718 while (marked_index >= 0) {
719 struct transient_alloc_header *header = alloc_header(block, marked_index);
720 asan_unpoison_memory_region(header, sizeof *header, true);
721 VALUE obj = header->obj;
722 TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
723 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("transient_heap_block_evacuate: wrong header %p %s\n", (void *)header, rb_obj_info(obj));
724
725 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", (void *)header, rb_obj_info(obj));
726
727 if (obj != Qnil) {
728 RB_DEBUG_COUNTER_INC(theap_evacuate);
729
730 switch (BUILTIN_TYPE(obj)) {
731 case T_ARRAY:
732 rb_ary_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
733 break;
734 case T_OBJECT:
735 rb_obj_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
736 break;
737 case T_STRUCT:
738 rb_struct_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
739 break;
740 case T_HASH:
741 rb_hash_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
742 break;
743 default:
744 rb_bug("unsupported: %s\n", rb_obj_info(obj));
745 }
746 header->obj = Qundef; /* for debug */
747 }
748 marked_index = header->next_marked_index;
749 asan_poison_memory_region(header, sizeof *header);
750 }
751}
752
753#if USE_RUBY_DEBUG_LOG
754static const char *
755transient_heap_status_cstr(enum transient_heap_status status)
756{
757 switch (status) {
758 case transient_heap_none: return "none";
759 case transient_heap_marking: return "marking";
760 case transient_heap_escaping: return "escaping";
761 }
762 UNREACHABLE_RETURN(NULL);
763}
764#endif
765
766static void
767transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
768{
769 RUBY_DEBUG_LOG("%s -> %s",
770 transient_heap_status_cstr(theap->status),
771 transient_heap_status_cstr(status));
772
773 TH_ASSERT(theap->status != status);
774 theap->status = status;
775}
776
777static void
778transient_heap_evacuate(void *dmy)
779{
780 struct transient_heap* theap = transient_heap_get();
781
782 if (theap->total_marked_objects == 0) return;
783 if (ruby_single_main_ractor == NULL) rb_bug("not single ractor mode");
784 if (theap->status == transient_heap_marking) {
785 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
786 }
787 else {
788 VALUE gc_disabled = rb_gc_disable_no_rest();
789 {
790 struct transient_heap_block* block;
791
792 RUBY_DEBUG_LOG("start gc_disabled:%d", RTEST(gc_disabled));
793
794 if (TRANSIENT_HEAP_DEBUG >= 1) {
795 int i;
796 fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
797 if (TRANSIENT_HEAP_DEBUG >= 4) {
798 for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
799 }
800 }
801 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
802
803 TH_ASSERT(theap->status == transient_heap_none);
804 transient_heap_update_status(theap, transient_heap_escaping);
805
806 /* evacuate from marked blocks */
807 block = theap->marked_blocks;
808 while (block) {
809 transient_heap_block_evacuate(theap, block);
810 block = block->info.next_block;
811 }
812
813 /* evacuate from using blocks
814 only affect incremental marking */
815 block = theap->using_blocks;
816 while (block) {
817 transient_heap_block_evacuate(theap, block);
818 block = block->info.next_block;
819 }
820
821 /* all objects in marked_objects are escaped. */
822 transient_heap_reset();
823
824 if (TRANSIENT_HEAP_DEBUG > 0) {
825 fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
826 }
827
828 transient_heap_verify(theap);
829 transient_heap_update_status(theap, transient_heap_none);
830 }
831 if (gc_disabled != Qtrue) rb_gc_enable();
832 RUBY_DEBUG_LOG("finish");
833 }
834}
835
836void
837rb_transient_heap_evacuate(void)
838{
839 transient_heap_evacuate(NULL);
840}
841
842static void
843clear_marked_index(struct transient_heap_block* block)
844{
845 int marked_index = block->info.last_marked_index;
846
847 while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
848 struct transient_alloc_header *header = alloc_header(block, marked_index);
849 /* header is poisoned to prevent buffer overflow, should
850 * unpoison first... */
851 asan_unpoison_memory_region(header, sizeof *header, false);
852 TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);
853 if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", (void *)block, marked_index);
854
855 marked_index = header->next_marked_index;
856 header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
857 }
858
859 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
860}
861
862static void
863blocks_clear_marked_index(struct transient_heap_block* block)
864{
865 while (block) {
866 clear_marked_index(block);
867 block = block->info.next_block;
868 }
869}
870
871static void
872transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block)
873{
874 int marked_index = block->info.last_marked_index;
875
876 while (marked_index >= 0) {
877 struct transient_alloc_header *header = alloc_header(block, marked_index);
878
879 asan_unpoison_memory_region(header, sizeof *header, false);
880
881 header->obj = rb_gc_location(header->obj);
882
883 marked_index = header->next_marked_index;
884 asan_poison_memory_region(header, sizeof *header);
885 }
886}
887
888static void
889transient_heap_blocks_update_refs(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
890{
891 while (block) {
892 transient_heap_block_update_refs(theap, block);
893 block = block->info.next_block;
894 }
895}
896
897void
898rb_transient_heap_update_references(void)
899{
900 ASSERT_vm_locking();
901
902 struct transient_heap* theap = transient_heap_get();
903 int i;
904
905 transient_heap_blocks_update_refs(theap, theap->using_blocks, "using_blocks");
906 transient_heap_blocks_update_refs(theap, theap->marked_blocks, "marked_blocks");
907
908 for (i=0; i<theap->promoted_objects_index; i++) {
909 VALUE obj = theap->promoted_objects[i];
910 theap->promoted_objects[i] = rb_gc_location(obj);
911 }
912}
913
914void
915rb_transient_heap_start_marking(int full_marking)
916{
917 ASSERT_vm_locking();
918 RUBY_DEBUG_LOG("full?:%d", full_marking);
919
920 struct transient_heap* theap = transient_heap_get();
921
922 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promoted:%d full_marking:%d\n",
923 theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
924 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
925
926 blocks_clear_marked_index(theap->marked_blocks);
927 blocks_clear_marked_index(theap->using_blocks);
928
929 if (theap->using_blocks) {
930 if (theap->using_blocks->info.objects > 0) {
931 append_to_marked_blocks(theap, theap->using_blocks);
932 theap->using_blocks = NULL;
933 }
934 else {
935 append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
936 theap->using_blocks->info.next_block = NULL;
937 }
938 }
939
940 if (theap->using_blocks == NULL) {
941 theap->using_blocks = transient_heap_allocatable_block(theap);
942 }
943
944 TH_ASSERT(theap->status == transient_heap_none);
945 transient_heap_update_status(theap, transient_heap_marking);
946 theap->total_marked_objects = 0;
947
948 if (full_marking) {
949 theap->promoted_objects_index = 0;
950 }
951 else { /* mark promoted objects */
952 int i;
953 for (i=0; i<theap->promoted_objects_index; i++) {
954 VALUE obj = theap->promoted_objects[i];
955 const void *ptr = transient_heap_ptr(obj, TRUE);
956 if (ptr) {
957 rb_transient_heap_mark(obj, ptr);
958 }
959 }
960 }
961
962 transient_heap_verify(theap);
963}
964
965void
966rb_transient_heap_finish_marking(void)
967{
968 ASSERT_vm_locking();
969 struct transient_heap* theap = transient_heap_get();
970
971 RUBY_DEBUG_LOG("objects:%d, marked:%d",
972 theap->total_objects,
973 theap->total_marked_objects);
974 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
975
976 TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
977
978 TH_ASSERT(theap->status == transient_heap_marking);
979 transient_heap_update_status(theap, transient_heap_none);
980
981 if (theap->total_marked_objects > 0) {
982 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
983 rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
984 }
985 else {
986 transient_heap_reset();
987 }
988
989 transient_heap_verify(theap);
990}
991#endif /* USE_TRANSIENT_HEAP */
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:177
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
Identical to rb_postponed_job_register_one(), except it additionally checks for duplicated registrati...
Definition vm_trace.c:1703
#define Qundef
Old name of RUBY_Qundef.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define Qtrue
Old name of RUBY_Qtrue.
#define Qnil
Old name of RUBY_Qnil.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition error.c:794
#define RARRAY(obj)
Convenient casting macro.
Definition rarray.h:56
static bool RARRAY_TRANSIENT_P(VALUE ary)
Queries if the array is a transient array.
Definition rarray.h:364
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:152
static bool RB_OBJ_PROMOTED_RAW(VALUE obj)
This is the implementation of RB_OBJ_PROMOTED().
Definition rgengc.h:323
static VALUE * ROBJECT_IVPTR(VALUE obj)
Queries the instance variables.
Definition robject.h:162
#define RTEST
This is an old name of RB_TEST.
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:375