ruby-changes:66089
From: Matt <ko1@a...>
Date: Thu, 6 May 2021 22:18:43 +0900 (JST)
Subject: [ruby-changes:66089] 8bbd319806 (master): Allow newobj_of0 and newobj_slowpath to allocate into multiple heap slots
https://git.ruby-lang.org/ruby.git/commit/?id=8bbd319806 From 8bbd3198068f5e8335ab01f0b29cdae225b25b5b Mon Sep 17 00:00:00 2001 From: Matt Valentine-House <matt@e...> Date: Tue, 30 Mar 2021 13:34:14 +0100 Subject: Allow newobj_of0 and newobj_slowpath to allocate into multiple heap slots --- ext/objspace/objspace.c | 2 + gc.c | 457 +++++++++++++++++++++++++++++++++---- include/ruby/internal/value_type.h | 2 + internal/gc.h | 29 ++- vm_eval.c | 1 + 5 files changed, 435 insertions(+), 56 deletions(-) diff --git a/ext/objspace/objspace.c b/ext/objspace/objspace.c index 7afdfc1..2784a88 100644 --- a/ext/objspace/objspace.c +++ b/ext/objspace/objspace.c @@ -66,6 +66,7 @@ total_i(VALUE v, void *ptr) https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace.c#L66 case T_IMEMO: case T_ICLASS: case T_NODE: + case T_PAYLOAD: case T_ZOMBIE: return; default: @@ -224,6 +225,7 @@ type2sym(enum ruby_value_type i) https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace.c#L225 CASE_TYPE(T_ICLASS); CASE_TYPE(T_MOVED); CASE_TYPE(T_ZOMBIE); + CASE_TYPE(T_PAYLOAD); #undef CASE_TYPE default: rb_bug("type2sym: unknown type (%d)", i); } diff --git a/gc.c b/gc.c index dcdc8fd..6c9664c 100644 --- a/gc.c +++ b/gc.c @@ -551,6 +551,7 @@ typedef struct gc_profile_record { https://github.com/ruby/ruby/blob/trunk/gc.c#L551 } gc_profile_record; #define FL_FROM_FREELIST FL_USER0 +#define FL_FROM_PAYLOAD FL_USER0 struct RMoved { VALUE flags; @@ -564,12 +565,30 @@ struct RMoved { https://github.com/ruby/ruby/blob/trunk/gc.c#L565 #pragma pack(push, 4) /* == SIZEOF_VALUE: magic for reducing sizeof(RVALUE): 24 -> 20 */ #endif +struct RPayload { + VALUE flags; +}; +#define RPAYLOAD(obj) ((struct RPayload *)obj) +static unsigned short +RPAYLOAD_LEN(VALUE obj) { + unsigned short len = (unsigned short)(RPAYLOAD(obj)->flags >> FL_USHIFT); + return len; +} + +static void +RPAYLOAD_FLAGS_SET(VALUE obj, unsigned short len) +{ + // as len is the only thing in the user bits, we can overwrite it every time + RPAYLOAD(obj)->flags = T_PAYLOAD | (len << FL_USHIFT); +} + typedef struct RVALUE { union { struct { VALUE flags; /* always 0 for freed obj */ struct RVALUE *next; } free; + struct RPayload payload; struct RMoved moved; struct RBasic basic; struct RObject object; @@ -1266,6 +1285,35 @@ RVALUE_FLAGS_AGE(VALUE flags) https://github.com/ruby/ruby/blob/trunk/gc.c#L1285 return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT); } +#if USE_RVARGC +static VALUE +payload_or_self(VALUE obj) +{ + struct heap_page *p = GET_HEAP_PAGE(obj); + VALUE cur = (VALUE)p->start; + + while (cur != obj && GET_HEAP_PAGE(cur) == p) { + VALUE p = cur; + void *poisoned = asan_poisoned_object_p((VALUE)p); + asan_unpoison_object((VALUE)p, false); + + if (BUILTIN_TYPE(cur) == T_PAYLOAD) { + if (cur < obj && obj < cur + RPAYLOAD_LEN(cur) * sizeof(RVALUE)) { + return cur; + } + cur += RPAYLOAD_LEN(cur) * sizeof(RVALUE); + } else { + cur += sizeof(RVALUE); + } + if (poisoned) { + asan_poison_object((VALUE)p); + } + } + + return obj; +} +#endif + static int check_rvalue_consistency_force(const VALUE obj, int terminate) { @@ -1473,6 +1521,18 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag https://github.com/ruby/ruby/blob/trunk/gc.c#L1521 { MARK_IN_BITMAP(&page->uncollectible_bits[0], obj); objspace->rgengc.old_objects++; + +#if USE_RVARGC + if (BUILTIN_TYPE(obj) == T_PAYLOAD) { + int plen = RPAYLOAD_LEN(obj); + + for (int i = 1; i < plen; i++) { + VALUE pbody = obj + i * sizeof(RVALUE); + MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(pbody), pbody); + } + objspace->rgengc.old_objects += plen - 1; + } +#endif rb_transient_heap_promote(obj); #if RGENGC_PROFILE >= 2 @@ -1562,6 +1622,11 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj) https://github.com/ruby/ruby/blob/trunk/gc.c#L1622 if (RVALUE_MARKED(obj)) { objspace->rgengc.old_objects--; +#if USE_RVARGC + if (BUILTIN_TYPE(obj) == T_PAYLOAD) { + objspace->rgengc.old_objects -= RPAYLOAD_LEN(obj) - 1; + } +#endif } check_rvalue_consistency(obj); @@ -2203,10 +2268,144 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, https://github.com/ruby/ruby/blob/trunk/gc.c#L2268 return obj; } +static unsigned long +rvargc_slot_count(size_t size) +{ + // roomof == ceiling division, so we don't have to do div then mod + return roomof(size + sizeof(struct RPayload), sizeof(RVALUE)); +} + +static RVALUE * +rvargc_find_contiguous_slots(int slots, RVALUE *freelist) +{ + RVALUE *cursor = freelist; + RVALUE *previous_region = NULL; + + while(cursor) { + int i; + RVALUE *search = cursor; + for (i = 0; i < (slots - 1); i++) { + + // Peek ahead to see if the region is contiguous + if (search->as.free.next == (search - 1)) { + search = search->as.free.next; + } else { + // Next slot is not contiguous + if (search->as.free.next) { + cursor = search->as.free.next; + previous_region = search; + + break; + } else { + // Hit the end of the free list + return NULL; + } + } + } + + if (i == slots - 1) { + if (previous_region) { + previous_region->as.free.next = search->as.free.next; + search->as.free.next = freelist; + } + return search; + } + } + rb_bug("rvargc_find_contiguous_slots: unreachable"); +} + +static inline bool heap_add_freepage(rb_heap_t *heap, struct heap_page *page); +static struct heap_page * heap_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap); +static inline void ractor_set_cache(rb_ractor_t *cr, struct heap_page *page); + +static inline void * +rvargc_find_region(size_t size, rb_ractor_t *cr, RVALUE *freelist) +{ + // maintain master behaviour when we only need one slot + if (size == sizeof(RVALUE)) + return freelist; + + if (!freelist) return freelist; + + rb_objspace_t *objspace = &rb_objspace; + int slots = (int)rvargc_slot_count(size); + RVALUE * p = rvargc_find_contiguous_slots(slots, freelist); + + // We found a contiguous space on the freelist stored in the ractor cache + if (p) { + struct heap_page *page = GET_HEAP_PAGE(p); + + page->free_slots -= slots; + asan_unpoison_memory_region(p, sizeof(RVALUE) * slots, false); + return p; + } + else { + struct heap_page *search_page; + heap_allocatable_pages_set(objspace, heap_allocatable_pages + 1); + + while (!p) { + // search_page is the page we're going to search for contiguous slots + search_page = heap_next_freepage(objspace, heap_eden); + p = rvargc_find_contiguous_slots(slots, search_page->freelist); + + if (p) { + // Remove the region from the freelist + search_page->freelist = p->as.free.next; + search_page->free_slots -= slots; + + // If we started sweeping, the object cache can be removed + // from the ractor. Set it to the page we found + if (!cr->newobj_cache.using_page) { + ractor_set_cache(cr, search_page); + } + // Otherwise we need to add this page back to the list of free + // pages. + else { + // make this pointer point at the Ractor's freelist + p->as.free.next = freelist; + } + + asan_unpoison_memory_region(p, sizeof(RVALUE) * slots, false); + return p; + } + } + } + return NULL; +} + +int +rb_slot_size() +{ + return sizeof(RVALUE); +} + +VALUE +rb_rvargc_payload_init(VALUE obj, size_t size) +{ + rb_objspace_t * objspace = &rb_objspace; + struct RPayload *ph = (struct RPayload *)obj; + memset(ph, 0, rvargc_slot_count(size) * sizeof(RVALUE)); + + RPAYLOAD_FLAGS_SET((VALUE)ph, rvargc_slot_count(size)); + objspace->total_allocated_objects += rvargc_slot_count(size); + + return (VALUE)ph; +} + +void * +rb_rvargc_payload_data_ptr(VALUE phead) +{ + return (void *)(phead + sizeof(struct RPayload)); +} + static inline VALUE -ractor_cached_freeobj(rb_objspace_t *objspace, rb_ractor_t *cr) +ractor_cached_free_region(rb_objspace_t *objspace, rb_ractor_t *cr, size_t size) { +#if USE_RVARGC + RVALUE *p = rvargc_find_region(size, cr, cr->newobj_cache.freelist); +#else RVALUE *p = cr->newobj_cache.freelist; +#endif if (p) { VALUE obj = (VALUE)p; @@ -2241,13 +2440,9 @@ heap_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap) https://github.com/ruby/ruby/blob/trunk/gc.c#L2440 } static inline void -ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr) +ractor_set_cache(rb_ractor_t *cr, struct heap_page *page) { - ASSERT_vm_locking(); - GC_ASSERT(cr->newobj_cache.freelist == NULL); - - struct heap_page *page = heap_next_freepage(objspace, heap_eden); - + gc_report(3, &rb_objspace, "ractor_set_cache: Using page %p\n", (void *)GET_PAGE_BODY(page->start)); cr->newobj_cache.using_page = page; cr->newobj_cache.freelist = page->freelist; page->free_slots = 0; @@ -2258,6 +2453,16 @@ ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr) https://github.com/ruby/ruby/blob/trunk/gc.c#L2453 (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/