https://git.ruby-lang.org/ruby.git/commit/?id=91d99025e4

From 91d99025e4776885ceea809123a129cb31cd1db6 Mon Sep 17 00:00:00 2001
From: Koichi Sasada <ko1@a...>
Date: Thu, 3 Dec 2020 15:57:39 +0900
Subject: per-ractor object allocation

Now object allocation requires VM global lock to synchronize objspace.
However, of course, it introduces huge overhead.
This patch caches some slots (in a page) by each ractor and use cached
slots for object allocation. If there is no cached slots, acquire the global lock
and get new cached slots, or start GC (marking or lazy sweeping).

diff --git a/gc.c b/gc.c
index 306ad0e..ea21e61 100644
--- a/gc.c
+++ b/gc.c
@@ -645,10 +645,7 @@ typedef struct mark_stack { https://github.com/ruby/ruby/blob/trunk/gc.c#L645
 } mark_stack_t;
 
 typedef struct rb_heap_struct {
-    RVALUE *freelist;
-
     struct heap_page *free_pages;
-    struct heap_page *using_page;
     struct list_head pages;
     struct heap_page *sweeping_page; /* iterator for .pages */
     struct heap_page *compact_cursor;
@@ -1081,6 +1078,7 @@ static inline void gc_prof_set_heap_info(rb_objspace_t *); https://github.com/ruby/ruby/blob/trunk/gc.c#L1078
 #endif
 PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
 static const char *obj_info(VALUE obj);
+static const char *obj_type_name(VALUE obj);
 
 /*
  * 1 - TSC (H/W Time Stamp Counter)
@@ -1676,11 +1674,13 @@ heap_allocatable_pages_set(rb_objspace_t *objspace, size_t s) https://github.com/ruby/ruby/blob/trunk/gc.c#L1674
     heap_pages_expand_sorted(objspace);
 }
 
-
 static inline void
 heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
 {
+    ASSERT_vm_locking();
+
     RVALUE *p = (RVALUE *)obj;
+
     asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
 
     p->as.free.flags = 0;
@@ -1697,20 +1697,28 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj https://github.com/ruby/ruby/blob/trunk/gc.c#L1697
     }
 
     asan_poison_object(obj);
-
     gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
 }
 
-static inline void
+static inline bool
 heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
 {
     asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
     GC_ASSERT(page->free_slots != 0);
+
     if (page->freelist) {
-	page->free_next = heap->free_pages;
-	heap->free_pages = page;
+        page->free_next = heap->free_pages;
+        heap->free_pages = page;
+
+        RUBY_DEBUG_LOG("page:%p freelist:%p", page, page->freelist);
+
+        asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
+        return true;
+    }
+    else {
+        asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
+        return false;
     }
-    asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
 }
 
 #if GC_ENABLE_INCREMENTAL_MARK
@@ -2030,57 +2038,6 @@ heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap) https://github.com/ruby/ruby/blob/trunk/gc.c#L2038
     }
 }
 
-static RVALUE *
-heap_get_freeobj_from_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap)
-{
-    struct heap_page *page;
-    RVALUE *p;
-
-    while (heap->free_pages == NULL) {
-	heap_prepare(objspace, heap);
-    }
-    page = heap->free_pages;
-    heap->free_pages = page->free_next;
-    heap->using_page = page;
-
-    GC_ASSERT(page->free_slots != 0);
-    asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
-    p = page->freelist;
-    page->freelist = NULL;
-    asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
-    page->free_slots = 0;
-    asan_unpoison_object((VALUE)p, true);
-    return p;
-}
-
-static inline VALUE
-heap_get_freeobj_head(rb_objspace_t *objspace, rb_heap_t *heap)
-{
-    RVALUE *p = heap->freelist;
-    if (LIKELY(p != NULL)) {
-        heap->freelist = p->as.free.next;
-    }
-    asan_unpoison_object((VALUE)p, true);
-    return (VALUE)p;
-}
-
-static inline VALUE
-heap_get_freeobj(rb_objspace_t *objspace, rb_heap_t *heap)
-{
-    RVALUE *p = heap->freelist;
-
-    while (1) {
-	if (LIKELY(p != NULL)) {
-            asan_unpoison_object((VALUE)p, true);
-	    heap->freelist = p->as.free.next;
-	    return (VALUE)p;
-	}
-	else {
-	    p = heap_get_freeobj_from_next_freepage(objspace, heap);
-	}
-    }
-}
-
 void
 rb_objspace_set_event_hook(const rb_event_flag_t event)
 {
@@ -2111,34 +2068,22 @@ gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb https://github.com/ruby/ruby/blob/trunk/gc.c#L2068
 } while (0)
 
 static inline VALUE
-newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, rb_objspace_t *objspace, VALUE obj)
+newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
 {
 #if !__has_feature(memory_sanitizer)
     GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
     GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
 #endif
-
-    /* OBJSETUP */
-    struct RVALUE buf = {
-        .as = {
-            .values =  {
-                .basic = {
-                    .flags = flags,
-                    .klass = klass,
-                },
-                .v1 = v1,
-                .v2 = v2,
-                .v3 = v3,
-            },
-        },
-    };
-    MEMCPY(RANY(obj), &buf, RVALUE, 1);
+    RVALUE *p = RANY(obj);
+    p->as.basic.flags = flags;
+    *((VALUE *)&p->as.basic.klass) = klass;
 
 #if RACTOR_CHECK_MODE
     rb_ractor_setup_belonging(obj);
 #endif
 
 #if RGENGC_CHECK_MODE
+    p->as.values.v1 = p->as.values.v2 = p->as.values.v3 = 0;
     GC_ASSERT(RVALUE_MARKED(obj) == FALSE);
     GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
     GC_ASSERT(RVALUE_OLD_P(obj) == FALSE);
@@ -2154,9 +2099,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote https://github.com/ruby/ruby/blob/trunk/gc.c#L2099
 #endif
 
     if (UNLIKELY(wb_protected == FALSE)) {
+        ASSERT_vm_locking();
         MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
     }
 
+    // TODO: make it atomic, or ractor local
+    objspace->total_allocated_objects++;
+
 #if RGENGC_PROFILE
     if (wb_protected) {
         objspace->profile.total_generated_normal_object_count++;
@@ -2171,7 +2120,6 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote https://github.com/ruby/ruby/blob/trunk/gc.c#L2120
 #endif
     }
 #endif
-    objspace->total_allocated_objects++;
 
 #if GC_DEBUG
     RANY(obj)->file = rb_source_location_cstr(&RANY(obj)->line);
@@ -2199,88 +2147,179 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote https://github.com/ruby/ruby/blob/trunk/gc.c#L2147
     }
 #endif
     check_rvalue_consistency(obj);
+
+    // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, obj_type_name(obj));
     return obj;
 }
 
 static inline VALUE
-newobj_slowpath(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace, int wb_protected)
+ractor_cached_freeobj(rb_objspace_t *objspace, rb_ractor_t *cr)
+{
+    RVALUE *p = cr->newobj_cache.freelist;
+
+    if (p) {
+        VALUE obj = (VALUE)p;
+        cr->newobj_cache.freelist = p->as.free.next;
+        asan_unpoison_object(obj, true);
+        return obj;
+    }
+    else {
+        return Qfalse;
+    }
+}
+
+static struct heap_page *
+heap_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap)
+{
+    ASSERT_vm_locking();
+
+    struct heap_page *page;
+
+    while (heap->free_pages == NULL) {
+	heap_prepare(objspace, heap);
+    }
+    page = heap->free_pages;
+    heap->free_pages = page->free_next;
+
+    GC_ASSERT(page->free_slots != 0);
+    RUBY_DEBUG_LOG("page:%p freelist:%p cnt:%d", page, page->freelist, page->free_slots);
+
+    asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
+
+    return page;
+}
+
+static inline void
+ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr)
+{
+    ASSERT_vm_locking();
+    GC_ASSERT(cr->newobj_cache.freelist == NULL);
+
+    struct heap_page *page = heap_next_freepage(objspace, heap_eden);
+
+    cr->newobj_cache.using_page = page;
+    cr->newobj_cache.freelist = page->freelist;
+    page->free_slots = 0;
+    page->freelist = NULL;
+
+    GC_ASSERT(RB_TYPE_P((VALUE)cr->newobj_cache.freelist, T_NONE));
+}
+
+ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected));
+
+static inline VALUE
+newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected)
 {
     VALUE obj;
+    unsigned int lev;
 
-    if (UNLIKELY(during_gc || ruby_gc_stressful)) {
-        if (during_gc) {
-            dont_gc_on();
-            during_gc = 0;
-            rb_bug("object allocation during garbage collection phase");
-        }
+    RB_VM_LOCK_ENTER_CR_LEV(cr, &lev);
+    {
+        if (UNLIKELY(during_gc || ruby_gc_stressful)) {
+            if (during_gc) {
+                dont_gc_on();
+                during_gc = 0;
+                rb_bug("object allocation during garbage collection phase");
+            }
 
-        if (ruby_gc_stressful) {
-            if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
-                rb_memerror();
+            if (ruby_gc_stressful) {
+                if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
+                    rb_memerror();
+                }
             }
         }
-    }
 
-    obj = heap_get_freeobj(objspace, heap_eden);
+        // allocate new slot
+        while ((obj = ractor_cached_freeobj(objspace, cr)) == 0) {
+            ractor_cache_slots(objspace, cr);
+        }
+        GC_ASSERT(obj != 0);
+        newobj_init(klass, flags, wb_protected, objspace, obj);
+        gc_event_hook(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj);
+    }
+    RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev);
 
-    newobj_init(klass, flags, v1, v2, v3, wb_protected, objspace, obj);
-    gc_event_hook(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj);
     return obj;
 }
 
-NOINLIN (... truncated)

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/