tenderlove	2016-09-17 02:44:45 +0900 (Sat, 17 Sep 2016)

  New Revision: 56174

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=56174

  Log:
    Use JSON lines format for full heap dumps.
    
    This commit changes full heap dumps back to using JSON lines format
    (http://jsonlines.org) so that we can process very large heaps without
    loading the entire heap in to memory at once.

  Modified files:
    trunk/ChangeLog
    trunk/ext/objspace/objspace_dump.c
    trunk/test/objspace/test_objspace.rb
Index: test/objspace/test_objspace.rb
===================================================================
--- test/objspace/test_objspace.rb	(revision 56173)
+++ test/objspace/test_objspace.rb	(revision 56174)
@@ -316,7 +316,7 @@ class TestObjSpace < Test::Unit::TestCas https://github.com/ruby/ruby/blob/trunk/test/objspace/test_objspace.rb#L316
       assert_ruby_status(args, "#{<<~"begin;"}\n#{<<~"end;"}")
       begin;
         IO.popen(ARGV) do |f|
-          JSON.load(f)
+          f.each_line.map { |x| JSON.load(x) }
         end
       end;
     end
Index: ChangeLog
===================================================================
--- ChangeLog	(revision 56173)
+++ ChangeLog	(revision 56174)
@@ -1,3 +1,10 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1
+Sat Sep 17 02:40:52 2016  Aaron Patterson <tenderlove@r...>
+
+	* ext/objspace/objspace_dump.c: Fix stream processing support for heap
+	  dumps. Full heap dumps should use JSON lines (http://jsonlines.org)
+	  so that we can process very large heaps without loading the entire
+	  heap dump in to memory at once.
+
 Fri Sep 16 22:10:31 2016  Naohisa Goto  <ngotogenome@g...>
 
 	* ext/openssl/ossl_ssl.c (ssl_npn_select_cb_common): Fix compile error
Index: ext/objspace/objspace_dump.c
===================================================================
--- ext/objspace/objspace_dump.c	(revision 56173)
+++ ext/objspace/objspace_dump.c	(revision 56174)
@@ -191,7 +191,7 @@ dump_append_string_content(struct dump_c https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L191
 }
 
 static void
-dump_object(VALUE obj, struct dump_config *dc, int part)
+dump_object(VALUE obj, struct dump_config *dc)
 {
     size_t memsize;
     struct allocation_info *ainfo;
@@ -211,11 +211,7 @@ dump_object(VALUE obj, struct dump_confi https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L211
     if (dc->cur_obj == dc->string)
 	return;
 
-    if (part)
-	dump_append(dc, "\"%p\":{", (void *)obj);
-    else
-	dump_append(dc, "{\"address\":\"%p\", ", (void *)obj);
-    dump_append(dc, "\"type\":\"%s\"", obj_type(obj));
+    dump_append(dc, "{\"address\":\"%p\", \"type\":\"%s\"", (void *)obj, obj_type(obj));
 
     if (dc->cur_obj_klass)
 	dump_append(dc, ", \"class\":\"%p\"", (void *)dc->cur_obj_klass);
@@ -287,8 +283,7 @@ dump_object(VALUE obj, struct dump_confi https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L283
 	break;
 
       case T_ZOMBIE:
-	dump_append(dc, "}");
-	dc->roots++;
+	dump_append(dc, "}\n");
 	return;
     }
 
@@ -317,8 +312,7 @@ dump_object(VALUE obj, struct dump_confi https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L312
 	dump_append(dc, "}");
     }
 
-    dump_append(dc, "}");
-    dc->roots++;
+    dump_append(dc, "}\n");
 }
 
 static int
@@ -327,10 +321,8 @@ heap_i(void *vstart, void *vend, size_t https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L321
     struct dump_config *dc = (struct dump_config *)data;
     VALUE v = (VALUE)vstart;
     for (; v != (VALUE)vend; v += stride) {
-	if (RBASIC(v)->flags && v != dc->string) {
-	    if (dc->roots++) dump_append(dc, ",\n");
-	    dump_object(v, dc, 1);
-	}
+	if (RBASIC(v)->flags)
+	    dump_object(v, data);
     }
     return 0;
 }
@@ -341,11 +333,9 @@ root_obj_i(const char *category, VALUE o https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L333
     struct dump_config *dc = (struct dump_config *)data;
 
     if (dc->root_category != NULL && category != dc->root_category)
-	dump_append(dc, "]},\n");
-    if (dc->root_category == NULL || category != dc->root_category) {
-	dump_append(dc, "\"%p\":", (void *)obj);
+	dump_append(dc, "]}\n");
+    if (dc->root_category == NULL || category != dc->root_category)
 	dump_append(dc, "{\"type\":\"ROOT\", \"root\":\"%s\", \"references\":[\"%p\"", category, (void *)obj);
-    }
     else
 	dump_append(dc, ", \"%p\"", (void *)obj);
 
@@ -429,8 +419,7 @@ objspace_dump(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L419
 
     output = dump_output(&dc, opts, sym_string, filename);
 
-    dump_object(obj, &dc, 0);
-    if (dc.roots) dump_append(&dc, "\n");
+    dump_object(obj, &dc);
 
     return dump_result(&dc, output);
 }
@@ -462,14 +451,12 @@ objspace_dump_all(int argc, VALUE *argv, https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L451
 
     output = dump_output(&dc, opts, sym_file, filename);
 
-    dump_append(&dc, "{\n");
     /* dump roots */
     rb_objspace_reachable_objects_from_root(root_obj_i, &dc);
-    if (dc.roots) dump_append(&dc, "]}");
+    if (dc.roots) dump_append(&dc, "]}\n");
 
     /* dump all objects */
     rb_objspace_each_objects(heap_i, &dc);
-    dump_append(&dc, "\n}");
 
     return dump_result(&dc, output);
 }

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/