ruby-changes:31506
From: tmm1 <ko1@a...>
Date: Sat, 9 Nov 2013 02:07:03 +0900 (JST)
Subject: [ruby-changes:31506] tmm1:r43585 (trunk): * ext/objspace/object_tracing.c: Add experimental methods to dump
tmm1 2013-11-09 02:06:55 +0900 (Sat, 09 Nov 2013) New Revision: 43585 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=43585 Log: * ext/objspace/object_tracing.c: Add experimental methods to dump objectspace as json: ObjectSpace.dump_all and ObjectSpace.dump(obj). These methods are useful for debugging reference leaks and memory growth in large ruby applications. [Bug #9026] [ruby-core:57893] [Fixes GH-423] Added files: trunk/ext/objspace/objspace.h trunk/ext/objspace/objspace_dump.c Modified files: trunk/ChangeLog trunk/ext/objspace/depend trunk/ext/objspace/object_tracing.c trunk/ext/objspace/objspace.c trunk/test/objspace/test_objspace.rb Index: ChangeLog =================================================================== --- ChangeLog (revision 43584) +++ ChangeLog (revision 43585) @@ -1,3 +1,12 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1 +Sat Nov 9 01:59:18 2013 Aman Gupta <ruby@t...> + + * ext/objspace/object_tracing.c: Add experimental methods to + dump objectspace as json: ObjectSpace.dump_all and + ObjectSpace.dump(obj). These methods are useful for debugging + reference leaks and memory growth in large ruby applications. + [Bug #9026] [ruby-core:57893] [Fixes GH-423] + * test/objspace/test_objspace.rb: tests for above. + Sat Nov 9 00:26:50 2013 Nobuyoshi Nakada <nobu@r...> * file.c (GetLastError): already defined in windows.h on nowadays Index: ext/objspace/depend =================================================================== --- ext/objspace/depend (revision 43584) +++ ext/objspace/depend (revision 43585) @@ -9,3 +9,6 @@ objspace.o: $(HDRS) $(ruby_headers) \ https://github.com/ruby/ruby/blob/trunk/ext/objspace/depend#L9 $(top_srcdir)/regint.h $(top_srcdir)/internal.h gc_hook.o: $(HDRS) $(ruby_headers) $(hdrdir)/ruby/debug.h object_tracing.o: $(HDRS) $(ruby_headers) $(hdrdir)/ruby/debug.h +objspace_dump.o: $(HDRS) $(ruby_headers) $(hdrdir)/ruby/debug.h \ + $(hdrdir)/ruby/encoding.h $(hdrdir)/ruby/io.h \ + $(top_srcdir)/node.h $(top_srcdir)/vm_core.h $(top_srcdir)/gc.h Index: ext/objspace/objspace.c =================================================================== --- ext/objspace/objspace.c (revision 43584) +++ ext/objspace/objspace.c (revision 43585) @@ -719,6 +719,7 @@ reachable_objects_from_root(VALUE self) https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace.c#L719 void Init_object_tracing(VALUE rb_mObjSpace); void Init_gc_hook(VALUE rb_mObjSpace); +void Init_objspace_dump(VALUE rb_mObjSpace); /* * Document-module: ObjectSpace @@ -770,4 +771,5 @@ Init_objspace(void) https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace.c#L771 Init_object_tracing(rb_mObjSpace); Init_gc_hook(rb_mObjSpace); + Init_objspace_dump(rb_mObjSpace); } Index: ext/objspace/objspace.h =================================================================== --- ext/objspace/objspace.h (revision 0) +++ ext/objspace/objspace.h (revision 43585) @@ -0,0 +1,20 @@ https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace.h#L1 +#ifndef OBJSPACE_H +#define OBJSPACE_H 1 + +/* object_tracing.c */ +struct allocation_info { + /* all of information don't need marking. */ + int living; + VALUE flags; + VALUE klass; + + /* allocation info */ + const char *path; + unsigned long line; + const char *class_path; + VALUE mid; + size_t generation; +}; +struct allocation_info *objspace_lookup_allocation_info(VALUE obj); + +#endif Index: ext/objspace/objspace_dump.c =================================================================== --- ext/objspace/objspace_dump.c (revision 0) +++ ext/objspace/objspace_dump.c (revision 43585) @@ -0,0 +1,415 @@ https://github.com/ruby/ruby/blob/trunk/ext/objspace/objspace_dump.c#L1 +/********************************************************************** + + objspace_dump.c - Heap dumping ObjectSpace extender for MRI. + + $Author$ + created at: Sat Oct 11 10:11:00 2013 + + NOTE: This extension library is not expected to exist except C Ruby. + + All the files in this distribution are covered under the Ruby's + license (see the file COPYING). + +**********************************************************************/ + +#include "ruby/ruby.h" +#include "ruby/debug.h" +#include "ruby/encoding.h" +#include "ruby/io.h" +#include "gc.h" +#include "node.h" +#include "vm_core.h" +#include "objspace.h" + +/* from string.c */ +#define STR_NOEMBED FL_USER1 +#define STR_SHARED FL_USER2 /* = ELTS_SHARED */ +#define STR_ASSOC FL_USER3 +#define STR_SHARED_P(s) FL_ALL((s), STR_NOEMBED|ELTS_SHARED) +#define STR_ASSOC_P(s) FL_ALL((s), STR_NOEMBED|STR_ASSOC) +#define STR_NOCAPA (STR_NOEMBED|ELTS_SHARED|STR_ASSOC) +#define STR_NOCAPA_P(s) (FL_TEST((s),STR_NOEMBED) && FL_ANY((s),ELTS_SHARED|STR_ASSOC)) +#define STR_EMBED_P(str) (!FL_TEST((str), STR_NOEMBED)) +#define is_ascii_string(str) (rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT) +#define is_broken_string(str) (rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN) +/* from hash.c */ +#define HASH_PROC_DEFAULT FL_USER2 + +static VALUE sym_output, sym_stdout, sym_string, sym_file; + +struct dump_config { + VALUE type; + FILE *stream; + VALUE string; + int roots; + const char *root_category; + VALUE cur_obj; + VALUE cur_obj_klass; + size_t cur_obj_references; +}; + +static void +dump_append(struct dump_config *dc, const char *format, ...) +{ + va_list vl; + va_start(vl, format); + + if (dc->stream) { + vfprintf(dc->stream, format, vl); + fflush(dc->stream); + } + else if (dc->string) + rb_str_vcatf(dc->string, format, vl); + + va_end(vl); +} + +static void +dump_append_string_value(struct dump_config *dc, VALUE obj) +{ + int i; + char c, *value; + + dump_append(dc, "\""); + for (i = 0, value = RSTRING_PTR(obj); i < RSTRING_LEN(obj); i++) { + switch ((c = value[i])) { + case '\\': + case '"': + dump_append(dc, "\\%c", c); + break; + case '\0': + dump_append(dc, "\\u0000"); + break; + case '\b': + dump_append(dc, "\\b"); + break; + case '\t': + dump_append(dc, "\\t"); + break; + case '\f': + dump_append(dc, "\\f"); + break; + case '\n': + dump_append(dc, "\\n"); + break; + case '\r': + dump_append(dc, "\\r"); + break; + default: + dump_append(dc, "%c", c); + } + } + dump_append(dc, "\""); +} + +static inline const char * +obj_type(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { +#define CASE_TYPE(type) case T_##type: return #type; break + CASE_TYPE(NONE); + CASE_TYPE(NIL); + CASE_TYPE(OBJECT); + CASE_TYPE(CLASS); + CASE_TYPE(ICLASS); + CASE_TYPE(MODULE); + CASE_TYPE(FLOAT); + CASE_TYPE(STRING); + CASE_TYPE(REGEXP); + CASE_TYPE(ARRAY); + CASE_TYPE(HASH); + CASE_TYPE(STRUCT); + CASE_TYPE(BIGNUM); + CASE_TYPE(FILE); + CASE_TYPE(FIXNUM); + CASE_TYPE(TRUE); + CASE_TYPE(FALSE); + CASE_TYPE(DATA); + CASE_TYPE(MATCH); + CASE_TYPE(SYMBOL); + CASE_TYPE(RATIONAL); + CASE_TYPE(COMPLEX); + CASE_TYPE(UNDEF); + CASE_TYPE(NODE); + CASE_TYPE(ZOMBIE); +#undef CASE_TYPE + } + return "UNKNOWN"; +} + +static void +reachable_object_i(VALUE ref, void *data) +{ + struct dump_config *dc = (struct dump_config *)data; + + if (dc->cur_obj_klass == ref) + return; + + if (dc->cur_obj_references == 0) + dump_append(dc, ", \"references\":[\"%p\"", (void *)ref); + else + dump_append(dc, ", \"%p\"", (void *)ref); + + dc->cur_obj_references++; +} + +static void +dump_object(VALUE obj, struct dump_config *dc) +{ + int enc; + long length; + size_t memsize; + struct allocation_info *ainfo; + rb_io_t *fptr; + + dc->cur_obj = obj; + dc->cur_obj_references = 0; + dc->cur_obj_klass = BUILTIN_TYPE(obj) == T_NODE ? 0 : RBASIC_CLASS(obj); + + if (dc->cur_obj == dc->string) + return; + + dump_append(dc, "{\"address\":\"%p\", \"type\":\"%s\"", (void *)obj, obj_type(obj)); + + if (dc->cur_obj_klass) + dump_append(dc, ", \"class\":\"%p\"", (void *)dc->cur_obj_klass); + if (rb_obj_frozen_p(obj)) + dump_append(dc, ", \"frozen\":true"); + + switch (BUILTIN_TYPE(obj)) { + case T_NODE: + dump_append(dc, ", \"node_type\":\"%s\"", ruby_node_name(nd_type(obj))); + break; + + case T_STRING: + if (STR_EMBED_P(obj)) + dump_append(dc, ", \"embedded\":true"); + if (STR_ASSOC_P(obj)) + dump_append(dc, ", \"associated\":true"); + if (is_broken_string(obj)) + dump_append(dc, ", \"broken\":true"); + if (STR_SHARED_P(obj)) + dump_append(dc, ", \"shared\":true"); + else { + dump_append(dc, ", \"bytesize\":%ld", RSTRING_LEN(obj)); + if (!STR_EMBED_P(obj) && !STR_NOCAPA_P(obj) && rb_str_capacity(obj) != RSTRING_LEN(obj)) + dump_append(dc, ", \"capacity\":%ld", rb_str_capacity(obj)); + + if (is_ascii_string(obj)) { + dump_append(dc, ", \"value\":"); + dump_append_string_value(dc, obj); + } + } + + if (!ENCODING_IS_ASCII8BIT(obj)) + dump_append(dc, ", \"encoding\":\"%s\"", rb_enc_name(rb_enc_from_index(ENCODING_GET(obj)))); + break; + + case T_HASH: + dump_append(dc, ", \"size\":%ld", RHASH_SIZE(obj)); + if (FL_TEST(obj, HASH_PROC_DEFAULT)) + dump_append(dc, ", \"default\":\"%p\"", (void *)RHASH_IFNONE(obj)); + break; + + case T_ARRAY: + dump_append(dc, ", \"length\":%ld", RARRAY_LEN(obj)); + if (RARRAY_LEN(obj) > 0 && FL_TEST(obj, ELTS_SHARED)) + dump_append(dc, ", \"shared\":true"); + if (RARRAY_LEN(obj) > 0 && FL_TEST(obj, RARRAY_EMBED_FLAG)) + dump_append(dc, ", \"embedded\":true"); + break; + + case T_CLASS: + case T_MODULE: + if (dc->cur_obj_klass) + dump_append(dc, ", \"name\":\"%s\"", rb_class2name(obj)); + break; + + case T_DATA: + if (RTYPEDDATA_P(obj)) + dump_append(dc, ", \"struct\":\"%s\"", RTYPEDDATA_TYPE(obj)->wrap_struct_name); + break; + + case T_FLOAT: + dump_append(dc, ", \"value\":\"%g\"", RFLOAT_VALUE(obj)); + break; + + case T_OBJECT: + dump_append(dc, ", \"ivars\":%ld", ROBJECT_NUMIV(obj)); + break; + + case T_FILE: + fptr = RFILE(obj)->fptr; + dump_append(dc, ", \"fd\":%d", fptr->fd); + break; + + case T_ZOMBIE: + dump_append(dc, "}\n"); + return; + } + + rb_objspace_reachable_objects_from(obj, reachable_object_i, dc); + if (dc->cur_obj_references > 0) + dump_append(dc, "]"); + + if ((ainfo = objspace_lookup_allocation_info(obj))) { + dump_append(dc, ", \"file\":\"%s\", \"line\":%lu", ainfo->path, ainfo->line); + if (RTEST(ainfo->mid)) + dump_append(dc, ", \"method\":\"%s\"", rb_id2name(SYM2ID(ainfo->mid))); + dump_append(dc, ", \"generation\":%zu", ainfo->generation); + } + + if ((memsize = rb_obj_memsize_of(obj)) > 0) + dump_append(dc, ", \"memsize\":%zu", memsize); + + dump_append(dc, "}\n"); +} + +static int +heap_i(void *vstart, void *vend, size_t stride, void *data) +{ + VALUE v = (VALUE)vstart; + for (; v != (VALUE)vend; v += stride) { + if (RBASIC(v)->flags) + dump_object(v, data); + } + return 0; +} + +static void +root_obj_i(const char *category, VALUE obj, void *data) +{ + struct dump_config *dc = (struct dump_config *)data; + + if (dc->root_category != NULL && category != dc->root_category) + dump_append(dc, "]}\n"); + if (dc->root_category == NULL || category != dc->root_category) + dump_append(dc, "{\"type\":\"ROOT\", \"root\":\"%s\", \"references\":[\"%p\"", category, (void *)obj); + else + dump_append(dc, ", \"%p\"", (void *)obj); + + dc->root_category = category; + dc->roots++; +} + +/* + * call-seq: + * ObjectSpace.dump(obj[, output: :string]) # => "{ ... }" + * ObjectSpace.dump(obj, output: :file) # => "/tmp/rubyobj000000" + * ObjectSpace.dump(obj, output: :stdout) # => nil + * + * Dump the contents of a ruby object as JSON. + * + * This method is only expected to work with C Ruby. + * This is an experimental method and is subject to change. + * In particular, the function signature and output format are + * not guaranteed to be compatible in future versions of ruby. + */ + +static VALUE +objspace_dump(int argc, VALUE *argv, VALUE os) +{ + int fd; + char filename[] = "/tmp/rubyobjXXXXXX"; + VALUE obj = Qnil, opts = Qnil, output; + struct dump_config dc = {0,}; + + rb_scan_args(argc, argv, "1:", &obj, &opts); + + if (RTEST(opts)) + output = rb_hash_aref(opts, sym_output); + + if (output == sym_stdout) + dc.stream = stdout; + else if (output == sym_file) { + fd = mkstemp(filename); + if (fd == -1) rb_sys_fail_path(rb_str_new_cstr(filename)); + dc.stream = fdopen(fd, "w"); + } + else { + output = sym_string; + dc.string = rb_str_new2(""); + } + + dump_object(obj, &dc); + + if (output == sym_string) + return dc.string; + else if (output == sym_file) { + fclose(dc.stream); + return rb_str_new2(filename); + } + else + return Qnil; +} + +/* + * call-seq: + * ObjectSpace.dump_all([output: :file]) # => "/tmp/rubyheap000000" + * ObjectSpace.dump_all(output: :stdout) # => nil + * ObjectSpace.dump_all(output: :string) # => "{...}\n{...}\n..." + * + * Dump the contents of the ruby heap as JSON. + * + * This method is only expected to work with C Ruby. + * This is an experimental method and is subject to change. + * In particular, the function signature and output format are + * not guaranteed to be compatible in future versions of ruby. + */ + +static VALUE +objspace_dump_all(int argc, VALUE *argv, VALUE os) +{ + int fd; + char filename[] = "/tmp/rubyheapXXXXXX"; + VALUE opts = Qnil, output; + struct dump_config dc = {0,}; + + rb_scan_args(argc, argv, "0:", &opts); + + if (RTEST(opts)) + output = rb_hash_aref(opts, sym_output); + + if (output == sym_string) + dc.string = rb_str_new2(""); + else if (output == sym_stdout) + dc.stream = stdout; + else { + output = sym_file; + fd = mkstemp(filename); + if (fd == -1) rb_sys_fail_path(rb_str_new_cstr(filename)); + dc.stream = fdopen(fd, "w"); + } + + /* dump roots */ + rb_objspace_reachable_objects_from_root(root_obj_i, &dc); + if (dc.roots) dump_append(&dc, "]}\n"); + + /* dump all objects */ + rb_objspace_each_objects(heap_i, &dc); + + if (output == sym_string) + return dc.string; + else if (output == sym_file) { + fclose(dc.stream); + return rb_str_new2(filename); + } + else + return Qnil; +} + +void +Init_objspace_dump(VALUE rb_mObjSpace) +{ +#if 0 + rb_mObjSpace = rb_define_module("ObjectSpace"); /* let rdoc know */ +#endif + + rb_define_module_function(rb_mObjSpace, "dump", objspace_dump, -1); + rb_define_module_function(rb_mObjSpace, "dump_all", objspace_dump_all, -1); + + sym_output = ID2SYM(rb_intern("output")); + sym_stdout = ID2SYM(rb_intern("stdout")); + sym_string = ID2SYM(rb_intern("string")); + sym_file = ID2SYM(rb_intern("file")); +} Index: ext/objspace/object_tracing.c =================================================================== --- ext/objspace/object_tracing.c (revision 43584) +++ ext/objspace/object_tracing.c (revision 43585) @@ -15,6 +15,7 @@ https://github.com/ruby/ruby/blob/trunk/ext/objspace/object_tracing.c#L15 #include "ruby/ruby.h" #include "ruby/debug.h" +#include "objspace.h" size_t rb_gc_count(void); /* from gc.c */ @@ -28,20 +29,6 @@ struct traceobj_arg { https://github.com/ruby/ruby/blob/trunk/ext/objspace/object_tracing.c#L29 struct traceobj_arg *prev_traceobj_arg; }; -/* all of information don't need marking. */ -struct allocation_info { - int living; - VALUE flags; - VALUE klass; - - /* allocation info */ - const char *path; - unsigned long line; - const char *class_path; - VALUE mid; - size_t generation; -}; - static const char * make_unique_str(st_table *tbl, const char *str, long len) { @@ -341,6 +328,12 @@ lookup_allocation_info(VALUE obj) https://github.com/ruby/ruby/blob/trunk/ext/objspace/object_tracing.c#L328 return NULL; } +struct allocation_info * +objspace_lookup_allocation_info(VALUE obj) +{ + return lookup_allocation_info(obj); +} + /* * call-seq: allocation_sourcefile(object) -> string * Index: test/objspace/test_objspace.rb =================================================================== --- test/objspace/test_objspace.rb (revision 43584) +++ test/objspace/test_objspace.rb (revision 43585) @@ -203,4 +203,31 @@ class TestObjSpace < Test::Unit::TestCas https://github.com/ruby/ruby/blob/trunk/test/objspace/test_objspace.rb#L203 end; end end + + def test_dump + info = nil + ObjectSpace.trace_object_allocations do + str = "hello world" + info = ObjectSpace.dump(str) + end + + assert_match /"type":"STRING"/, info + assert_match /"embedded":true, "bytesize":11, "value":"hello world", "encoding":"UTF-8"/, info + assert_match /"file":"#{Regexp.escape __FILE__}", "line":#{__LINE__-6}/, info + assert_match /"method":"test_dump"/, info + end + + def test_dump_all + entry = /"value":"this is a test string", "encoding":"UTF-8", "file":"-", "line":4, "method":"dump_my_heap_please"/ + assert_in_out_err(%w[-robjspace], <<-'end;', entry) + def dump_my_heap_please + ObjectSpace.trace_object_allocations_start + GC.start + "this is a test string".force_encoding("UTF-8") + ObjectSpace.dump_all(output: :stdout) + end + + dump_my_heap_please + end; + end end -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/