ruby-changes:26427
From: ko1 <ko1@a...>
Date: Thu, 20 Dec 2012 07:29:47 +0900 (JST)
Subject: [ruby-changes:26427] ko1:r38478 (trunk): * vm.c: support variable VM/Machine stack sizes.
ko1 2012-12-20 07:29:18 +0900 (Thu, 20 Dec 2012) New Revision: 38478 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=38478 Log: * vm.c: support variable VM/Machine stack sizes. Specified by the following environment variaables: - RUBY_THREAD_VM_STACK_SIZE: vm stack size used at thread creation. default: 128KB (32bit CPU) or 256KB (64bit CPU). - RUBY_THREAD_MACHINE_STACK_SIZE: machine stack size used at thread creation. default: 512KB or 1024KB. - RUBY_FIBER_VM_STACK_SIZE: vm stack size used at fiber creation. default: 64KB or 128KB. - RUBY_FIBER_MACHINE_STACK_SIZE: machine stack size used at fiber creation. default: 256KB or 256KB. This values are specified at launched timing. You can not change these values at running time. Environ variables are only *hints* because: - They are aligned to 4KB. - They have minimum values (depend on OSs). - Machine stack settings are ignored by some OSs. Default values especially fiber stack sizes are increased. This change affect Fiber's behavior: (1) You can run more complex program on a Fiber. (2) You can not make many (thousands) Fibers because of lack of address space (on 32bit CPU). If (2) bothers you, (a) Use 64bit CPU with big memory, or (b) Specify RUBY_FIBER_(VM|MACHINE)_STACK_SIZE correctly. You need to choose correct stack size carefully. These values are completely rely on systems (OS/compiler and so on). * vm_core.h (rb_vm_t::default_params): add to record above settings. * vm.c (RubyVM::DEFAULT_PARAMS): add new constant to see above setting. * thread_pthread.c: support RUBY_THREAD_MACHINE_STACK_SIZE. * cont.c: support RUBY_FIBER_(VM|MACHINE)_STACK_SIZE. * test/ruby/test_fiber.rb: add tests for above. * test/ruby/test_thread.rb: ditto. Modified files: trunk/ChangeLog trunk/cont.c trunk/test/ruby/test_fiber.rb trunk/test/ruby/test_thread.rb trunk/thread_pthread.c trunk/vm.c trunk/vm_core.h Index: ChangeLog =================================================================== --- ChangeLog (revision 38477) +++ ChangeLog (revision 38478) @@ -1,3 +1,45 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1 +Thu Dec 20 06:59:52 2012 Koichi Sasada <ko1@a...> + + * vm.c: support variable VM/Machine stack sizes. + Specified by the following environment variaables: + - RUBY_THREAD_VM_STACK_SIZE: vm stack size used at thread creation. + default: 128KB (32bit CPU) or 256KB (64bit CPU). + - RUBY_THREAD_MACHINE_STACK_SIZE: machine stack size used at thread + creation. default: 512KB or 1024KB. + - RUBY_FIBER_VM_STACK_SIZE: vm stack size used at fiber creation. + default: 64KB or 128KB. + - RUBY_FIBER_MACHINE_STACK_SIZE: machine stack size used at fiber + creation. default: 256KB or 256KB. + This values are specified at launched timing. You can not change + these values at running time. + Environ variables are only *hints* because: + - They are aligned to 4KB. + - They have minimum values (depend on OSs). + - Machine stack settings are ignored by some OSs. + Default values especially fiber stack sizes are increased. + This change affect Fiber's behavior: + (1) You can run more complex program on a Fiber. + (2) You can not make many (thousands) Fibers because of + lack of address space (on 32bit CPU). + If (2) bothers you, + (a) Use 64bit CPU with big memory, or + (b) Specify RUBY_FIBER_(VM|MACHINE)_STACK_SIZE correctly. + You need to choose correct stack size carefully. These values + are completely rely on systems (OS/compiler and so on). + + * vm_core.h (rb_vm_t::default_params): add to record above settings. + + * vm.c (RubyVM::DEFAULT_PARAMS): add new constant to see + above setting. + + * thread_pthread.c: support RUBY_THREAD_MACHINE_STACK_SIZE. + + * cont.c: support RUBY_FIBER_(VM|MACHINE)_STACK_SIZE. + + * test/ruby/test_fiber.rb: add tests for above. + + * test/ruby/test_thread.rb: ditto. + Thu Dec 20 06:25:44 2012 Koichi Sasada <ko1@a...> * test/ruby/test_fiber.rb: remove a strange single quote character. Index: thread_pthread.c =================================================================== --- thread_pthread.c (revision 38477) +++ thread_pthread.c (revision 38478) @@ -584,26 +584,6 @@ static struct { https://github.com/ruby/ruby/blob/trunk/thread_pthread.c#L584 #endif } native_main_thread; - -enum { -#ifdef __SYMBIAN32__ - RUBY_STACK_MIN_LIMIT = 64 * 1024, /* 64KB: Let's be slightly more frugal on mobile platform */ -#else - RUBY_STACK_MIN_LIMIT = 512 * 1024, /* 512KB */ -#endif - RUBY_STACK_SPACE_LIMIT = 1024 * 1024, - RUBY_STACK_SPACE_RATIO = 5 -}; -#ifdef PTHREAD_STACK_MIN -#define RUBY_STACK_MIN ((RUBY_STACK_MIN_LIMIT < PTHREAD_STACK_MIN) ? \ - PTHREAD_STACK_MIN * 2 : RUBY_STACK_MIN_LIMIT) -#else -#define RUBY_STACK_MIN (RUBY_STACK_MIN_LIMIT) -#endif -#define RUBY_STACK_MIN_SPACE RUBY_STACK_MIN/RUBY_STACK_SPACE_RATIO -#define RUBY_STACK_SPACE ((RUBY_STACK_MIN_SPACE > RUBY_STACK_SPACE_LIMIT) ? \ - RUBY_STACK_SPACE_LIMIT : RUBY_STACK_MIN_SPACE) - #ifdef STACK_END_ADDRESS extern void *STACK_END_ADDRESS; #endif @@ -830,6 +810,23 @@ use_cached_thread(rb_thread_t *th) https://github.com/ruby/ruby/blob/trunk/thread_pthread.c#L810 return result; } +enum { + RUBY_STACK_SPACE_LIMIT = 1024 * 1024, /* 1024KB */ + RUBY_STACK_SPACE_RATIO = 5 +}; + +static size_t +space_size(size_t stack_size) +{ + size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO; + if (space_size > RUBY_STACK_SPACE_LIMIT) { + return RUBY_STACK_SPACE_LIMIT; + } + else { + return space_size; + } +} + static int native_thread_create(rb_thread_t *th) { @@ -840,8 +837,8 @@ native_thread_create(rb_thread_t *th) https://github.com/ruby/ruby/blob/trunk/thread_pthread.c#L837 } else { pthread_attr_t attr; - const size_t stack_size = RUBY_STACK_MIN; - const size_t space = RUBY_STACK_SPACE; + const size_t stack_size = th->vm->default_params.thread_machine_stack_size; + const size_t space = space_size(stack_size); th->machine_stack_maxsize = stack_size - space; #ifdef __ia64 Index: vm_core.h =================================================================== --- vm_core.h (revision 38477) +++ vm_core.h (revision 38478) @@ -393,8 +393,30 @@ typedef struct rb_vm_struct { https://github.com/ruby/ruby/blob/trunk/vm_core.h#L393 struct RArray at_exit; VALUE *defined_strings; + + /* params */ + struct { /* size in byte */ + size_t thread_vm_stack_size; + size_t thread_machine_stack_size; + size_t fiber_vm_stack_size; + size_t fiber_machine_stack_size; + } default_params; } rb_vm_t; +/* default values */ + +#define RUBY_VM_SIZE_ALIGN 4096 + +#define RUBY_VM_THREAD_VM_STACK_SIZE ( 32 * 1024 * sizeof(VALUE)) /* 128 KB or 256 KB */ +#define RUBY_VM_THREAD_VM_STACK_SIZE_MIN ( 2 * 1024 * sizeof(VALUE)) /* 8 KB or 16 KB */ +#define RUBY_VM_THREAD_MACHINE_STACK_SIZE ( 128 * 1024 * sizeof(VALUE)) /* 512 KB or 1024 KB */ +#define RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN ( 16 * 1024 * sizeof(VALUE)) /* 64 KB or 128 KB */ + +#define RUBY_VM_FIBER_VM_STACK_SIZE ( 16 * 1024 * sizeof(VALUE)) /* 64 KB or 128 KB */ +#define RUBY_VM_FIBER_VM_STACK_SIZE_MIN ( 2 * 1024 * sizeof(VALUE)) /* 8 KB or 16 KB */ +#define RUBY_VM_FIBER_MACHINE_STACK_SIZE ( 64 * 1024 * sizeof(VALUE)) /* 256 KB or 512 KB */ +#define RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN ( 16 * 1024 * sizeof(VALUE)) /* 64 KB or 128 KB */ + #ifndef VM_DEBUG_BP_CHECK #define VM_DEBUG_BP_CHECK 1 #endif @@ -469,7 +491,7 @@ typedef struct rb_thread_struct { https://github.com/ruby/ruby/blob/trunk/vm_core.h#L491 /* execution information */ VALUE *stack; /* must free, must mark */ - unsigned long stack_size; + size_t stack_size; /* size in word (byte size / sizeof(VALUE)) */ rb_control_frame_t *cfp; int safe_level; int raised_flag; @@ -620,9 +642,6 @@ RUBY_EXTERN VALUE rb_mRubyVMFrozenCore; https://github.com/ruby/ruby/blob/trunk/vm_core.h#L642 #pragma GCC visibility pop #endif -/* each thread has this size stack : 128KB */ -#define RUBY_VM_THREAD_STACK_SIZE (128 * 1024) - #define GetProcPtr(obj, ptr) \ GetCoreDataFromValue((obj), rb_proc_t, (ptr)) Index: cont.c =================================================================== --- cont.c (revision 38477) +++ cont.c (revision 38478) @@ -47,12 +47,6 @@ https://github.com/ruby/ruby/blob/trunk/cont.c#L47 #define RB_PAGE_SIZE (pagesize) #define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1)) static long pagesize; - - #if SIZEOF_VOIDP==8 - #define FIBER_MACHINE_STACK_ALLOCATION_SIZE (0x20000) - #else - #define FIBER_MACHINE_STACK_ALLOCATION_SIZE (0x10000) - #endif #endif /*FIBER_USE_NATIVE*/ #define CAPTURE_JUST_VALID_VM_STACK 1 @@ -631,7 +625,7 @@ fiber_setcontext(rb_fiber_t *newfib, rb_ https://github.com/ruby/ruby/blob/trunk/cont.c#L625 rb_thread_t *th = GET_THREAD(), *sth = &newfib->cont.saved_thread; if (newfib->status != RUNNING) { - fiber_initialize_machine_stack_context(newfib, FIBER_MACHINE_STACK_ALLOCATION_SIZE); + fiber_initialize_machine_stack_context(newfib, th->vm->default_params.fiber_machine_stack_size); } /* restore thread context */ @@ -1002,8 +996,6 @@ rb_cont_call(int argc, VALUE *argv, VALU https://github.com/ruby/ruby/blob/trunk/cont.c#L996 * */ -#define FIBER_VM_STACK_SIZE (4 * 1024) - static const rb_data_type_t fiber_data_type = { "fiber", {fiber_mark, fiber_free, fiber_memsize,}, @@ -1054,7 +1046,7 @@ fiber_init(VALUE fibval, VALUE proc) https://github.com/ruby/ruby/blob/trunk/cont.c#L1046 fiber_link_join(fib); - th->stack_size = FIBER_VM_STACK_SIZE; + th->stack_size = th->vm->default_params.fiber_vm_stack_size / sizeof(VALUE); th->stack = ALLOC_N(VALUE, th->stack_size); th->cfp = (void *)(th->stack + th->stack_size); Index: vm.c =================================================================== --- vm.c (revision 38477) +++ vm.c (revision 38478) @@ -1613,6 +1613,82 @@ static const rb_data_type_t vm_data_type https://github.com/ruby/ruby/blob/trunk/vm.c#L1613 {rb_vm_mark, vm_free, vm_memsize,}, }; + +static VALUE +vm_default_params(void) +{ + rb_vm_t *vm = GET_VM(); + VALUE result = rb_hash_new(); +#define SET(name) rb_hash_aset(result, ID2SYM(rb_intern(#name)), SIZET2NUM(vm->default_params.name)); + SET(thread_vm_stack_size); + SET(thread_machine_stack_size); + SET(fiber_vm_stack_size); + SET(fiber_machine_stack_size); +#undef SET + rb_obj_freeze(result); + return result; +} + +static size_t +get_param(const char *name, size_t default_value, size_t min_value) +{ + const char *envval; + size_t result = default_value; + if ((envval = getenv(name)) != 0) { + long val = atol(envval); + if (val < (long)min_value) { + val = (long)min_value; + } + result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN); + } + if (0) fprintf(stderr, "%s: %d\n", name, result); /* debug print */ + + return result; +} + +static void +check_machine_stack_size(size_t *sizep) +{ + size_t size = *sizep; +#ifdef __SYMBIAN32__ + *sizep = 64 * 1024; /* 64KB: Let's be slightly more frugal on mobile platform */ +#endif + +#ifdef PTHREAD_STACK_MIN + if (size < PTHREAD_STACK_MIN) { + *sizep = PTHREAD_STACK_MIN * 2; + } +#endif +} + +static void +vm_default_params_setup(rb_vm_t *vm) +{ + vm->default_params.thread_vm_stack_size = + get_param("RUBY_THREAD_VM_STACK_SIZE", + RUBY_VM_THREAD_VM_STACK_SIZE, + RUBY_VM_THREAD_VM_STACK_SIZE_MIN); + + vm->default_params.thread_machine_stack_size = + get_param("RUBY_THREAD_MACHINE_STACK_SIZE", + RUBY_VM_THREAD_MACHINE_STACK_SIZE, + RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN); + + vm->default_params.fiber_vm_stack_size = + get_param("RUBY_FIBER_VM_STACK_SIZE", + RUBY_VM_FIBER_VM_STACK_SIZE, + RUBY_VM_FIBER_VM_STACK_SIZE_MIN); + + vm->default_params.fiber_machine_stack_size = + get_param("RUBY_FIBER_MACHINE_STACK_SIZE", + RUBY_VM_FIBER_MACHINE_STACK_SIZE, + RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN); + + /* environment dependent check */ + check_machine_stack_size(&vm->default_params.thread_machine_stack_size); + check_machine_stack_size(&vm->default_params.fiber_machine_stack_size); +} + static void vm_init2(rb_vm_t *vm) { @@ -1620,6 +1696,8 @@ vm_init2(rb_vm_t *vm) https://github.com/ruby/ruby/blob/trunk/vm.c#L1696 vm->src_encoding_index = -1; vm->at_exit.basic.flags = (T_ARRAY | RARRAY_EMBED_FLAG) & ~RARRAY_EMBED_LEN_MASK; /* len set 0 */ vm->at_exit.basic.klass = 0; + + vm_default_params_setup(vm); } /* Thread */ @@ -1635,6 +1713,7 @@ static VALUE * https://github.com/ruby/ruby/blob/trunk/vm.c#L1713 thread_recycle_stack(size_t size) { if (thread_recycle_stack_count) { + /* TODO: check stack size if stack sizes are variable */ return thread_recycle_stack_slot[--thread_recycle_stack_count]; } else { @@ -1839,7 +1918,10 @@ th_init(rb_thread_t *th, VALUE self) https://github.com/ruby/ruby/blob/trunk/vm.c#L1918 /* altstack of main thread is reallocated in another place */ th->altstack = malloc(rb_sigaltstack_size()); #endif - th->stack_size = RUBY_VM_THREAD_STACK_SIZE; + /* th->stack_size is word number. + * th->vm->default_params.thread_vm_stack_size is byte size. + */ + th->stack_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); th->stack = thread_recycle_stack(th->stack_size); th->cfp = (void *)(th->stack + th->stack_size); @@ -1864,9 +1946,9 @@ ruby_thread_init(VALUE self) https://github.com/ruby/ruby/blob/trunk/vm.c#L1946 rb_vm_t *vm = GET_THREAD()->vm; GetThreadPtr(self, th); + th->vm = vm; th_init(th, self); rb_iv_set(self, "locals", rb_hash_new()); - th->vm = vm; th->top_wrapper = 0; th->top_self = rb_vm_top_self(); @@ -2189,6 +2271,14 @@ Init_VM(void) https://github.com/ruby/ruby/blob/trunk/vm.c#L2271 /* ::RubyVM::INSTRUCTION_NAMES */ rb_define_const(rb_cRubyVM, "INSTRUCTION_NAMES", rb_insns_name_array()); + /* ::RubyVM::DEFAULT_PARAMS + * This constant variable shows VM's default parameters. + * Note that changing these values does not affect VM exection. + * Specification is not stable and you should not depend on this value. + * Of course, this constant is MRI specific. + */ + rb_define_const(rb_cRubyVM, "DEFAULT_PARAMS", vm_default_params()); + /* debug functions ::RubyVM::SDR(), ::RubyVM::NSDR() */ #if VMDEBUG rb_define_singleton_method(rb_cRubyVM, "SDR", sdr, 0); @@ -2266,7 +2356,6 @@ Init_BareVM(void) https://github.com/ruby/ruby/blob/trunk/vm.c#L2356 exit(EXIT_FAILURE); } MEMZERO(th, rb_thread_t, 1); - rb_thread_set_current_raw(th); vm_init2(vm); @@ -2276,8 +2365,8 @@ Init_BareVM(void) https://github.com/ruby/ruby/blob/trunk/vm.c#L2365 ruby_current_vm = vm; Init_native_thread(); - th_init(th, 0); th->vm = vm; + th_init(th, 0); ruby_thread_init_stack(th); } Index: test/ruby/test_fiber.rb =================================================================== --- test/ruby/test_fiber.rb (revision 38477) +++ test/ruby/test_fiber.rb (revision 38478) @@ -4,6 +4,7 @@ require 'continuation' https://github.com/ruby/ruby/blob/trunk/test/ruby/test_fiber.rb#L4 require_relative './envutil' class TestFiber < Test::Unit::TestCase +if false def test_normal f = Fiber.current assert_equal(:ok2, @@ -280,3 +281,44 @@ class TestFiber < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_fiber.rb#L281 end end + def invoke_rec script, vm_stack_size, machine_stack_size, use_length = true + env = {} + env['RUBY_FIBER_VM_STACK_SIZE'] = vm_stack_size.to_s if vm_stack_size + env['RUBY_FIBER_MACHINE_STACK_SIZE'] = machine_stack_size.to_s if machine_stack_size + out, = EnvUtil.invoke_ruby([env, '-e', script], '', true, true) + use_length ? out.length : out + end + + def test_stack_size + h_default = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', nil, nil, false)) + h_0 = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 0, 0, false)) + h_large = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 1024 * 1024 * 10, 1024 * 1024 * 10, false)) + + assert(h_default[:fiber_vm_stack_size] > h_0[:fiber_vm_stack_size]) + assert(h_default[:fiber_vm_stack_size] < h_large[:fiber_vm_stack_size]) + assert(h_default[:fiber_machine_stack_size] >= h_0[:fiber_machine_stack_size]) + assert(h_default[:fiber_machine_stack_size] <= h_large[:fiber_machine_stack_size]) + + # check VM machine stack size + script = 'def rec; print "."; rec; end; Fiber.new{rec}.resume' + size_default = invoke_rec script, nil, nil + assert(size_default > 0, size_default.to_s) + size_0 = invoke_rec script, 0, nil + assert(size_default > size_0, [size_default, size_0].inspect) + size_large = invoke_rec script, 1024 * 1024 * 10, nil + assert(size_default < size_large, [size_default, size_large].inspect) + + return if /mswin|mingw/ =~ RUBY_PLATFORM + + # check machine stack size + # Note that machine stack size may not change size (depend on OSs) + script = 'def rec; print "."; 1.times{1.times{1.times{rec}}}; end; Fiber.new{rec}.resume' + vm_stack_size = 1024 * 1024 + size_default = invoke_rec script, vm_stack_size, nil + size_0 = invoke_rec script, vm_stack_size, 0 + assert(size_default >= size_0, [size_default, size_0].inspect) + size_large = invoke_rec script, vm_stack_size, 1024 * 1024 * 10 + assert(size_default <= size_large, [size_default, size_large].inspect) + end +end + Index: test/ruby/test_thread.rb =================================================================== --- test/ruby/test_thread.rb (revision 38477) +++ test/ruby/test_thread.rb (revision 38478) @@ -870,4 +870,44 @@ Thread.new(Thread.current) {|mth| https://github.com/ruby/ruby/blob/trunk/test/ruby/test_thread.rb#L870 th.kill if th end end + + def invoke_rec script, vm_stack_size, machine_stack_size, use_length = true + env = {} + env['RUBY_THREAD_VM_STACK_SIZE'] = vm_stack_size.to_s if vm_stack_size + env['RUBY_THREAD_MACHINE_STACK_SIZE'] = machine_stack_size.to_s if machine_stack_size + out, = EnvUtil.invoke_ruby([env, '-e', script], '', true, true) + use_length ? out.length : out + end + + def test_stack_size + h_default = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', nil, nil, false)) + h_0 = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 0, 0, false)) + h_large = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 1024 * 1024 * 10, 1024 * 1024 * 10, false)) + + assert(h_default[:thread_vm_stack_size] > h_0[:thread_vm_stack_size]) + assert(h_default[:thread_vm_stack_size] < h_large[:thread_vm_stack_size]) + assert(h_default[:thread_machine_stack_size] >= h_0[:thread_machine_stack_size]) + assert(h_default[:thread_machine_stack_size] <= h_large[:thread_machine_stack_size]) + + # check VM machine stack size + script = 'def rec; print "."; rec; end; rec' + size_default = invoke_rec script, nil, nil + assert(size_default > 0, size_default.to_s) + size_0 = invoke_rec script, 0, nil + assert(size_default > size_0, [size_default, size_0].inspect) + size_large = invoke_rec script, 1024 * 1024 * 10, nil + assert(size_default < size_large, [size_default, size_large].inspect) + + return if /mswin|mingw/ =~ RUBY_PLATFORM + + # check machine stack size + # Note that machine stack size may not change size (depend on OSs) + script = 'def rec; print "."; 1.times{1.times{1.times{rec}}}; end; Thread.new{rec}.join' + vm_stack_size = 1024 * 1024 + size_default = invoke_rec script, vm_stack_size, nil + size_0 = invoke_rec script, vm_stack_size, 0 + assert(size_default >= size_0, [size_default, size_0].inspect) + size_large = invoke_rec script, vm_stack_size, 1024 * 1024 * 10 + assert(size_default <= size_large, [size_default, size_large].inspect) + end end -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/