nobu	2007-10-24 17:09:40 +0900 (Wed, 24 Oct 2007)

  New Revision: 13766

  Modified files:
    trunk/ChangeLog
    trunk/parse.y
    trunk/version.h

  Log:
    * parse.y (parser_tokspace): make space in token buffer.
    
    * parse.y (parser_yylex): fix encoding of single character literal.


  http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/version.h?r1=13766&r2=13765
  http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/parse.y?r1=13766&r2=13765
  http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/ChangeLog?r1=13766&r2=13765

Index: ChangeLog
===================================================================
--- ChangeLog	(revision 13765)
+++ ChangeLog	(revision 13766)
@@ -1,3 +1,9 @@
+Wed Oct 24 17:09:39 2007  Nobuyoshi Nakada  <nobu@r...>
+
+	* parse.y (parser_tokspace): make space in token buffer.
+
+	* parse.y (parser_yylex): fix encoding of single character literal.
+
 Tue Oct 23 13:44:14 2007  Nobuyoshi Nakada  <nobu@r...>
 
 	* parse.y (call_args2): nd_head of NODE_BLOCK_PASS should be a list.
Index: parse.y
===================================================================
--- parse.y	(revision 13765)
+++ parse.y	(revision 13766)
@@ -4494,6 +4494,7 @@
 # define nextc()                   parser_nextc(parser)
 # define pushback(c)               parser_pushback(parser, c)
 # define newtok()                  parser_newtok(parser)
+# define tokspace(n)               parser_tokspace(parser, n)
 # define tokadd(c)                 parser_tokadd(parser, c)
 # define read_escape(m)            parser_read_escape(parser, m)
 # define tokadd_escape(t,m)        parser_tokadd_escape(parser, t, m)
@@ -4913,6 +4914,18 @@
     return tokenbuf;
 }
 
+static char *
+parser_tokspace(struct parser_params *parser, int n)
+{
+    int idx = tokidx + n;
+
+    if (idx >= toksiz) {
+	do {toksiz *= 2;} while (toksiz < idx);
+	REALLOC_N(tokenbuf, char, toksiz);
+    }
+    return &tokenbuf[tokidx];
+}
+
 static void
 parser_tokadd(struct parser_params *parser, int c)
 {
@@ -6086,21 +6099,24 @@
 	}
 	newtok();
 	if (parser_ismbchar()) {
+	    mb = ENC_CODERANGE_MULTI;
 	    tokadd_mbchar(c);
 	}
 	else if ((rb_enc_isalnum(c, parser->enc) || c == '_') &&
 		 lex_p < lex_pend && is_identchar(lex_p, lex_pend, parser->enc)) {
 	    goto ternary;
 	}
-	else if (c == '\\') {
-	    c = read_escape(0);
-	    tokadd(c);
+	else if (c == '\\' && (c = read_escape(0)) >= 0x80) {
+	    rb_encoding *enc = parser->enc;
+	    mb = ENC_CODERANGE_UNKNOWN;
+	    rb_enc_mbcput(c, tokspace(rb_enc_codelen(c, enc)), enc);
 	}
 	else {
+	    mb = ENC_CODERANGE_SINGLE;
 	    tokadd(c);
 	}
 	tokfix();
-	set_yylval_str(STR_NEW(tok(), toklen()));
+	set_yylval_str(STR_NEW3(tok(), toklen(), mb));
 	lex_state = EXPR_ENDARG;
 	return tCHAR;
 
@@ -6865,9 +6881,9 @@
 	break;
     }
 
-    mb = 0;
+    mb = ENC_CODERANGE_SINGLE;
     do {
-	if (!ISASCII(c)) mb = 1;
+	if (!ISASCII(c)) mb = ENC_CODERANGE_UNKNOWN;
 	tokadd_mbchar(c);
 	c = nextc();
     } while (parser_is_identchar());
@@ -6920,7 +6936,7 @@
 		}
 	    }
 
-	    if (!mb && lex_state != EXPR_DOT) {
+	    if (mb == ENC_CODERANGE_SINGLE && lex_state != EXPR_DOT) {
 		const struct kwtable *kw;
 
 		/* See if it is a reserved word.  */
Index: version.h
===================================================================
--- version.h	(revision 13765)
+++ version.h	(revision 13766)
@@ -1,7 +1,7 @@
 #define RUBY_VERSION "1.9.0"
-#define RUBY_RELEASE_DATE "2007-10-23"
+#define RUBY_RELEASE_DATE "2007-10-24"
 #define RUBY_VERSION_CODE 190
-#define RUBY_RELEASE_CODE 20071023
+#define RUBY_RELEASE_CODE 20071024
 #define RUBY_PATCHLEVEL 0
 
 #define RUBY_VERSION_MAJOR 1
@@ -9,7 +9,7 @@
 #define RUBY_VERSION_TEENY 0
 #define RUBY_RELEASE_YEAR 2007
 #define RUBY_RELEASE_MONTH 10
-#define RUBY_RELEASE_DAY 23
+#define RUBY_RELEASE_DAY 24
 
 #ifdef RUBY_EXTERN
 RUBY_EXTERN const char ruby_version[];

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml