ruby-changes:3572
From: ko1@a...
Date: Tue, 15 Jan 2008 17:27:38 +0900 (JST)
Subject: [ruby-changes:3572] nobu - Ruby:r15061 (trunk): * enc/utf_8.c: renamed as IANA name.
nobu 2008-01-15 17:26:54 +0900 (Tue, 15 Jan 2008)
New Revision: 15061
Added files:
trunk/enc/utf_8.c
Removed files:
trunk/enc/utf8.c
Modified files:
trunk/ChangeLog
trunk/bcc32/Makefile.sub
trunk/common.mk
Log:
* enc/utf_8.c: renamed as IANA name.
http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/enc/utf8.c
http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/ChangeLog?r1=15061&r2=15060&diff_format=u
http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/bcc32/Makefile.sub?r1=15061&r2=15060&diff_format=u
http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/enc/utf_8.c
http://svn.ruby-lang.org/cgi-bin/viewvc.cgi/trunk/common.mk?r1=15061&r2=15060&diff_format=u
Index: ChangeLog
===================================================================
--- ChangeLog (revision 15060)
+++ ChangeLog (revision 15061)
@@ -1,3 +1,7 @@
+Tue Jan 15 17:26:51 2008 Nobuyoshi Nakada <nobu@r...>
+
+ * enc/utf_8.c: renamed as IANA name.
+
Tue Jan 15 16:59:31 2008 Nobuyoshi Nakada <nobu@r...>
* ruby.c (proc_options): encoding libraries cannot be loaded until
@@ -136,7 +140,7 @@
Mon Jan 14 12:33:07 2008 Eric Hodel <drbrain@s...>
* lib/rdoc/markup*: Renamespace from SM::SimpleMarkup to
- RDoc::Markup.
+ RDoc::Markup.
Mon Jan 14 10:45:45 2008 Martin Duerst <duerst@i...>
Index: enc/utf8.c
===================================================================
--- enc/utf8.c (revision 15060)
+++ enc/utf8.c (revision 15061)
@@ -1,482 +0,0 @@
-/**********************************************************************
- utf8.c - Oniguruma (regular expression library)
-**********************************************************************/
-/*-
- * Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "regenc.h"
-
-#define USE_INVALID_CODE_SCHEME
-
-#ifdef USE_INVALID_CODE_SCHEME
-/* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
-#define INVALID_CODE_FE 0xfffffffe
-#define INVALID_CODE_FF 0xffffffff
-#define VALID_CODE_LIMIT 0x7fffffff
-#endif
-
-#define utf8_islead(c) ((UChar )((c) & 0xc0) != 0x80)
-
-static const int EncLen_UTF8[] = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
-};
-
-typedef enum {
- FAILURE = -2,
- ACCEPT,
- S0, S1, S2, S3,
- S4, S5, S6, S7
-} state_t;
-#define A ACCEPT
-#define F FAILURE
-static const signed char trans[][0x100] = {
- { /* S0 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 1 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 2 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 3 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 4 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 5 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 6 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 7 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* c */ F, F, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* d */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* e */ 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3,
- /* f */ 5, 6, 6, 6, 7, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S1 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* 9 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* a */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* b */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S2 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* a */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S3 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* a */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S4 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S5 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 9 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* a */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* b */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S6 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* 9 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* a */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* b */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
- { /* S7 0 1 2 3 4 5 6 7 8 9 a b c d e f */
- /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* 8 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
- },
-};
-#undef A
-#undef F
-
-static int
-mbc_enc_len(const UChar* p, const UChar* e, OnigEncoding enc)
-{
- int firstbyte = *p++;
- state_t s;
- s = trans[0][firstbyte];
- if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(1) :
- ONIGENC_CONSTRUCT_MBCLEN_INVALID();
-
- if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_UTF8[firstbyte]-1);
- s = trans[s][*p++];
- if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(2) :
- ONIGENC_CONSTRUCT_MBCLEN_INVALID();
-
- if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_UTF8[firstbyte]-2);
- s = trans[s][*p++];
- if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(3) :
- ONIGENC_CONSTRUCT_MBCLEN_INVALID();
-
- if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_UTF8[firstbyte]-3);
- s = trans[s][*p++];
- return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(4) :
- ONIGENC_CONSTRUCT_MBCLEN_INVALID();
-}
-
-static int
-is_mbc_newline(const UChar* p, const UChar* end, OnigEncoding enc)
-{
- if (p < end) {
- if (*p == 0x0a) return 1;
-
-#ifdef USE_UNICODE_ALL_LINE_TERMINATORS
-#ifndef USE_CRNL_AS_LINE_TERMINATOR
- if (*p == 0x0d) return 1;
-#endif
- if (p + 1 < end) {
- if (*(p+1) == 0x85 && *p == 0xc2) /* U+0085 */
- return 1;
- if (p + 2 < end) {
- if ((*(p+2) == 0xa8 || *(p+2) == 0xa9)
- && *(p+1) == 0x80 && *p == 0xe2) /* U+2028, U+2029 */
- return 1;
- }
- }
-#endif
- }
-
- return 0;
-}
-
-static OnigCodePoint
-mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
-{
- int c, len;
- OnigCodePoint n;
-
- len = enclen(ONIG_ENCODING_UTF8, p, end);
- c = *p++;
- if (len > 1) {
- len--;
- n = c & ((1 << (6 - len)) - 1);
- while (len--) {
- c = *p++;
- n = (n << 6) | (c & ((1 << 6) - 1));
- }
- return n;
- }
- else {
-#ifdef USE_INVALID_CODE_SCHEME
- if (c > 0xfd) {
- return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
- }
-#endif
- return (OnigCodePoint )c;
- }
-}
-
-static int
-code_to_mbclen(OnigCodePoint code, OnigEncoding enc)
-{
- if ((code & 0xffffff80) == 0) return 1;
- else if ((code & 0xfffff800) == 0) return 2;
- else if ((code & 0xffff0000) == 0) return 3;
- else if ((code & 0xffe00000) == 0) return 4;
- else if ((code & 0xfc000000) == 0) return 5;
- else if ((code & 0x80000000) == 0) return 6;
-#ifdef USE_INVALID_CODE_SCHEME
- else if (code == INVALID_CODE_FE) return 1;
- else if (code == INVALID_CODE_FF) return 1;
-#endif
- else
- return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
-}
-
-static int
-code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
-{
-#define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
-#define UTF8_TRAIL0(code) (UChar )(((code) & 0x3f) | 0x80)
-
- if ((code & 0xffffff80) == 0) {
- *buf = (UChar )code;
- return 1;
- }
- else {
- UChar *p = buf;
-
- if ((code & 0xfffff800) == 0) {
- *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
- }
- else if ((code & 0xffff0000) == 0) {
- *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
- *p++ = UTF8_TRAILS(code, 6);
- }
- else if ((code & 0xffe00000) == 0) {
- *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
- *p++ = UTF8_TRAILS(code, 12);
- *p++ = UTF8_TRAILS(code, 6);
- }
- else if ((code & 0xfc000000) == 0) {
- *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
- *p++ = UTF8_TRAILS(code, 18);
- *p++ = UTF8_TRAILS(code, 12);
- *p++ = UTF8_TRAILS(code, 6);
- }
- else if ((code & 0x80000000) == 0) {
- *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
- *p++ = UTF8_TRAILS(code, 24);
- *p++ = UTF8_TRAILS(code, 18);
- *p++ = UTF8_TRAILS(code, 12);
- *p++ = UTF8_TRAILS(code, 6);
- }
-#ifdef USE_INVALID_CODE_SCHEME
- else if (code == INVALID_CODE_FE) {
- *p = 0xfe;
- return 1;
- }
- else if (code == INVALID_CODE_FF) {
- *p = 0xff;
- return 1;
- }
-#endif
- else {
- return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
- }
-
- *p++ = UTF8_TRAIL0(code);
- return p - buf;
- }
-}
-
-static int
-mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
- const UChar* end, UChar* fold, OnigEncoding enc)
-{
- const UChar* p = *pp;
-
- if (ONIGENC_IS_MBC_ASCII(p)) {
-#ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
- if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
- if (*p == 0x49) {
- *fold++ = 0xc4;
- *fold = 0xb1;
- (*pp)++;
- return 2;
- }
- }
-#endif
-
- *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
- (*pp)++;
- return 1; /* return byte length of converted char to lower */
- }
- else {
- return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
- pp, end, fold);
- }
-}
-
-#if 0
-static int
-is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp, const UChar* end)
-{
- const UChar* p = *pp;
-
- if (ONIGENC_IS_MBC_ASCII(p)) {
- (*pp)++;
- return ONIGENC_IS_ASCII_CODE_CASE_AMBIG(*p);
- }
- else {
- (*pp) += enclen(ONIG_ENCODING_UTF8, p);
-
- if (*p == 0xc3) {
- int c = *(p + 1);
- if (c >= 0x80) {
- if (c <= (UChar )0x9e) { /* upper */
- if (c == (UChar )0x97) return FALSE;
- return TRUE;
- }
- else if (c >= (UChar )0xa0 && c <= (UChar )0xbe) { /* lower */
- if (c == (UChar )'\267') return FALSE;
- return TRUE;
- }
- else if (c == (UChar )0x9f &&
- (flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
- return TRUE;
- }
- }
- }
- }
-
- return FALSE;
-}
-#endif
-
-
-static int
-get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
- const OnigCodePoint* ranges[], OnigEncoding enc)
-{
- *sb_out = 0x80;
- return onigenc_unicode_ctype_code_range(ctype, ranges);
-}
-
-
-static UChar*
-left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
-{
- const UChar *p;
-
- if (s <= start) return (UChar* )s;
- p = s;
-
- while (!utf8_islead(*p) && p > start) p--;
- return (UChar* )p;
-}
-
-static int
-get_case_fold_codes_by_str(OnigCaseFoldType flag,
- const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[],
- OnigEncoding enc)
-{
- return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
- flag, p, end, items);
-}
-
-OnigEncodingDefine(utf8, UTF8) = {
- mbc_enc_len,
- "UTF-8", /* name */
- 6, /* max byte length */
- 1, /* min byte length */
- is_mbc_newline,
- mbc_to_code,
- code_to_mbclen,
- code_to_mbc,
- mbc_case_fold,
- onigenc_unicode_apply_all_case_fold,
- get_case_fold_codes_by_str,
- onigenc_unicode_property_name_to_ctype,
- onigenc_unicode_is_code_ctype,
- get_ctype_code_range,
- left_adjust_char_head,
- onigenc_always_true_is_allowed_reverse_match
-};
Index: enc/utf_8.c
===================================================================
--- enc/utf_8.c (revision 0)
+++ enc/utf_8.c (revision 15061)
@@ -0,0 +1,486 @@
+/**********************************************************************
+ utf_8.c - Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "regenc.h"
+
+OnigEncodingDeclare(UTF_8);
+#undef ONIG_ENCODING_UTF8
+#define ONIG_ENCODING_UTF8 (&OnigEncodingName(UTF_8))
+
+#define USE_INVALID_CODE_SCHEME
+
+#ifdef USE_INVALID_CODE_SCHEME
+/* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
+#define INVALID_CODE_FE 0xfffffffe
+#define INVALID_CODE_FF 0xffffffff
+#define VALID_CODE_LIMIT 0x7fffffff
+#endif
+
+#define utf8_islead(c) ((UChar )((c) & 0xc0) != 0x80)
+
+static const int EncLen_UTF8[] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
+};
+
+typedef enum {
+ FAILURE = -2,
+ ACCEPT,
+ S0, S1, S2, S3,
+ S4, S5, S6, S7
+} state_t;
+#define A ACCEPT
+#define F FAILURE
+static const signed char trans[][0x100] = {
+ { /* S0 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 1 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 2 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 3 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 4 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 5 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 6 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 7 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* c */ F, F, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* d */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* e */ 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3,
+ /* f */ 5, 6, 6, 6, 7, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S1 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* 9 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* a */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* b */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S2 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* a */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S3 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* a */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S4 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S5 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 9 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* a */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* b */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S6 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* 9 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* a */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* b */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+ { /* S7 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* 8 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ },
+};
+#undef A
+#undef F
+
+static int
+mbc_enc_len(const UChar* p, const UChar* e, OnigEncoding enc)
+{
+ int firstbyte = *p++;
+ state_t s;
+ s = trans[0][firstbyte];
+ if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(1) :
+ ONIGENC_CONSTRUCT_MBCLEN_INVALID();
+
+ if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_UTF8[firstbyte]-1);
+ s = trans[s][*p++];
+ if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(2) :
+ ONIGENC_CONSTRUCT_MBCLEN_INVALID();
+
+ if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_UTF8[firstbyte]-2);
+ s = trans[s][*p++];
+ if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(3) :
+ ONIGENC_CONSTRUCT_MBCLEN_INVALID();
+
+ if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_UTF8[firstbyte]-3);
+ s = trans[s][*p++];
+ return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(4) :
+ ONIGENC_CONSTRUCT_MBCLEN_INVALID();
+}
+
+static int
+is_mbc_newline(const UChar* p, const UChar* end, OnigEncoding enc)
+{
+ if (p < end) {
+ if (*p == 0x0a) return 1;
+
+#ifdef USE_UNICODE_ALL_LINE_TERMINATORS
+#ifndef USE_CRNL_AS_LINE_TERMINATOR
+ if (*p == 0x0d) return 1;
+#endif
+ if (p + 1 < end) {
+ if (*(p+1) == 0x85 && *p == 0xc2) /* U+0085 */
+ return 1;
+ if (p + 2 < end) {
+ if ((*(p+2) == 0xa8 || *(p+2) == 0xa9)
+ && *(p+1) == 0x80 && *p == 0xe2) /* U+2028, U+2029 */
+ return 1;
+ }
+ }
+#endif
+ }
+
+ return 0;
+}
+
+static OnigCodePoint
+mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
+{
+ int c, len;
+ OnigCodePoint n;
+
+ len = enclen(ONIG_ENCODING_UTF8, p, end);
+ c = *p++;
+ if (len > 1) {
+ len--;
+ n = c & ((1 << (6 - len)) - 1);
+ while (len--) {
+ c = *p++;
+ n = (n << 6) | (c & ((1 << 6) - 1));
+ }
+ return n;
+ }
+ else {
+#ifdef USE_INVALID_CODE_SCHEME
+ if (c > 0xfd) {
+ return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
+ }
+#endif
+ return (OnigCodePoint )c;
+ }
+}
+
+static int
+code_to_mbclen(OnigCodePoint code, OnigEncoding enc)
+{
+ if ((code & 0xffffff80) == 0) return 1;
+ else if ((code & 0xfffff800) == 0) return 2;
+ else if ((code & 0xffff0000) == 0) return 3;
+ else if ((code & 0xffe00000) == 0) return 4;
+ else if ((code & 0xfc000000) == 0) return 5;
+ else if ((code & 0x80000000) == 0) return 6;
+#ifdef USE_INVALID_CODE_SCHEME
+ else if (code == INVALID_CODE_FE) return 1;
+ else if (code == INVALID_CODE_FF) return 1;
+#endif
+ else
+ return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
+}
+
+static int
+code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
+{
+#define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
+#define UTF8_TRAIL0(code) (UChar )(((code) & 0x3f) | 0x80)
+
+ if ((code & 0xffffff80) == 0) {
+ *buf = (UChar )code;
+ return 1;
+ }
+ else {
+ UChar *p = buf;
+
+ if ((code & 0xfffff800) == 0) {
+ *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
+ }
+ else if ((code & 0xffff0000) == 0) {
+ *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
+ *p++ = UTF8_TRAILS(code, 6);
+ }
+ else if ((code & 0xffe00000) == 0) {
+ *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
+ *p++ = UTF8_TRAILS(code, 12);
+ *p++ = UTF8_TRAILS(code, 6);
+ }
+ else if ((code & 0xfc000000) == 0) {
+ *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
+ *p++ = UTF8_TRAILS(code, 18);
+ *p++ = UTF8_TRAILS(code, 12);
+ *p++ = UTF8_TRAILS(code, 6);
+ }
+ else if ((code & 0x80000000) == 0) {
+ *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
+ *p++ = UTF8_TRAILS(code, 24);
+ *p++ = UTF8_TRAILS(code, 18);
+ *p++ = UTF8_TRAILS(code, 12);
+ *p++ = UTF8_TRAILS(code, 6);
+ }
+#ifdef USE_INVALID_CODE_SCHEME
+ else if (code == INVALID_CODE_FE) {
+ *p = 0xfe;
+ return 1;
+ }
+ else if (code == INVALID_CODE_FF) {
+ *p = 0xff;
+ return 1;
+ }
+#endif
+ else {
+ return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
+ }
+
+ *p++ = UTF8_TRAIL0(code);
+ return p - buf;
+ }
+}
+
+static int
+mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
+ const UChar* end, UChar* fold, OnigEncoding enc)
+{
+ const UChar* p = *pp;
+
+ if (ONIGENC_IS_MBC_ASCII(p)) {
+#ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
+ if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
+ if (*p == 0x49) {
+ *fold++ = 0xc4;
+ *fold = 0xb1;
+ (*pp)++;
+ return 2;
+ }
+ }
+#endif
+
+ *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
+ (*pp)++;
+ return 1; /* return byte length of converted char to lower */
+ }
+ else {
+ return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
+ pp, end, fold);
+ }
+}
+
+#if 0
+static int
+is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp, const UChar* end)
+{
+ const UChar* p = *pp;
+
+ if (ONIGENC_IS_MBC_ASCII(p)) {
+ (*pp)++;
+ return ONIGENC_IS_ASCII_CODE_CASE_AMBIG(*p);
+ }
+ else {
+ (*pp) += enclen(ONIG_ENCODING_UTF8, p);
+
+ if (*p == 0xc3) {
+ int c = *(p + 1);
+ if (c >= 0x80) {
+ if (c <= (UChar )0x9e) { /* upper */
+ if (c == (UChar )0x97) return FALSE;
+ return TRUE;
+ }
+ else if (c >= (UChar )0xa0 && c <= (UChar )0xbe) { /* lower */
+ if (c == (UChar )'\267') return FALSE;
+ return TRUE;
+ }
+ else if (c == (UChar )0x9f &&
+ (flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
+ return TRUE;
+ }
+ }
+ }
+ }
+
+ return FALSE;
+}
+#endif
+
+
+static int
+get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
+ const OnigCodePoint* ranges[], OnigEncoding enc)
+{
+ *sb_out = 0x80;
+ return onigenc_unicode_ctype_code_range(ctype, ranges);
+}
+
+
+static UChar*
+left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
+{
+ const UChar *p;
+
+ if (s <= start) return (UChar* )s;
+ p = s;
+
+ while (!utf8_islead(*p) && p > start) p--;
+ return (UChar* )p;
+}
+
+static int
+get_case_fold_codes_by_str(OnigCaseFoldType flag,
+ const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[],
+ OnigEncoding enc)
+{
+ return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
+ flag, p, end, items);
+}
+
+OnigEncodingDefine(utf_8, UTF_8) = {
+ mbc_enc_len,
+ "UTF-8", /* name */
+ 6, /* max byte length */
+ 1, /* min byte length */
+ is_mbc_newline,
+ mbc_to_code,
+ code_to_mbclen,
+ code_to_mbc,
+ mbc_case_fold,
+ onigenc_unicode_apply_all_case_fold,
+ get_case_fold_codes_by_str,
+ onigenc_unicode_property_name_to_ctype,
+ onigenc_unicode_is_code_ctype,
+ get_ctype_code_range,
+ left_adjust_char_head,
+ onigenc_always_true_is_allowed_reverse_match
+};
Property changes on: enc/utf_8.c
___________________________________________________________________
Name: svn:eol-style
+ LF
Name: svn:keywords
+ Author Date Id Revision
Index: common.mk
===================================================================
--- common.mk (revision 15060)
+++ common.mk (revision 15061)
@@ -668,7 +668,7 @@
unicode.$(OBJEXT): {$(VPATH)}unicode.c {$(VPATH)}regint.h \
{$(VPATH)}config.h {$(VPATH)}defines.h {$(VPATH)}regenc.h \
{$(VPATH)}oniguruma.h {$(VPATH)}st.h
-utf8.$(OBJEXT): {$(VPATH)}utf8.c {$(VPATH)}regenc.h {$(VPATH)}config.h \
+utf_8.$(OBJEXT): {$(VPATH)}utf_8.c {$(VPATH)}regenc.h {$(VPATH)}config.h \
{$(VPATH)}defines.h {$(VPATH)}oniguruma.h
INSNS = opt_sc.inc optinsn.inc optunifs.inc insns.inc \
Index: bcc32/Makefile.sub
===================================================================
--- bcc32/Makefile.sub (revision 15060)
+++ bcc32/Makefile.sub (revision 15061)
@@ -565,7 +565,7 @@
string.$(OBJEXT): win32.h
struct.$(OBJEXT): win32.h
time.$(OBJEXT): win32.h
-utf8.$(OBJEXT): win32.h
+utf_8.$(OBJEXT): win32.h
util.$(OBJEXT): win32.h
variable.$(OBJEXT): win32.h
version.$(OBJEXT): win32.h
--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/