? japanese_tokenize.4.diff
Index: nsBayesianFilter.cpp
===================================================================
RCS file: /cvsroot/mozilla/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp,v
retrieving revision 1.35.6.7
diff -u -r1.35.6.7 nsBayesianFilter.cpp
--- nsBayesianFilter.cpp	19 Oct 2004 21:25:42 -0000	1.35.6.7
+++ nsBayesianFilter.cpp	14 Jan 2005 15:50:47 -0000
@@ -443,6 +443,153 @@
   } 
 }
 
+/*
+    Character unicode memo
+    
+    Hiragana:
+        Hiragana: 3040-309F
+    
+    Katakana:
+        Katakana: 30A0-30FF (except 30FB: KATAKANA MIDDLE DOT)
+        Halfwidth and Fullwidth Forms (FF00-FFEF): FF66-FF9F
+        * Katakana Phonetic Extensions: 31F0-31FF
+    
+    Kanji:
+        CJK Radicals Supplement: 2E80-2EFF
+        Kangxi Radicals: 2F00-2FDF
+        CJK Unified Ideographs: 4E00-9FAF
+        * Ideographic Description Characters: 2FF0-2FFF
+        * CJK Unified Ideographs Extension A: 3400-4DBF
+        * CJK Compatibility Ideographs: F900-FAFF
+    
+    Kigou:
+        * Arrows: 2190-21FF
+        * Mathematical Operators: 2200-22FF
+        * Box Drawing: 2500-257F
+        * Geometric Shapes: 25A0-25FF
+        * Miscellaneous Symbols: 2600-26FF
+        * CJK Symbols and Punctuation (3000-303F): 3003-303F
+        * Enclosed CJK Letters and Months: 3200-32FF
+        * CJK Compatibility: 3300-33FF
+        * CJK Compatibility Forms: FE30-FE4F
+        * Halfwidth and Fullwidth Forms (FF00-FFEF): FF5F-FF60, FF62-FF63, FFE0-FFF6, FFE8-FFEE
+        * and others?
+    
+    Space:
+        C0 Controls and Basic Latin (0000-007F): 0020
+        CJK Symbols and Punctuation (3000-303F): 3000
+        
+    Kuten:
+        CJK Symbols and Punctuation (3000-303F): 3001
+        Halfwidth and Fullwidth Forms (FF00-FFEF): FF0E, FF64
+        
+    Touten:
+        CJK Symbols and Punctuation (3000-303F): 3002
+        Halfwidth and Fullwidth Forms (FF00-FFEF): FF0C, FF61
+    
+    Fullwidth characters of latain:
+        Halfwidth and Fullwidth Forms (FF00-FFEF): FF01-FF5E (correspond to 0021-007E)
+    
+    Fullwidth numeral:
+        Halfwidth and Fullwidth Forms (FF00-FFEF): FF10-FF19
+    
+    see http://www.unicode.org/charts/
+*/
+
+// one substract and one conditional jump should be faster than two conditional jump on most recent system.
+//#define IN_RANGE(x, low, high)  ((low<=(x))&&((x)<=high))
+#define IN_RANGE(x, low, high)  ((PRUint16)((x)-(low)) <= (high)-(low))
+
+#define IS_JA_HIRAGANA(x)   IN_RANGE(x, 0x3040, 0x309F)
+// swapping the range using xor operation to reduce conditional jump.
+//#define IS_JA_KATAKANA(x)   ((IN_RANGE(x, 0x30A0, 0x30FF)&&((x)!=0x30FB))||IN_RANGE(x, 0xFF66, 0xFF9F))
+#define IS_JA_KATAKANA(x)	(IN_RANGE(x^0x0004, 0x30A0, 0x30FE)||(IN_RANGE(x, 0xFF66, 0xFF9F)))
+#define IS_JA_KANJI(x)      (IN_RANGE(x, 0x2E80, 0x2FDF)||IN_RANGE(x, 0x4E00, 0x9FAF))
+#define IS_JA_KUTEN(x)      (((x)==0x3001)||((x)==0xFF64)||((x)==0xFF0E))
+#define IS_JA_TOUTEN(x)     (((x)==0x3002)||((x)==0xFF61)||((x)==0xFF0C))
+#define IS_JA_SPACE(x)      ((x)==0x3000)
+#define IS_JA_FWLATAIN(x)   IN_RANGE(x, 0xFF01, 0xFF5E)
+#define IS_JA_FWNUMERAL(x)  IN_RANGE(x, 0xFF10, 0xFF19)
+
+#define IS_JAPANESE_SPECIFIC(x) (IN_RANGE(x, 0x3040, 0x30FF)||IN_RANGE(x, 0xFF01, 0xFF9F))
+
+enum char_class{
+    others = 0,
+    space,
+    hiragana,
+    katakana,
+    kanji,
+    kuten,
+    touten,
+    kigou,
+    fwlatain,
+    ascii
+};
+
+char_class getCharClass(PRUnichar c)
+{
+    if(IS_JA_HIRAGANA(c)){
+        return hiragana;
+    }else if(IS_JA_KATAKANA(c)){
+        return katakana;
+    }else if(IS_JA_KANJI(c)){
+        return kanji;
+    }else if(IS_JA_KUTEN(c)){
+        return kuten;
+    }else if(IS_JA_TOUTEN(c)){
+        return touten;
+    }else if(IS_JA_FWLATAIN(c)){
+        return fwlatain;
+    }else{
+        return others;
+    }
+}
+
+static PRBool isJapanese(const char* word)
+{
+    nsString text = NS_ConvertUTF8toUCS2(word);
+    PRUnichar* p = (PRUnichar*)text.get();
+    PRUnichar c;
+    
+    // it is japanese chunk if it contains any hiragana or katakana.
+    while((c = *p++)){
+        if( IS_JAPANESE_SPECIFIC(c) ) return PR_TRUE;
+    }
+    return PR_FALSE;
+}
+
+PRBool isFWNumeral(const PRUnichar* p1, const PRUnichar* p2)
+{
+    for(;p1<p2;p1++){
+        if(!IS_JA_FWNUMERAL(*p1)) return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
+void Tokenizer::tokenize_japanese_words(char* chunk)
+{
+    PR_LOG(BayesianFilterLogModule, PR_LOG_ALWAYS, ("entering tokenize_japanese_words(%s)", chunk));
+    
+    nsString srcStr = NS_ConvertUTF8toUCS2(chunk);
+    const PRUnichar* p1 = srcStr.get();
+    const PRUnichar* p2 = p1;
+    if(!*p2) return;
+    
+    char_class cc = getCharClass(*p2);
+    while(*(++p2)){
+        if(cc == getCharClass(*p2)) continue;
+        
+        nsCString token = NS_ConvertUCS2toUTF8(p1, p2-p1);
+        nsCString addToken = nsCString("JA:") + token;
+        if( (!isDecimalNumber(token.get())) && (!isFWNumeral(p1, p2)) ){
+            add(addToken.get());
+        }
+        
+        cc = getCharClass(*p2);
+        p1 = p2;
+    }
+}
+
 nsresult Tokenizer::stripHTML(const nsAString& inString, nsAString& outString)
 {
   nsresult rv = NS_OK;
@@ -484,7 +631,17 @@
     nsString text = NS_ConvertUTF8toUCS2(aText);
     nsString strippedUCS2;
     stripHTML(text, strippedUCS2);
-
+    
+    // convert 0x3000(full width space) into 0x0020
+    nsString::iterator substr_start, substr_end;
+    strippedUCS2.BeginWriting(substr_start);
+    strippedUCS2.EndWriting(substr_end);
+    while (substr_start != substr_end) {
+        if (*substr_start == 0x3000)
+            *substr_start = 0x0020;
+        ++substr_start;
+    }
+    
     nsCString strippedStr = NS_ConvertUCS2toUTF8(strippedUCS2);
     char * strippedText = (char *) strippedStr.get(); // bleh
     PR_LOG(BayesianFilterLogModule, PR_LOG_ALWAYS, ("tokenize stripped html: %s", strippedText));
@@ -496,6 +653,8 @@
         if (isDecimalNumber(word)) continue;
         if (isASCII(word))
             tokenize_ascii_word(word);
+        else if (isJapanese(word))
+            tokenize_japanese_words(word);
         else {
             nsresult rv;
             // use I18N  scanner to break this word into meaningful semantic units.
Index: nsBayesianFilter.h
===================================================================
RCS file: /cvsroot/mozilla/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h,v
retrieving revision 1.12.54.1
diff -u -r1.12.54.1 nsBayesianFilter.h
--- nsBayesianFilter.h	17 May 2004 00:57:48 -0000	1.12.54.1
+++ nsBayesianFilter.h	14 Jan 2005 15:50:47 -0000
@@ -123,6 +123,7 @@
 private:
     char* copyWord(const char* word, PRUint32 len);
     void tokenize_ascii_word(char * word);
+    void tokenize_japanese_words(char* chunk);
     inline void addTokenForHeader(const char * aTokenPrefix, nsACString& aValue, PRBool aTokenizeValue = false);
     nsresult stripHTML(const nsAString& inString, nsAString& outString);
 
