Back to index

lightning-sunbird  0.9+nobinonly
nsHebrewProber.cpp
Go to the documentation of this file.
00001 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
00002 /* ***** BEGIN LICENSE BLOCK *****
00003  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
00004  *
00005  * The contents of this file are subject to the Mozilla Public License Version
00006  * 1.1 (the "License"); you may not use this file except in compliance with
00007  * the License. You may obtain a copy of the License at
00008  * http://www.mozilla.org/MPL/
00009  *
00010  * Software distributed under the License is distributed on an "AS IS" basis,
00011  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
00012  * for the specific language governing rights and limitations under the
00013  * License.
00014  *
00015  * The Original Code is Mozilla Universal charset detector code.
00016  *
00017  * The Initial Developer of the Original Code is
00018  *          Shy Shalom <shooshX@gmail.com>
00019  * Portions created by the Initial Developer are Copyright (C) 2005
00020  * the Initial Developer. All Rights Reserved.
00021  *
00022  * Contributor(s):
00023  *
00024  * Alternatively, the contents of this file may be used under the terms of
00025  * either the GNU General Public License Version 2 or later (the "GPL"), or
00026  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
00027  * in which case the provisions of the GPL or the LGPL are applicable instead
00028  * of those above. If you wish to allow use of your version of this file only
00029  * under the terms of either the GPL or the LGPL, and not to allow others to
00030  * use your version of this file under the terms of the MPL, indicate your
00031  * decision by deleting the provisions above and replace them with the notice
00032  * and other provisions required by the GPL or the LGPL. If you do not delete
00033  * the provisions above, a recipient may use your version of this file under
00034  * the terms of any one of the MPL, the GPL or the LGPL.
00035  *
00036  * ***** END LICENSE BLOCK ***** */
00037 
00038 #include "nsHebrewProber.h"
00039 #include <stdio.h>
00040 
00041 // windows-1255 / ISO-8859-8 code points of interest
00042 #define FINAL_KAF ('\xea')
00043 #define NORMAL_KAF ('\xeb')
00044 #define FINAL_MEM ('\xed')
00045 #define NORMAL_MEM ('\xee')
00046 #define FINAL_NUN ('\xef')
00047 #define NORMAL_NUN ('\xf0')
00048 #define FINAL_PE ('\xf3')
00049 #define NORMAL_PE ('\xf4')
00050 #define FINAL_TSADI ('\xf5')
00051 #define NORMAL_TSADI ('\xf6')
00052 
00053 // Minimum Visual vs Logical final letter score difference.
00054 // If the difference is below this, don't rely solely on the final letter score distance.
00055 #define MIN_FINAL_CHAR_DISTANCE (5)
00056 
00057 // Minimum Visual vs Logical model score difference.
00058 // If the difference is below this, don't rely at all on the model score distance.
00059 #define MIN_MODEL_DISTANCE (0.01)
00060 
00061 #define VISUAL_HEBREW_NAME ("ISO-8859-8")
00062 #define LOGICAL_HEBREW_NAME ("windows-1255")
00063 
00064 PRBool nsHebrewProber::isFinal(char c)
00065 {
00066   return ((c == FINAL_KAF) || (c == FINAL_MEM) || (c == FINAL_NUN) || (c == FINAL_PE) || (c == FINAL_TSADI));
00067 }
00068 
00069 PRBool nsHebrewProber::isNonFinal(char c)
00070 {
00071   return ((c == NORMAL_KAF) || (c == NORMAL_MEM) || (c == NORMAL_NUN) || (c == NORMAL_PE));
00072   // The normal Tsadi is not a good Non-Final letter due to words like 
00073   // 'lechotet' (to chat) containing an apostrophe after the tsadi. This 
00074   // apostrophe is converted to a space in FilterWithoutEnglishLetters causing 
00075   // the Non-Final tsadi to appear at an end of a word even though this is not 
00076   // the case in the original text.
00077   // The letters Pe and Kaf rarely display a related behavior of not being a 
00078   // good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for 
00079   // example legally end with a Non-Final Pe or Kaf. However, the benefit of 
00080   // these letters as Non-Final letters outweighs the damage since these words 
00081   // are quite rare.
00082 }
00083 
00109 nsProbingState nsHebrewProber::HandleData(const char* aBuf, PRUint32 aLen)
00110 {
00111   // Both model probers say it's not them. No reason to continue.
00112   if (GetState() == eNotMe)
00113     return eNotMe;
00114 
00115   const char *curPtr, *endPtr = aBuf+aLen;
00116   char cur;
00117 
00118   for (curPtr = (char*)aBuf; curPtr < endPtr; ++curPtr)
00119   {
00120     cur = *curPtr;
00121     if (cur == ' ') // We stand on a space - a word just ended
00122     {
00123       if (mBeforePrev != ' ') // *(curPtr-2) was not a space so prev is not a 1 letter word
00124       {
00125         if (isFinal(mPrev)) // case (1) [-2:not space][-1:final letter][cur:space]
00126           ++mFinalCharLogicalScore;
00127         else if (isNonFinal(mPrev)) // case (2) [-2:not space][-1:Non-Final letter][cur:space]
00128           ++mFinalCharVisualScore;
00129       }
00130     }
00131     else  // Not standing on a space
00132     {
00133       if ((mBeforePrev == ' ') && (isFinal(mPrev)) && (cur != ' ')) // case (3) [-2:space][-1:final letter][cur:not space]
00134         ++mFinalCharVisualScore;
00135     }
00136     mBeforePrev = mPrev;
00137     mPrev = cur;
00138   }
00139 
00140   // Forever detecting, till the end or until both model probers return eNotMe (handled above).
00141   return eDetecting;
00142 }
00143 
00144 // Make the decision: is it Logical or Visual?
00145 const char* nsHebrewProber::GetCharSetName()
00146 {
00147   // If the final letter score distance is dominant enough, rely on it.
00148   PRInt32 finalsub = mFinalCharLogicalScore - mFinalCharVisualScore;
00149   if (finalsub >= MIN_FINAL_CHAR_DISTANCE) 
00150     return LOGICAL_HEBREW_NAME;
00151   if (finalsub <= -(MIN_FINAL_CHAR_DISTANCE))
00152     return VISUAL_HEBREW_NAME;
00153 
00154   // It's not dominant enough, try to rely on the model scores instead.
00155   float modelsub = mLogicalProb->GetConfidence() - mVisualProb->GetConfidence();
00156   if (modelsub > MIN_MODEL_DISTANCE)
00157     return LOGICAL_HEBREW_NAME;
00158   if (modelsub < -(MIN_MODEL_DISTANCE))
00159     return VISUAL_HEBREW_NAME;
00160 
00161   // Still no good, back to final letter distance, maybe it'll save the day.
00162   if (finalsub < 0) 
00163     return VISUAL_HEBREW_NAME;
00164 
00165   // (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
00166   return LOGICAL_HEBREW_NAME;
00167 }
00168 
00169 
00170 void nsHebrewProber::Reset(void)
00171 {
00172   mFinalCharLogicalScore = 0;
00173   mFinalCharVisualScore = 0;
00174 
00175   // mPrev and mBeforePrev are initialized to space in order to simulate a word 
00176   // delimiter at the beginning of the data
00177   mPrev = ' ';
00178   mBeforePrev = ' ';
00179 }
00180 
00181 nsProbingState nsHebrewProber::GetState(void) 
00182 {
00183   // Remain active as long as any of the model probers are active.
00184   if ((mLogicalProb->GetState() == eNotMe) && (mVisualProb->GetState() == eNotMe))
00185     return eNotMe;
00186   return eDetecting;
00187 }
00188 
00189 #ifdef DEBUG_chardet
00190 void  nsHebrewProber::DumpStatus()
00191 {
00192   printf("  HEB: %d - %d [Logical-Visual score]\r\n", mFinalCharLogicalScore, mFinalCharVisualScore);
00193 }
00194 #endif