root/lang/php/ZendFramework_ext/library/Twk/Search/Lucene/Analysis/Analyzer/Common/Utf8MbcsUnigram.php

<?php
/**
 * Multi Byte Character uni-gram Analyzer
 * like Java CJKAnalyzer
 * 
 * @version 20080616
 * @author twk (http://nonn-et-twk.net/twk/)
 */

// Requires analyzer.php
// @see http://framework.zend.com/issues/browse/ZF-991
require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';

/** Zend_Search_Lucene_Analysis_Analyzer_Common */
//require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';


/**
 * @category   Twk
 * @package    Twk_Search_Lucene
 * @subpackage Analysis
 * @license    http://framework.zend.com/license/new-bsd     New BSD License
 */

class Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram extends Zend_Search_Lucene_Analysis_Analyzer_Common
{
    /**
     * Current char position in an UTF-8 stream
     *
     * @var integer
     */
    private $_position;

    /**
     * Current binary position in an UTF-8 stream
     *
     * @var integer
     */
    private $_bytePosition;

    private $_lastPosition;
    private $_lastBytePosition;
    
    /**
     * Stream length
     *
     * @var integer
     */
    private $_streamLength;

    const CATEGORY_ALNUM = 1;
    const CATEGORY_PUNCT = 2;
    const CATEGORY_CNTRL = 3;
    const CATEGORY_SPACE = 4;
    const CATEGORY_SBCS_OTHER = 5;
    const CATEGORY_MBCS = 0x10;
    const CATEGORY_HIRAGANA = 0x11;
    const CATEGORY_KATAKANA = 0x12;
    const CATEGORY_JAPANESE_KANJI = 0x13;
    const CATEGORY_FULLWIDTH_ALNUM = 0x14;
    const CATEGORY_JAPANESE_PUNCT = 0x14;
    const CATEGORY_MBCS_OTHER = 0x15;

    /**
     * Reset token stream
     */
    public function reset()
    {
        $this->_position     = 0;
        $this->_bytePosition = 0;
        
        $this->_lastPosition = 0;
        $this->_lastBytePosition = 0;

        // convert input into UTF-8
        if (strcasecmp($this->_encoding, 'utf8' ) != 0  &&
            strcasecmp($this->_encoding, 'utf-8') != 0 ) {
                $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input);
                $this->_encoding = 'UTF-8';
        }

        // Get UTF-8 string length.
        // It also checks if it's a correct utf-8 string
        $this->_streamLength = iconv_strlen($this->_input, 'UTF-8');
    }

    /**
     * @return CATEGORY_xxx
     */
    private static function _getCharCategory($char)
    {
        if (!self::_isMbcs($char))
        {
            if (ctype_alnum($char))
                $category = self::CATEGORY_ALNUM;
            else if (ctype_punct($char))
                $category = self::CATEGORY_PUNCT;
            else if (ctype_cntrl($char))
                $category = self::CATEGORY_CNTRL;
            else if (ctype_space($char))
                $category = self::CATEGORY_SPACE;
               else
                $category = self::CATEGORY_SBCS_OTHER;
        }
        else
        {
            if (extension_loaded('mbstring'))
            {
                // @see http://ablog.seesaa.net/article/20969848.html
                // 記号は代表的な物のみ指定している。pregと/u使っても書けそう
                if (mb_ereg_match('^[ぁ-ん]+$', $char))
                    $category = self::CATEGORY_HIRAGANA;
                else if (mb_ereg_match('^[ァ-ヴー]+$', $char))
                    $category = self::CATEGORY_KATAKANA;
                else if (mb_ereg_match('^[一-龠々〆ヵヶ]+$', $char))
                    $category = self::CATEGORY_JAPANESE_KANJI;
                else if (mb_ereg_match('^[、。！？（）「」『』【】]+$', $char))
                    $category = self::CATEGORY_JAPANESE_PUNCT;
                else if (mb_ereg_match('^[ａ-ｚＡ-Ｚ０-９]+$', $char))
                    $category = self::CATEGORY_FULLWIDTH_ALNUM;
                else
                    $category = self::CATEGORY_MBCS_OTHER;
            }
            else
            {
                $category = self::CATEGORY_MBCS_OTHER;
            }
        }
        
        return $category;
    }
    
    /**
     * Check, how that character is 
     *
     * @param string $char
     * @return boolean
     */
    private static function _isWhiteSpaceCategory($char)
    {
        static $whiteSpaceCategories = array(self::CATEGORY_PUNCT, self::CATEGORY_CNTRL, self::CATEGORY_SPACE, self::CATEGORY_JAPANESE_PUNCT);
        
        $category = self::_getCharCategory($char);
        //echo $category, "\n";
        
        return in_array($category, $whiteSpaceCategories);
    }
    
    /**
     * Get next UTF-8 char
     * _position and _bytePosition are modified
     * last positions are stored to _lastPosition and _lastBytePosition
     *
     * @param string $char
     * @return boolean
     */
    private function _nextChar()
    {
        $this->_lastBytePosition = $this->_bytePosition;
        $this->_lastPosition = $this->_position;
        
        $char = $this->_input[$this->_bytePosition++];
        
        $ordChar = ord($char);
        if (( $ordChar & 0xC0 ) == 0xC0) {
            $addBytes = 1;
            if ($ordChar & 0x20 ) {
                $addBytes++;
                if ($ordChar & 0x10 ) {
                    $addBytes++;
                }
            }
            $char .= substr($this->_input, $this->_bytePosition, $addBytes);
            $this->_bytePosition += $addBytes;
        }

        $this->_position++;

        return $char;
    }

    private function _undoLastChar()
    {
        $this->_bytePosition = $this->_lastBytePosition;
        $this->_position = $this->_lastPosition;
    }

    /**
     * @return boolean
     */
    private static function _isMbcs($char)
    {
        return strlen($char) > 1;
    }
    
    /**
     * Tokenization stream API
     * Get next token
     * Returns null at the end of stream
     *
     * @return Zend_Search_Lucene_Analysis_Token|null
     */
    public function nextToken()
    {
        if ($this->_input === null) {
            return null;
        }

        while ($this->_position < $this->_streamLength) {
            // skip white spaces
            while ($this->_position < $this->_streamLength &&
                   self::_isWhiteSpaceCategory($char = $this->_nextChar())) {
                $char = '';
            }
            if ($char === '') {
                return null;
            }
            
            $wasMbcs = $this->_isMbcs($char);
               $this->_undoLastChar();
            // echo $this->_position, '=', $char;
            
            // read token
            $termStartPosition = $this->_position;
            $termText = '';
            $isTokenDivided = false;
            while ($this->_position < $this->_streamLength) {
                $char = $this->_nextChar();
                if (self::_isWhiteSpaceCategory($char))
                {
                    $isTokenDivided = true;
                    break;
                }
                else if (!$wasMbcs)
                {
                    if ($this->_isMbcs($char))
                    {
                        $isTokenDivided = true;
                        break;
                    }
                }
                else
                {
                    if (!$this->_isMbcs($char) || iconv_strlen($termText, 'UTF-8') >= 1)
                    {
                        $isTokenDivided = true;
                        break;
                    }
                }
                $termText .= $char;
            }
            
            // Empty token, end of stream.
            if ($termText == '') {
                return null;
            }
            
            // if not End, undo the last char
            // echo $this->_position ,"<=", $this->_streamLength;
            if ($isTokenDivided) {
                $this->_undoLastChar();
            }
            
            $termEndPosition = $this->_position;
            $token = new Zend_Search_Lucene_Analysis_Token(
                                      $termText,
                                      $termStartPosition,
                                      $termEndPosition);
            $token = $this->normalize($token);
            if ($token !== null) {
                // echo "$termText $termStartPosition $termEndPosition"
                return $token;
            }
            
            // Continue if token is skipped
        }

        return null;
    }
}


// ここから先は直接実行の時のみ動きます。チェックは甘め
if (!count(debug_backtrace()))
{
    $fp = fopen(__FILE__, 'r');
    fseek($fp, __COMPILER_HALT_OFFSET__ + 2); // 2 for ? and >
    echo stream_get_contents($fp);
    
    $text = '日本でWindows 95が発売されたのはいつでしょう?';
    $analyzer = new Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram();
    $a = $analyzer->tokenize($text, 'UTF-8');
    print_r($a);
}

__halt_compiler();?>
<html>
<head>
<title>Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram</title>
</head>
<body>
<h1>Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram</h1>
<pre>
    Zend_Search_Lucene用日本語アナライザー
    作者 twk

    <a href="http://coderepos.org/share/browser/lang/php/ZendFramework_ext/library/Twk/Search/Lucene/Analysis/Analyzer/Common/">配布場所</a>
    
    コード例:
    <code>
        $text = '日本でWindows 95が発売されたのはいつでしょう?';
        $analyzer = new Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram();
        $a = $analyzer-&gt;tokenize($text, 'UTF-8');
        print_r($a);    
    </code>
</pre>
</body>
</html>