root/lang/python/yoyaku/yoyaku/engine/TopicSegmentationJP.py

#!/usr/local/Python25/bin/python
# _*_ coding: utf-8 _*_
# Copyright (C)  2007 Ayukawa Hiroshi
from collections import defaultdict
import re
import sys
import MeCab
import math

from yoyaku.engine import TopicSegmentation
from yoyaku.engine.HyokiyureData import HYOKIYURE
    
def sentence_generator(tagger, src, mecabencoding="utf-8"):
    MARU = u"。".encode(mecabencoding)
    node = tagger.parseToNode(src)
    nodes = []
    while node:
        nodes.append(node)
        if node.surface == MARU:
            yield nodes
            nodes = []
        node = node.next
    if nodes: yield nodes

def stopworddetector(mecabencoding="utf-8"):
    NUM_CHAR = re.compile(ur"^[０-９0-9\-\.]+$")
    MEISHI = u"名詞".encode(mecabencoding)
    MICHI = u"未知".encode(mecabencoding)
    DOUSHI = u"動詞".encode(mecabencoding)
    HIRA = re.compile(u"^[ぁ-ん]+$")
    STOPWORD = [u"（", u"(", u"）", u")", u"等", u"同", u".", u"/", u"その後"]
    def hantei(node):
        w = unicode(node.surface, mecabencoding, "ignore")
        stop = not w.replace(u"。", u"") in STOPWORD
        hira = not HIRA.match(w)
        hinshi = node.feature.startswith(MEISHI) or node.feature.startswith(MICHI) or node.feature.startswith(DOUSHI)
        num = not NUM_CHAR.match(w)
        return node.surface and hinshi and hira and num and stop
    return hantei

KANJI = re.compile(u"^[一-龠]")
def gettf(sentencenodes, swdetect, mecab_encoding):
    DOUSHI = u"動詞".encode(mecab_encoding)
    tf = defaultdict(int)
    preword = None
    for node in sentencenodes:
        #print node.feature
        word = unicode(node.feature.split(",")[6], mecab_encoding, "ignore")
        if word == u"*":
            word = unicode(node.surface, mecab_encoding, "ignore")
        word = HYOKIYURE.get(word, word)
        preword = word
        if swdetect(node):
            if node.feature.startswith(DOUSHI):
                tf[word] += 0.5
            else:
                tf[word] += 1
                for w in word:
                    if KANJI.match(w):
                        tf[w] += 0.3
    #for k in tf.keys():
    #    print "TF", k, "=", tf[k]
    return tf

def test(window, me, ie, oe, evaluate_coeff):
    lines = []
    tfs = []
    MARU = u"。".encode(ie)
    for line in sys.stdin:
        line = line.strip()
        if line:
            if not line.endswith(MARU):
                line += MARU
            lines.append(line)
    body = "".join(lines)
    tagger = MeCab.Tagger()
    gen = sentence_generator(tagger, unicode(body, ie, "ignore").encode(me, "ignore"), me)
    swdetect = stopworddetector(me)
    lines = []
    for sentence in gen:
        tf = gettf(sentence, swdetect, me)
        line = "".join([x.surface for x in sentence])
        tfs.append(tf)
        lines.append(line)
    df = defaultdict(float)
    for tf in tfs:
        for k in tf.keys():
            df[k] += 1.0
    n = float(len(tfs))
    for tf in tfs:
        for k in tf.keys():
            tf[k] *= math.log(n/df[k])
    #for k in sorted(df.keys(), key=lambda x: df[x]):
    #    print "DF", k.encode(oe, "ignore"), math.log(n/df[k])
    segs = TopicSegmentation.segmentation(tfs, window, evaluate_coeff)
    segs.sort()
    print "="*20
    print "      RESULT"
    print "="*20
    for i, line in enumerate(lines):
        if i in segs:
            print "-"*20
        print unicode(line, me, "ignore").encode(oe, "ignore")
    

if __name__ == "__main__":
    import optparse
    
    parser = optparse.OptionParser(u"""
    標準入力から入力された、一行一文の日本語文テキストをトピックごとに分割します。
    """)
    parser.add_option("-w", "--window", dest="window", help=u"sharpenmatrix作成時の幅係数", type="int", default=2)
    parser.add_option("-e", "--evaluate", dest="c", help=u"分割振幅係数", type="float", default=0.05)
    parser.add_option("-m", "--mecab-encoding", dest="mecab", help=u"mecab辞書のエンコーディング default: utf-8", default="utf-8")
    parser.add_option("-i", "--input-encoding", dest="input", help=u"入力のエンコーディング default: utf-8", default="utf-8")
    parser.add_option("-o", "--output-encoding", dest="output", help=u"出力のエンコーディング default: utf-8", default="utf-8")

    (options, args) = parser.parse_args()
    
    test(options.window, options.mecab, options.input, options.output, options.c)