Changeset 11041 for lang/python

Timestamp:

05/04/08 00:56:21 (7 months ago)

Author:

ayu

Message:

• added TopicSegmentation? for Japanese text.
  

Location:

lang/python/yoyaku/yoyaku/engine

Files:

• HyokiyureData.py (added)
• TopicSegmentation.py (modified) (4 diffs)
• TopicSegmentationJP.py (added)

lang/python/yoyaku/yoyaku/engine/TopicSegmentation.py

@@ -22 +22 @@
         for j in range(len(tfs)):
             if i < j:
-                ans[i][j] = sum([tfs[i][x]*tfs[j][x] for x in set(tfs[i].keys())&set(tfs[j].keys())]) / (tflen[i]*tflen[j])
+                w = (tflen[i]*tflen[j])
+                if w != 0:
+                    ans[i][j] = sum([tfs[i][x]*tfs[j][x] for x in set(tfs[i].keys())&set(tfs[j].keys())]) / w
+                else:
+                    ans[i][j] = 0.0
             elif i > j:
                 ans[i][j] = ans[j][i]
@@ -79 +83 @@
     return w / float(a)
 
-def split_matrix_rec(mat):
+def split_matrix_rec(mat, evaluate_coeff=1.2):
     MAX_TOPIC = 50
     size = len(mat)
@@ -86 +90 @@
     orgssep = []
     D.append(mat[0][size-1]/float(size**2))
-    c = 1.2
     def evaluate(s):
         a = sum(s) / float(len(s))
         v = sum([(x-a)**2 for x in s])/float(len(s))
-        ans = a+c*math.sqrt(v)
+        ans = a+evaluate_coeff*math.sqrt(v)
         #print "evaluate", ans
         return ans
@@ -128 +131 @@
     return orgssep[:b]
 
-def segmentation(tfs, sharpen_window=5):
+def segmentation(tfs, sharpen_window=5, evaluate_coeff=1.2):
     matrix = precluster_matrix(sharpen_matrix(tfmatrix(tfs), window=sharpen_window))
-    return split_matrix_rec(matrix)
+    return split_matrix_rec(matrix, evaluate_coeff)