Changeset 14401 for lang/python

Timestamp:

06/22/08 08:57:12 (5 months ago)

Author:

ayu

Message:

• completed
  

Files:

• lang/python/incsearch/incsearch/indexer.py (modified) (2 diffs)

lang/python/incsearch/incsearch/indexer.py

@@ -7 +7 @@
 from  incsearch.db import DocDbWriter
 
+TITLE = re.compile(r'<title>([^<]+)</title>')
+TITLENUM = re.compile('^[0-9\. ]+')
 FUNC = re.compile(r'<tt id[^>]*>([^<]+)</tt>')
 def extractor(cont):
+    t = TITLE.search(cont)
+    title = ""
+    if t:
+        title = unicode(t.group(1), "euc_jp", "ignore").encode("utf-8", "ignore")
+    titlehead = TITLENUM.sub("", title)
+    yield dict(pos=None, title=title, word=titlehead.lower())
     scanner = FUNC.scanner(cont)
     m = scanner.search()
     while m:
-        yield dict(pos=m.start(), title=m.group(1), word=m.group(1))
+        yield dict(pos=m.start(), title=m.group(1)+" - "+titlehead, word=m.group(1).lower())
         m = scanner.search()
 
@@ -44 +52 @@
         pre = 0
         for i, ipoint in enumerate(extractor(whole)):
-            ipoint.update({"fname": fname})
-            db.put(ipoint["word"], "%s\t%d\t%s" % (fname, ipoint["pos"], ipoint["title"]))
-            nametag = "<a name='incsearch_%d'/>" % i
-            tagged.append(whole[pre:ipoint["pos"]])
-            tagged.append(nametag)
-            pre = ipoint["pos"]
+            if ipoint["pos"]:
+                db.put(ipoint["word"], "%s\t%d\t%s" % (fname, i, ipoint["title"]))
+                nametag = "<a name='incsearch_%d'/>" % i
+                tagged.append(whole[pre:ipoint["pos"]])
+                tagged.append(nametag)
+                pre = ipoint["pos"]
+            else:
+                db.put(ipoint["word"], "%s\t-\t%s" % (fname, ipoint["title"]))
         tagged.append(whole[pre:])
         gp = file(os.path.join(outdir, os.path.basename(fname)), "w")