Changeset 9987

Timestamp:

04/20/08 16:52:19 (5 years ago)

Author:

hio

Message:

lang/perl/tiarra: 見出し抽出の追加, 200以外でも抽出を行えるように.

Files:

• lang/perl/tiarra/trunk/module/Auto/FetchTitle.pm (modified) (13 diffs)

lang/perl/tiarra/trunk/module/Auto/FetchTitle.pm

@@ -1045 +1045 @@
   my $config = [
     {
+      # 1. ぷりんと楽譜.
       url        => 'http://www.print-gakufu.com/*',
       recv_limit => 8*1024,
@@ -1051 +1052 @@
     },
     {
+      # 2. zakzak.
       url        => 'http://www.zakzak.co.jp/*',
       recv_limit => 10*1024,
@@ -1056 +1058 @@
     },
     {
+      # 3. nikkei.
       url        => 'http://www.nikkei.co.jp/*',
       recv_limit => 16*1024,
@@ -1065 +1068 @@
     },
     {
+      # 4. nhkニュース.
       url     => 'http://www*.nhk.or.jp/news/*',
       extract => qr{<p class="newstitle">(.*?)</p>},
     },
     {
+      # 5. creative (timeout).
       url     => 'http://*.creative.com/*',
       timeout => 5,
     },
     {
+      # 6. soundhouse news.
       url        => 'http://www.soundhouse.co.jp/shop/News.asp?NewsNo=*',
       recv_limit => 50*1024,
@@ -1078 +1084 @@
     },
     {
-      # trac changeset.
+      # 7. trac changeset.
       url        => '*/changeset/*',
       extract    => qr{<dd class="message" id="searchable"><p>(.*?)</p>}s,
     },
     {
+      # 8a. amazon (page size).
       url        => 'http://www.amazon.co.jp/*',
       recv_limit => 15*1024,
     },
     {
+      # 8b. amazon (page size).
       url        => 'http://www.amazon.com/*',
       recv_limit => 15*1024,
+    },
+    {
+      # 9. ニコニコ動画 (メンテ画面).
+      status     => 503,
+      url        => 'http://www.nicovideo.jp/*',
+      extract    => sub{
+        if( m{<div class="mb16p4 TXT12">\s*<p>現在ニコニコ動画は(メンテナンス中)です。</p>\s*<p>(.*?)<br />}s )
+        {
+          "$1: $2";
+        }else
+        {
+          return;
+        }
+      },
+    },
+    {
+      # 10. sanspo.
+      url        => 'http://www.sanspo.com/*',
+      recv_limit => 5*1024,
+      extract    => qr{<h2>(.*?)</h2>}s,
+    },
+    {
+      # 11. sakura.
+      url        => 'http://www.sakura.ad.jp/news/archives/*',
+      recv_limit => 10*1024,
+      extract    => qr{<h3 class="newstitle">(.*?)</h3>}s,
     },
   ];
@@ -1107 +1141 @@
   my $type  = shift;
 
-  my $conflist = $this->_extract_heading_config();
-
-  foreach my $conf (@$conflist)
+  my $extract_list = $this->_extract_heading_config();
+
+  foreach my $conf (@$extract_list)
   {
     Mask::match($conf->{url}, $req->{url}) or next;
@@ -1147 +1181 @@
     return;
   }
-  if( $req->{result}{status_code}!=200 )
-  {
-    $DEBUG and $this->_debug($req, "debug: - - skip/not success:$req->{result}{status_code}");
-    return;
-  }
-
-  my $conflist = $this->_extract_heading_config();
+  my $status = $req->{result}{status_code};
+
+  my $extract_list = $this->_extract_heading_config();
 
   my $heading;
 
-  foreach my $conf (@$conflist)
+  foreach my $conf (@$extract_list)
   {
     Mask::match($conf->{url}, $req->{url}) or next;
     $DEBUG and $this->_debug($req, "debug: - $conf->{url}");
+
+    my $extract_status = $conf->{status} || 200;
+    if( $status != $extract_status )
+    {
+      $DEBUG and $this->_debug($req, "debug: - - status:$status not match with $extract_status");
+      next;
+    }
 
     my $extract_list = $conf->{extract};
@@ -1172 +1209 @@
       my $extract = $_extract; # sharrow-copy.
       $extract = ref($extract) ? $extract : qr/\Q$extract/;
-      my @match = $req->{result}{decoded_content} =~ $extract;
+      my @match;
+      if( ref($extract) eq 'CODE' )
+      {
+        local($_) = $req->{result}{decoded_content};
+        @match = $extract->($req);
+      }else
+      {
+        @match = $req->{result}{decoded_content} =~ $extract;
+      }
       @match or next;
+      @match==1 && !defined($match[0]) and next;
       $heading = $match[0];
       last;
@@ -1419 +1465 @@
     content_length => undef,
     decoded_content => undef,
+    fetch_length    => undef,
   };
 
@@ -1445 +1492 @@
   my $headers     = $res->{Header}; # hash-ref.
   my $content     = $res->{Content};
+  $result->{fetch_length} = defined($content) ? length($content) : undef;
   defined($content) or $content = '';
+  my @opts;
 
   $result->{status_code}    = $status_code;
@@ -1483 +1532 @@
   if( int($status_code / 100) != 2 && !$result->{redirect} )
   {
-    my @opts;
-    $status_msg and push(@opts, $status_msg);
+    $result->{title} = $status_msg;
     push(@opts, "http status $status_code");
-    if( $req->{redirected} )
-    {
-      my $redirs = $req->{redirected}==1 ? 'redir' : 'redirs';
-      push(@opts, "$req->{redirected} $redirs");
-    }
-    my $reply = shift @opts;
-    if( @opts )
-    {
-      $reply .= " (".join("; ", @opts).")";
-    }
-    $result->{result} = $reply;
-    return $result;
   }
 
@@ -1559 +1595 @@
     $title = $this->_fixup_title($title);
     $result->{title} = $title;
+  }else
+  {
+    $title = $result->{title};
   }
 
@@ -1595 +1634 @@
   }
 
-  my @opts;
   if( $reply eq '' || $ctype !~ /html/ )
   {