Changeset 6288

Timestamp:

02/06/08 15:58:41 (10 months ago)

Author:

tomi-ru

Message:

もう堂々とlabs.unoh.netからscrapeするということにする

Location:

lang/perl/Encode-JP-Mobile/trunk

Files:

• Rakefile (modified) (1 diff)
• tools/BUILD (modified) (1 diff)
• tools/convert-map-scrape.pl (moved) (moved from lang/perl/Encode-JP-Mobile/trunk/tools/make-convert-map.pl) (3 diffs)

lang/perl/Encode-JP-Mobile/trunk/Rakefile

@@ -40 +40 @@
 end
 
-unoh_files = %w(e2is i2es s2ie)
-file 'dat/convert-map-utf8.yaml' => unoh_files.map {|x| "dat/conv/emoji_#{x}.txt" } do
-    sh "#{perl} tools/make-convert-map.pl > dat/convert-map-utf8.yaml"
-end
-directory 'dat/conv/'
-unoh_files.each do |f|
-    file "dat/conv/emoji_#{f}.txt" => ['dat/conv/'] do
-        sh "wget http://labs.unoh.net/emoji_#{f}.txt -O dat/conv/emoji_#{f}.txt"
-    end
+file 'dat/convert-map-utf8.yaml' do
+    sh "#{perl} tools/convert-map-scrape.pl > dat/convert-map-utf8.yaml"
 end
 

lang/perl/Encode-JP-Mobile/trunk/tools/BUILD

@@ -22 +22 @@
 
 # Optional: build convert-utf8-map.yaml
-# ./tools/make-convert-map.pl
+# ./tools/convert-map-scrape.pl
 
 # Make x-utf8-*.ucm

lang/perl/Encode-JP-Mobile/trunk/tools/convert-map-scrape.pl

@@ -2 +2 @@
 
 # http://labs.unoh.net/2007/02/post_65.html to dat/convert-map-utf8.yaml
-# mkdir dat/conv; download emoji_*.txt to dat/conv/emoji_*.txt
-# perl tools/make-convert-map.pl > dat/convert-map-utf8.yaml
+# perl tools/convert-map-scrape.pl > dat/convert-map-utf8.yaml
 
-use utf8;
 use Encode;
 use Encode::JP::Mobile 0.09;
-use File::Slurp qw(slurp);
+use LWP::Simple;
 use YAML;
 
+my %files;
+for my $file (qw( emoji_e2is.txt emoji_i2es.txt emoji_s2ie.txt )) {
+    $files{$file} = decode('cp932', get("http://labs.unoh.net/$file"));
+}
+
 my $no2uni = {};
-for my $file (qw( emoji_e2is.txt emoji_i2es.txt emoji_s2ie.txt )) {
-    my @line = slurp "dat/conv/$file";
-    for my $line (@line) {
+for my $file (keys %files) {
+    for my $line (split /\n/, $files{$file}) {
         next unless $line =~ /^%/;
         my ($no, $byte) = split "\t", $line;
@@ -33 +35 @@
 
 my %map;
-for my $file (qw( emoji_e2is.txt emoji_i2es.txt emoji_s2ie.txt )) {
-    my @line = slurp "dat/conv/$file";
-    
-    for my $line (@line) {
+for my $file (keys %files) {
+    for my $line (split /\n/, $files{$file}) {
         next unless $line =~ /^%/;
         chomp $line;
-        $line = decode 'cp932', $line;
 
         $file eq 'emoji_i2es.txt' && do {
@@ -71 +70 @@
 }
 
-print YAML::Dump \%map;
+binmode STDOUT, ":utf8";
+print YAML::Dump(\%map);