root/lang/ruby/misc/tumblr_prefetch_images.rb @ 10725

Revision 10725, 2.2 kB (checked in by iskwrsk, 7 years ago)

Addition

  • Property svn:executable set to *
Line 
1#!/opt/local/bin/ruby -Ku
2
3# tumblrのfeedに表示される画像をProxyにキャッシュさせるスクリプト
4# http://d.bulkitem.com/20080310.html#p01
5#
6
7#$DEBUG = true
8$VERBOSE = true
9
10require 'open-uri'
11require 'pstore'
12
13module Tumblr
14  module Cache
15    FEED_LIST = "./prefetch.conf" #フィードのアドレスリスト
16    CACHED    = "./cache/prefetch" #キャッシュを保存するファイル
17    PROXY     = "http://localhost:8123/" # polipo (proxy)
18    GROWL     = false #終了をGrowlで通知するかどうか
19  end
20end
21
22Dir.chdir(File.dirname(__FILE__))
23
24
25routes = []
26uris = []
27fresh = []
28errors = []
29
30puts 'start'
31File.open(Tumblr::Cache::FEED_LIST) do |conf|
32  conf.each_line do |uri|
33    uri = uri.strip
34    next if uri =~ /^#/
35    next if uri == ""
36    routes.push uri
37  end
38end
39
40routes2 = routes.dup
41routes.uniq!
42(routes2 - routes).each do |redundant|
43  printf "redundancy: %s\n", redundant
44end
45
46printf "total: %d\n", routes.size
47routes.each_with_index do |route, idx|
48  begin
49    URI.extract(URI.parse(route).read, ['http']).each do |path|
50      if path =~ /^http\:\/\/media\.tumblr\.com\/.+\.(jpg|jpeg|png|gif)$/
51        uris.push path
52      end
53    end
54    printf "%d/%d: %s.\n", idx+1, routes.size, route
55  rescue Exception => e
56    errors.push e.to_s
57    printf "Error %d/%d: %s.\n", idx+1, routes.size, e.to_s
58  end
59end
60
61db = PStore.new(Tumblr::Cache::CACHED)
62db.transaction do
63  cache = db['cache'] || Array.new
64  fresh = uris - cache
65  uris.each do |line|
66    cache.push line
67  end
68  db['cache'] = cache.uniq
69end
70
71printf "total: %d\n", fresh.size
72fresh.each_with_index do |resrc, idx|
73  begin
74    open(resrc, {:proxy => Tumblr::Cache::PROXY})# {|f| f.read }
75    printf "%d/%d: %s.\n", idx+1, fresh.size, resrc
76  rescue Exception => e
77    errors.push e.to_s
78    printf "Error %d/%d: %s.\n", idx+1, fresh.size, e.to_s
79  end
80end
81
82puts "Done."
83
84if errors.size > 0
85  printf "Error: %d\n", errors.size
86  errors.each_with_index do |error, idx|
87    printf "%d/%d: %s.\n", idx+1, errors.size, error
88  end
89end
90
91
92if Tumblr::Cache::GROWL
93  require 'rubygems'
94  require 'ruby-growl'
95  Growl.new('localhost', "ruby-grow", ["ruby-growl Notification"]).notify("ruby-growl Notification", "Tumblr prefetcher", "Done.")
96end
Note: See TracBrowser for help on using the browser.