root/lang/ruby/misc/tumblr_prefetch_images.rb @ 10725

#!/opt/local/bin/ruby -Ku

# tumblrのfeedに表示される画像をProxyにキャッシュさせるスクリプト
# http://d.bulkitem.com/20080310.html#p01
#

#$DEBUG = true
$VERBOSE = true

require 'open-uri'
require 'pstore'

module Tumblr
  module Cache
    FEED_LIST = "./prefetch.conf" #フィードのアドレスリスト
    CACHED    = "./cache/prefetch" #キャッシュを保存するファイル
    PROXY     = "http://localhost:8123/" # polipo (proxy)
    GROWL     = false #終了をGrowlで通知するかどうか
  end
end

Dir.chdir(File.dirname(__FILE__))


routes = []
uris = []
fresh = []
errors = []

puts 'start'
File.open(Tumblr::Cache::FEED_LIST) do |conf|
  conf.each_line do |uri|
    uri = uri.strip
    next if uri =~ /^#/
    next if uri == ""
    routes.push uri
  end
end

routes2 = routes.dup
routes.uniq!
(routes2 - routes).each do |redundant|
  printf "redundancy: %s\n", redundant
end

printf "total: %d\n", routes.size
routes.each_with_index do |route, idx|
  begin
    URI.extract(URI.parse(route).read, ['http']).each do |path|
      if path =~ /^http\:\/\/media\.tumblr\.com\/.+\.(jpg|jpeg|png|gif)$/
        uris.push path
      end
    end
    printf "%d/%d: %s.\n", idx+1, routes.size, route
  rescue Exception => e
    errors.push e.to_s
    printf "Error %d/%d: %s.\n", idx+1, routes.size, e.to_s
  end
end

db = PStore.new(Tumblr::Cache::CACHED)
db.transaction do
  cache = db['cache'] || Array.new
  fresh = uris - cache
  uris.each do |line|
    cache.push line
  end
  db['cache'] = cache.uniq
end

printf "total: %d\n", fresh.size
fresh.each_with_index do |resrc, idx|
  begin
    open(resrc, {:proxy => Tumblr::Cache::PROXY})# {|f| f.read }
    printf "%d/%d: %s.\n", idx+1, fresh.size, resrc
  rescue Exception => e
    errors.push e.to_s
    printf "Error %d/%d: %s.\n", idx+1, fresh.size, e.to_s
  end
end

puts "Done."

if errors.size > 0
  printf "Error: %d\n", errors.size
  errors.each_with_index do |error, idx|
    printf "%d/%d: %s.\n", idx+1, errors.size, error
  end
end


if Tumblr::Cache::GROWL
  require 'rubygems'
  require 'ruby-growl'
  Growl.new('localhost', "ruby-grow", ["ruby-growl Notification"]).notify("ruby-growl Notification", "Tumblr prefetcher", "Done.")
end