root/lang/ruby/twitter_bot/47news/headline.rb @ 30649

#!/usr/bin/ruby

# 47news.jpの地方記事一覧からスクレイピングしていってrssにして保存
# それと新たにスクレイピングしたものを比較していって新しければ
# twitterにupdate

require 'rexml/document'
require 'rubygems'
require 'scrapi'
require 'open-uri'
require 'nkf'
require 'pp'
require 'kconv'
require 'date'
require 'rss'
require 'rubygems'
gem 'twitter'
require 'twitter'


$KCODE = 'u'
DIR_ROOT = '/var/www/html/lab/twitter/47news/'

location = ARGV[0]
rss_path = ARGV[1]
mail     = ARGV[2]
user     = ARGV[3]

scraper = Scraper.define {
    process 'div.frameline > div.topContent > ul > li > a', "a[href]", "urls[]"=>"@href","titles[]"=>:text
    result :urls,:titles
}
begin
res = open('http://www.47news.jp/localnews/'+location+'/archives.php').read
rescue OpenURI::HTTPError
  exit!
end

html = NKF::nkf('-w -m0', res)

news =  scraper.scrape(html, :parser => :html_parser)

#comp

if news.titles && news.urls
    #RSS read
    local_rss = rss_path + location + '.xml' #RSS
    
    local_rss = open(local_rss){ |file| RSS::Parser.parse(file.read) }
    
    if news.titles.size > 0
        for i in 0 .. news.titles.size - 1
            if news.titles[i] != ''
                title = ''
                sflg = false
                if local_rss
                    local_rss.items.each { |item|
                        title = news.titles[i].gsub(/[　]$/,"");
                        if title == item.title
                            #同じのがある
                            sflg = true;
                        end
                    }
                end
                #同じのがなければそれを送信
                unless sflg
                    twit = Twitter::Base.new(mail,user)
                    p title.to_s
                    twit.update(Kconv.kconv(title.to_s + news.urls[i], Kconv::UTF8))
                    sleep 5
                end
            end
        end
    end
else
    p 'Not Found'
end


#RSS write
rss = RSS::Maker.make("1.0") do |maker|
    maker.channel.about = "http://www.47news.jp/rss"
    maker.channel.title = ""
    maker.channel.description = ""
    maker.channel.link = "http://www.47news.jp/"

    if news.titles && news.urls
        for i in 0 .. news.titles.size - 1
            if news.titles[i] != ''
                item = maker.items.new_item
                item.link = news.urls[i]
                item.title = news.titles[i].gsub(/[　]$/,"")
            end
        end
    end
end

fname = DIR_ROOT + "rss/" + location + ".xml"

File.open(fname, "w") {|file|
    file.puts(rss.to_s.toutf8)
}