root/lang/perl/Net-Twitter-Scraper/trunk/lib/Net/Twitter/Scraper.pm @ 14505

package Net::Twitter::Scraper;

use warnings;
use strict;
our $VERSION = '0.00';

use Carp qw/croak/;
use DateTime::Format::W3CDTF;
use HTTP::Status;
use LWP::UserAgent;
use URI;
use Web::Scraper;

sub new {
  my($class, %opt) = @_;

  my $username = $opt{username} || undef;
  my $password = $opt{password} || undef;

  my $ua = LWP::UserAgent->new(
    cookie_jar => { },
    keepalive => 1,
    requests_redirectable => ['GET', 'POST']
  );

  my $self = bless {
    ua => $ua,
    username => $username,
    password => $password,
    is_login => 0,
    response_code => undef,
    response_message => undef,
  }, $class;

  $self->_login($username, $password)
    if defined $username && defined $password;

  $self;
}

sub _login {
  my($self, $username, $password) = @_;

  $self->{is_login} = 0;
  my $resp = $self->{ua}->post(
    'https://twitter.com/sessions',
    { username_or_email => $username, password => $password }
  );

  $resp->is_success && $resp->request->uri->as_string eq 'https://twitter.com/home'
    or croak "Can't login twitter.com";

  $self->{is_login} = 1;
}

sub _set_status {
  my($self, $code) = @_;

  $self->{response_code} = $code;
  $self->{response_message} = HTTP::Status::status_message($code);
}

sub _format_datetime {
  my($str) = @_;

  my $dt = DateTime::Format::W3CDTF->new->parse_datetime($str);
  $dt->strftime('%a %b %d %H:%M:%S %z %Y');
}

sub http_code {
  my($self) = @_;

  return $self->{response_code};
}

sub http_message {
  my($self) = @_;

  return $self->{response_message};
}

# todo: since, since_id
sub archive {
  my($self, $args) = @_;

  $self->{is_login}
    or croak "Can't call the operation without login";

  my $scraper = scraper {
    process '//table[@id="timeline"]/tr', 'statuses[]' => scraper {
      process '//abbr[@class="published"]',
        created_at => [ '@title', \&_format_datetime ];
##      process '//', favorited => '';
      process '//.',
        id => [ '@id', sub { $_[0] =~ /status_(\d+)/, $1 } ];
##      process '//', in_reply_to_status_id => '';
##      process '//', in_reply_to_user_id => '';
      process '//span[@class="meta entry-meta"]/a[2]',
        source => sub { $_->as_HTML };
      process '//span[@class="entry-content"]',
        text => [ 'text', sub { $_[0] =~ /^\s*(.*?)\s*$/, $1 } ];
##      process '//', truncated => '';
##      process '//', user => '';
    };
  };
  $scraper->user_agent($self->{ua});

  my $url = 'http://twitter.com/account/archive';
  my $page = defined $args && defined $args->{page} ? $args->{page} : 1;
  my @statuses;
  for (my $i = $page; $i <= 4 + $page; ++$i) {
    my $res;
    eval {
      $res = $scraper->scrape(URI->new($url . '?page=' . $i));
    };
    if ($@) {
      # ;-)
      $self->_set_status(500);
      return undef;
    }
    push @statuses, @{$res->{statuses}};
  }

  $self->_set_status(200);

  \@statuses;
}

1;

=head1 NAME

Net::Twitter::Scraper - Twitter Scraper

=head1 SYNOPSIS

  use YAML;
  use Net::Twitter::Scraper;

  my $tw = Net::Twitter::Scraper->new(
    username => 'your id',
    password => 'your pswd'
  );

  my $archive = $tw->archive();
  binmode STDOUT, ':utf8';
  print Dump $archive;

=head1 DESCRIPTION


=head1 SEE ALSO

L<Net::Twitter>

=head1 NOTES

Alpha quality

=head1 AUTHOR

hetappi E<lt>hetappi.pm at gmail.comE<gt>

=head1 COPYRIGHT AND LICENSE


=cut