Show
Ignore:
Timestamp:
09/13/08 16:15:19 (6 years ago)
Author:
lopnor
Message:

lang/perl/App-Hachero: hadoop streaming support, but not tested yet.

Location:
lang/perl/App-Hachero/trunk
Files:
8 added
7 modified

Legend:

Unmodified
Added
Removed
  • lang/perl/App-Hachero/trunk/config.yaml

    r18806 r19262  
    1 global: 
    2   work_path: t/work 
    3  
    41plugins: 
    52 
     
    1411        time_zone: Asia/Tokyo 
    1512 
    16   - module: Analyze::AccessCount 
    17  
    18   - module: Output::DBIC 
    19     config: 
    20         connect_info: 
    21             - dbi:mysql:test 
     13  - module: OutputLine::HadoopMap 
     14 
     15#  - module: Analyze::AccessCount 
     16# 
     17#  - module: Output::DBIC 
     18#    config: 
     19#        connect_info: 
     20#            - dbi:mysql:test 
  • lang/perl/App-Hachero/trunk/lib/App/Hachero.pm

    r19223 r19262  
    1818 
    1919sub new { 
    20     my $class = shift; 
    21     my $args = $_[0]; 
     20    my ($class,$args) = @_; 
     21    $class->_setup_plugins_static($args); 
     22    my $self = $class->SUPER::new($args); 
     23    $self->result({}); 
     24    $context = $self; 
     25    $self; 
     26} 
     27 
     28sub _setup_plugins_static { 
     29    my ($class, $args) = @_; 
    2230    my $config = $class->setup_config( $args->{config} ); 
    2331 
     
    3644        $packages_from_plugin_path = $collect->modules; 
    3745    } 
    38  
    3946    $class->load_plugins(@plugins); 
    40  
    41     my $self = $class->SUPER::new(@_); 
    42     $self->result({}); 
    43     $context = $self; 
    44     $self; 
    4547} 
    4648 
     
    5355sub run { 
    5456    my $self = shift; 
     57    $self->log(debug => sprintf ('run start: %s', scalar localtime)); 
    5558    $self->initialize; 
    5659    $self->run_hook('fetch'); 
     
    5962        $self->run_hook_and_check('classify') or next; 
    6063        $self->run_hook_and_check('analyze') or next; 
     64        $self->run_hook('output_line'); 
    6165    } 
    6266    $self->run_hook('associate'); 
    6367    $self->run_hook('output'); 
     68    $self->log(debug => sprintf ('run end: %s', scalar localtime)); 
    6469} 
    6570 
  • lang/perl/App-Hachero/trunk/lib/App/Hachero/Plugin/Analyze/AccessCount.pm

    r18806 r19262  
    44use base qw(App::Hachero::Plugin::Base); 
    55use DateTime::Format::MySQL; 
     6use Digest::MD5; 
    67 
    78sub analyze : Hook { 
    89    my ($self, $context, $args) = @_; 
    910    my $req = $context->currentinfo->{request} or return; 
     11    my $truncate = $self->config->{config}->{truncate_to} || 'minute'; 
    1012    my $time = DateTime::Format::MySQL->format_datetime( 
    11         $req->{datetime}->clone->truncate(to => 'minute') 
     13        $req->{datetime}->clone->truncate(to => $truncate) 
    1214    ); 
    13     $context->result->{'AccessCount'}->{$time} = { 
     15    my $hash = Digest::MD5::md5_base64($time); 
     16    $context->result->{'AccessCount'}->{$hash} = { 
    1417        datetime => $time, 
    15         count => ($context->result->{'AccessCount'}->{$time}->{count} || 0) + 1, 
     18        count => ($context->result->{'AccessCount'}->{$hash}->{count} || 0) + 1, 
    1619    } 
    1720} 
  • lang/perl/App-Hachero/trunk/lib/App/Hachero/Plugin/Input/File.pm

    r18891 r19262  
    44use base qw(App::Hachero::Plugin::Base); 
    55use File::Find::Rule; 
     6use File::stat; 
    67 
    78sub init { 
     
    1213    my ($self, $context) = @_; 
    1314    $self->{rule} ||= $self->_get_rule($context); 
    14     my $file; 
    15     while (1) { 
    16         $file = $self->{rule}->match or return; 
    17         $file !~ /^\.{1,2}/ and last; 
     15    my $file = $self->{rule}->match or return; 
     16 
     17    if ($context->conf->{global}->{log}->{level} eq 'debug') { 
     18        $context->log(debug => "opening file: $file"); 
     19        $self->{readsize} = 0; 
     20        $self->{filesize} = stat($file)->size; 
    1821    } 
    1922    open my $fh, '<', $file or die; 
     
    3437        my $fh = $self->{fh} || $self->_fetch($context) or return; 
    3538        $line = <$fh>; 
     39        if ($context->conf->{global}->{log}->{level} eq 'debug') { 
     40            $self->{readsize}+= do {use bytes; length $line} if $line; 
     41            print STDERR "\rreading file: $self->{readsize}/$self->{filesize}"; 
     42        } 
    3643        unless ($line) { 
    3744            close $self->{fh}; 
  • lang/perl/App-Hachero/trunk/t/plugin/analyze/02_accesscount.t

    r18891 r19262  
    1414        { 
    1515            module => 'Analyze::AccessCount', 
    16             config => { 
    17  
    18             }, 
     16            config => {truncate_to => 'hour'}, 
    1917        } 
    2018    ] 
     
    3533    $app->run_hook('parse'); 
    3634    $app->run_hook('analyze'); 
    37     is_deeply $app->result->{'AccessCount'}, $block->expected; 
     35    my $value = (values %{$app->result->{AccessCount}})[0]; 
     36    is_deeply $value, $block->expected; 
    3837} 
    3938 
     
    5655 
    5756--- expected 
    58 '2008-08-07 00:00:00':  
    59     datetime: '2008-08-07 00:00:00' 
    60     count: 1 
     57datetime: '2008-08-07 00:00:00' 
     58count: 1 
    6159 
    6260=== test2 
     
    7573 
    7674--- expected 
    77 '2008-08-06 15:05:00': 
    78     datetime: '2008-08-06 15:05:00' 
    79     count: 1 
     75datetime: '2008-08-06 15:00:00' 
     76count: 1 
     77 
  • lang/perl/App-Hachero/trunk/t/plugin/input/01_stdin.t

    r18891 r19262  
    99 
    1010{ 
    11     my $line = 'hogehoge'; 
     11    my $line = 'hoge'; 
    1212    my ($fh, $log) = File::Temp::tempfile; 
    1313    print $fh $line; 
  • lang/perl/App-Hachero/trunk/t/plugin/input/03_file.t

    r19113 r19262  
    1010 
    1111my $config = { 
    12     global => {work_path => 't/work'}, 
     12    global => { 
     13        work_path => 't/work', 
     14        log => {level => 'error'}, 
     15    }, 
    1316    plugins => [ 
    1417        {