Бывалый
Профиль
Группа: Участник
Сообщений: 218
Регистрация: 5.7.2007
Репутация: нет
Всего: нет
|
Вот что получилось, пожалуйста скажите что сделал не так, что лучше переделать, в общем буду рад любым конструктивным комментариям  . SYNOPSIS| Код |
#!/usr/bin/perl -w
use lib '.';
use strict;
use HTTP::Multitask;
use Data::Dumper;
my $urls = {
'http://bizmaticsinc.com/robots.txt' => 0,
'http://futureeast.com/robots.txt' => 0,
'http://gurudutt.com/robots.txt' => 0,
'http://hermeschem.com/robots.txt' => 0,
'http://malagabags.com/robots.txt' => 0,
'http://mribhatia.com/robots.txt' => 0,
'http://powericaltd.com/robots.txt' => 0,
'http://prognocis.com/robots.txt' => 0,
'http://rajeshpatkar.com/robots.txt' => 0,
'http://reflectash.com/robots.txt' => 0,
};
my $client = HTTP::Multitask->new(agent => 'Bot/0.1',
max_tasks => 150,
max_size => 10240,
max_open => 150,
max_per_host => 4,
keep_alive => 15,
delay => 1,
timeout => 10,
debug => 0,);
$client->head($urls);
print Dumper($urls);
|
| Код |
package HTTP::Multitask;
use strict;
use warnings;
use HTTP::Request;
use POE qw(Component::Client::HTTP Component::Client::Keepalive);
use Carp qw(croak);
sub new {
my $class = shift;
my %args = @_;
my $self = {};
bless($self, $class);
$self->{agent} = $self->agent($args{agent});
$self->{max_tasks} = $self->max_tasks($args{max_tasks});
$self->{max_size} = $self->max_size($args{max_size});
$self->{max_open} = $self->max_open($args{max_open});
$self->{max_per_host} = $self->max_per_host($args{max_per_host});
$self->{keep_alive} = $self->keep_alive($args{keep_alive});
$self->{delay} = $self->delay($args{delay});
$self->{timeout} = $self->timeout($args{timeout});
$self->{debug} = $self->debug($args{debug});
return $self;
}
sub head {
my $self = shift;
my $urls = shift;
unless (ref $urls eq 'HASH') {
croak("Argument of the method head should be hash reference");
return undef;
}
$self->{method} = 'HEAD';
$self->{urls} = $urls;
$self->_http_multitask();
return 1;
}
sub get {
my $self = shift;
my $urls = shift;
unless (ref $urls eq 'HASH') {
croak("Argument of the method get should be hash reference");
return undef;
}
$self->{method} = 'GET';
$self->{urls} = $urls;
$self->_http_multitask();
return 1;
}
sub _http_multitask {
my $self = shift;
my $pool = POE::Component::Client::Keepalive->new(
'keep_alive' => $self->{keep_alive},
'max_open' => $self->{max_open},
'max_per_host' => $self->{max_per_host},
'timeout' => $self->{timeout},
);
POE::Component::Client::HTTP->spawn(
'Alias' => 'HTTP_CLIENT',
'MaxSize' => $self->{max_size},
'Timeout' => $self->{timeout},
'Agent' => $self->{agent},
'ConnectionManager' => $pool,
);
POE::Session->create(
'inline_states' => {
'_start' => \&_sess_start,
'_stop' => \&_sess_stop,
'get_tasks' => \&_get_tasks,
'task_done' => \&_task_done,
},
'heap' => {
conf => {
max_tasks => $self->{max_tasks},
delay => $self->{delay},
debug => $self->{debug},
method => $self->{method}
},
urls => $self->{urls},
queue => [keys %{$self->{urls}}],
}
);
POE::Kernel->run;
}
sub _sess_start {
my ($kernel, $heap) = @_[KERNEL, HEAP];
print "_sess_start: STARTED\n" if $heap->{conf}->{debug};
$kernel->yield('get_tasks');
}
sub _sess_stop {
my ($kernel, $heap) = @_[KERNEL, HEAP];
print "_sess_stop: STOPPED\n" if $heap->{conf}->{debug};
}
sub _get_tasks {
my ($kernel, $heap) = @_[KERNEL, HEAP];
my $difference = $heap->{conf}->{max_tasks} - $kernel->call('HTTP_CLIENT' => 'pending_requests_count');
print "_get_tasks: DIFFERENCE $difference\n" if $heap->{conf}->{debug};
unless (($difference == $heap->{conf}->{max_tasks}) && (scalar(@{$heap->{queue}}) == 0)) {
$kernel->delay('get_tasks', $heap->{conf}->{delay});
if ($difference <= 0) {
print "_get_tasks: RETURN\n" if $heap->{conf}->{debug};
return;
}
my @tasks = splice(@{$heap->{queue}}, 0, $difference);
foreach my $task (@tasks) {
my $request = HTTP::Request->new($heap->{conf}->{method} => $task);
$kernel->post('HTTP_CLIENT' => 'request', 'task_done', $request);
}
print "_get_tasks: ADDED ".scalar(@tasks)."\n" if $heap->{conf}->{debug};
print "_get_tasks: REMAINS ".scalar(@{$heap->{queue}})."\n" if $heap->{conf}->{debug};
}
}
sub _task_done {
my ($heap, $request, $response) = ($_[HEAP], $_[ARG0]->[0], $_[ARG1]->[0]);
if ($heap->{conf}->{method} eq 'HEAD') {
$heap->{urls}->{$request->uri} = $response->code;
} else {
$heap->{urls}->{$request->uri} = $response->content;
}
print "_task_done: ".$response->code." (".$request->uri->host.")\n" if $heap->{conf}->{debug};
}
sub agent { shift->_param('agent', @_, 'HTTP-Multitask-perl/0.1'); }
sub max_tasks { shift->_param('max_tasks', @_, 150); }
sub max_size { shift->_param('max_size', @_, 524288); }
sub max_open { shift->_param('max_open', @_, 150); }
sub max_per_host { shift->_param('max_per_host', @_, 4); }
sub keep_alive { shift->_param('keep_alive', @_, 15); }
sub delay { shift->_param('delay', @_, 1); }
sub timeout { shift->_param('timeout', @_, 60*3); }
sub debug { shift->_param('debug', @_, 0); }
sub _param {
my ($self, $param, $value, $default) = @_;
$self->{$param} = defined $value ? $value : $default;
return $self->{$param};
}
1;
|
| Цитата |
$VAR1 = {
'http://mribhatia.com/robots.txt' => 200,
'http://malagabags.com/robots.txt' => 501,
'http://futureeast.com/robots.txt' => 200,
'http://hermeschem.com/robots.txt' => 500,
'http://bizmaticsinc.com/robots.txt' => 200,
'http://rajeshpatkar.com/robots.txt' => 200,
'http://reflectash.com/robots.txt' => 200,
'http://prognocis.com/robots.txt' => 200,
'http://gurudutt.com/robots.txt' => 200,
'http://powericaltd.com/robots.txt' => 200
};
|
Спасибо за внимание. Это сообщение отредактировал(а) nIkTo - 11.4.2010, 19:30
Присоединённый файл ( Кол-во скачиваний: 1 )
module.tgz 1,85 Kb |
Гляньте на код, укажите на проблемные места, HTTP Multitask на PoCo::Client::HTTP 
Дата 11.4.2010, 19:26 (
Загрузка ...
