#!/usr/bin/env perl

#use Smart::Comments;

use strict;
use warnings;

use FindBin;
use lib "$FindBin::Bin/../lib";
use WWW::Prefetcher;
use Getopt::Std;

my %opts;
getopts('x:m:', \%opts) or die "Usage: $0 -x <proxy> -m <timeout>\n";

local $/;
my $txt = <>;
my @urls = split /\s*\n\s*/, $txt;

#my $proxy = '10.62.164.60:3128';
my $proxy = $opts{x} or die "No proxy specified.\n";
if ($proxy !~ m{^http://}) { $proxy = "http://$proxy" }
my $timeout = $opts{m} || 180;
#die $proxy;
my $user_agent = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.4) Gecko/2008111317 Ubuntu/8.04 (hardy) Firefox/3.0.4';

my $prefetcher = WWW::Prefetcher->new({
    timeout => $timeout,
    user_agent => $user_agent,
    http_proxy => $proxy,
    before => sub { warn "Prefetching ${ $_[0] }...\n" },
    on_error => sub {
        my ($rurl, $curl, $retcode) = @_;
        warn "[error] Failed to process $$rurl: ",
            $curl->strerror($retcode),
            " ($retcode)";
    },
});

$prefetcher->process(\$_, 'html') for @urls;

__END__

    http://blog.agentzh.org
    http://openresty.org/admin/
    http://openresty.org/admin/login.html
    http://agentzh.org/misc/onccf/out/