use utf8; use strict; use warnings; use Digest::MD5 'md5_hex'; use WWW::OpenResty::Simple; use Getopt::Std; my %opts; getopts('u:s:p:h', \%opts) or help();; if ($opts{h}) { help (); } @ARGV >= 2 or help(); my ($prefetcher_pass, $firefox_pass) = @ARGV; sub help { die "Usage: $0 -u -p -s \n"; } my $user = $opts{u} or die "No OpenResty account name specified via option -u\n"; my $password = $opts{p} or die "No OpenResty account's Admin password specified via option -p\n"; my $server = $opts{s} || 'http://api.openresty.org'; my $resty = WWW::OpenResty::Simple->new( { server => $server } ); $resty->login($user, $password); $resty->delete("/=/role/Public/~/~"); $resty->delete("/=/role"); $resty->delete("/=/model"); $resty->delete("/=/view"); $resty->delete("/=/action"); $resty->delete("/=/feed"); $resty->delete("/=/action"); $resty->post( '/=/model/UrlTask', { description => "URL tasks", columns => [ { name => 'url', type => 'text', unique => JSON::XS::true, not_null => JSON::XS::true, label => 'URL' }, { name => 'status', type => 'char (1)', not_null => JSON::XS::true, label => 'Status', default => "'I'" }, { name => 'proxy_id', type => 'integer', label => 'HTTP proxy used' }, { name => 'priority', type => 'integer', label => 'Priority', default => 0 }, { name => 'orig_id', type => 'integer', label => 'Original ID' }, { name => 'completed_time', type => 'timestamp (0)', label => 'Completed time' }, { name => 'created_time', type => 'timestamp (0)', label => 'Created time' }, { name => 'extract', type => 'smallint', label => 'Extraction type (0: none, 1: main links)', not_null => JSON::XS::true, default => '0' }, { name => 'description', type => 'text', label => 'Description' }, ], } ); $resty->post( '/=/model/HttpProxy', { description => "HTTP Proxies", columns => [ { name => 'hostname', type => 'text', label => 'Proxy hostname' }, { name => 'port', type => 'integer', label => 'Proxy Port' }, { name => 'size', type => 'integer', label => 'Cache size (GB)' }, ], } ); $resty->post( '/=/view/PrefetcherGetTasks', { description => 'Get the next bulk of tasks for the prefetchers', definition => 'select * from prefetcher_get_tasks($count)', } ); $resty->post( '/=/view/PrefetcherSubmit', { description => 'Submit the results for prefetchers', definition => 'select prefetcher_submit($ok_ids, $nok_ids, $proxy) as affected', } ); $resty->post( '/=/view/FirefoxGetTasks', { description => 'Get the next bulk of tasks for the firefox crawlers', definition => 'select * from firefox_get_tasks($count)', } ); $resty->post( '/=/view/FirefoxSubmit', { description => 'Submit the results for firefox crawlers', definition => 'select firefox_submit($list_page_ids, $text_page_ids) as affected', } ); $resty->post( '/=/role/Prefetcher', { description => 'Prefetcher', login => 'password', password => md5_hex($prefetcher_pass), } ); $resty->post( '/=/role/Prefetcher/~/~', [ { url => '/=/model/HttpProxy/~/~' }, { url => '/=/view/PrefetcherGetTasks/~/~' }, { url => '/=/view/PrefetcherSubmit/~/~' }, ] ); $resty->post( '/=/role/Firefox', { description => 'Firefox', login => 'password', password => md5_hex($firefox_pass), } ); $resty->post( '/=/role/Firefox/~/~', [ { url => '/=/model/HttpProxy/~/~' }, { url => '/=/view/FirefoxGetTasks/~/~' }, { url => '/=/view/FirefoxSubmit/~/~' }, { method => 'POST', url => '/=/model/Link/~/~' }, ] );