From 54775ee86e54343a0fba66a0bbca3567de96cf46 Mon Sep 17 00:00:00 2001 From: Dirk Koopman Date: Sat, 11 Mar 2023 21:22:38 +0000 Subject: [PATCH] add 'in program' download cmd to replace wget --- Changes | 18 ++++++++++++++ cmd/Commands_en.hlp | 19 ++++++++++++++- cmd/download.pl | 58 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 cmd/download.pl diff --git a/Changes b/Changes index 9a4307e4..74c07256 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,21 @@ +10Mar23======================================================================= +1. I am aware that there are windows nodes out there on mojo (brave, impetuous + that you are) and wget is not an easy option to download badip files. So, + through the power of mojo I give you the new 'download' command. As it + stands at the moment, it will download a file from a webserver and place + it in /spider/local_data. If it downloads successfully, it will overwrite + any file with that name in /spider/local_data. Needless to say this is a + sysop only command. + +24 * * * * run_cmd('download http://www.dxspider.net/download/badip.torexit') +24 * * * * run_cmd('download http://www.dxspider.net/download/badip.torrelay') +24 * * * * run_cmd('download http://www.dxspider.net/download/badip.global') +25 * * * * run_cmd('load/badip') + + would be an obvious use. But it can also be used to download spot, USDB + and other data as well. If you do use it then you will still need to + do a spawn_cmd in crontab for the conversion programs, but that too is + going to change. 07Mar23======================================================================= 1. Fix odd (and old) regression dx command (line ending 59+++) '++' being the culprit. diff --git a/cmd/Commands_en.hlp b/cmd/Commands_en.hlp index 63cef603..6d37efb9 100644 --- a/cmd/Commands_en.hlp +++ b/cmd/Commands_en.hlp @@ -616,7 +616,24 @@ or everything (except yourself) with DISC all -=== 0^DX [BY ] ^Send a DX spot +=== 9^DOWNLOAD ^Download a file into local_data +This command is a direct replacement for the unix 'wget -Qn' command +that is used to download files like badip, spot data, user databases +like usdb. It is designed to work either on the command line in a console +or (more likely) in the crontab, like the example below: + +24 * * * * run_cmd('download http://www.dxspider.net/download/badip.torexit') +24 * * * * run_cmd('download http://www.dxspider.net/download/badip.torrelay') +24 * * * * run_cmd('download http://www.dxspider.net/download/badip.global') +25 * * * * run_cmd('load/badip') + +If you do use the crontab then *please* use a random minute between 15-40 +and not all use minute 24. + +Windows users may well find this particularly useful. + +=== 0^DX ^Send a DX spot +=== 2^DX [BY ] [ip ] ^Send a DX spot This is how you send a DX Spot to other users. You can, in fact, now enter the and the either way round. diff --git a/cmd/download.pl b/cmd/download.pl new file mode 100644 index 00000000..46cb627b --- /dev/null +++ b/cmd/download.pl @@ -0,0 +1,58 @@ +# +# download a file from t'internet +# +# A build in, non-spawning, wget -Qn +# +# Copyright 2023 Dirk Koopman G1TLH +# + +#use IO::Socket::SSL; +use File::Copy; + +my %h; + +sub handle +{ + my $self = shift; + return (1, $self->msg('e5')) if $self->priv < 9 || $self->remotecmd; + my $url = unpad(shift); + my $dest = unpad(shift) if @_; + dbg("download: url $url"); + my $ua = Mojo::UserAgent->new->insecure(1)->max_redirects(5); + my $res = $ua->get($url => sub {finish(@_, $self, $ua)}); + $self->{$res} = $res; + dbg("ua $ua start: $url") if isdbg('download'); +} + +sub finish { + my ($ua, $tx, $self, $ua) = @_; + +# $DB::single = 1; + + my $res = $tx->res; + + if ($res->is_success) { + #dbg("body: " . $res->body) if isdbg('download'); + my $tmp = localdata("tmp"); + mkdir $tmp, 0777 unless -e $tmp; + my $path = $tx->req->url->to_abs->path; + my @parts = split m|/|, $path; + my $fn = $parts[-1]; + dbg("ua $ua temp file: $tmp/$fn") if isdbg('download'); + $res->save_to("$tmp/$fn"); + my $target = localdata($fn); + if (-e "$tmp/$fn") { + LogDbg('dxcommand', "moving $tmp/$fn -> $target from "); + move "$tmp/$fn", $target; + unlink "$tmp/$fn"; + } + dbg("download: $target successfully downloaded") if isdbg('progress') + } elsif ($res->is_error) { + dbg("ua $ua err: " . $res->error) if isdbg('download'); + } elsif ($res->code == 301) { + dbg("redirect: " . $res->headers->location) + } else { + dbg("something else: " . $res->error->{message}); + } + delete $self->{$res}; +} -- 2.43.0