Merge branch 'test' into mojo

[spider.git] / perl / create_master_badip_files.pl

diff --git a/perl/create_master_badip_files.pl b/perl/create_master_badip_files.pl
index 3244983b0191d1c735aff3643f749740873e9fc8..f7b25be48ca5dfe472416245a79f410ff52f2e63 100755 (executable)
--- a/perl/create_master_badip_files.pl
+++ b/perl/create_master_badip_files.pl
@@ -33,6 +33,7 @@ use LWP::Simple;
 use JSON;
 use Date::Parse;
 use File::Copy;
+use DXUtil;
 
 DXDebug::dbginit();
 
@@ -69,6 +70,7 @@ my $data = decode_json($content);
 my $now = time;
 my $ecount = 0;
 my $rcount = 0;
+my $error = 0;
 
 my $rand = rand;
 open RELAY, ">$relayfn.$rand" or die "$0: cannot open $relayfn $!";
@@ -79,31 +81,43 @@ foreach my $e (@{$data->{relays}}) {
        my $seen = str2time($e->{last_seen});
        next unless $seen >= $now - $last_seen_window;
        
-       my @or = clean_addr(@{$e->{or_addresses}}) if exists $e->{or_addresses};
        my @exit = clean_addr(@{$e->{exit_addresses}}) if exists $e->{exit_addresses} ;
+       my @or = clean_addr(@{$e->{or_addresses}}) if exists $e->{or_addresses};
        my $ors = join ', ', @or;
        my $es = join ', ', @exit;
        dbg "$0: $e->{nickname} $e->{last_seen} relays: [$ors] exits: [$es]" if $debug;
-       for (@or) {
-               print RELAY "$_\n";
-               ++$rcount;
-       }
        for (@exit) {
-               print EXIT "$_\n";
-               ++$ecount;
+               if (is_ipaddr($_)) {
+                       print EXIT "$_\n";
+                       ++$ecount;
+               } else {
+                       print STDERR "$_\n";
+                       ++$error;
+               }
+       }
+       for (@or) {
+               if (is_ipaddr($_)) {
+                       print RELAY "$_\n";
+                       ++$rcount;
+               } else {
+                       print STDERR "$_\n";
+                       ++$error;
+               }
        }
 }
 
 close RELAY;
 close EXIT;
 
-dbg("$0: $rcount relays $ecount exits found");
+dbg("$0: $rcount relays $ecount exits $error error(s) found.");
 move "$relayfn.$rand", $relayfn if $rcount;
 move "$exitfn.$rand", $exitfn if $ecount;
 unlink "$relayfn.$rand";
 unlink "$exitfn.$rand";
 
-exit 0;
+exit $error;
+
+my %addr;
 
 sub clean_addr
 {
@@ -111,11 +125,17 @@ sub clean_addr
        foreach (@_) {
                my ($ipv4) = /^((?:\d+\.){3}\d+)/;
                if ($ipv4) {
+                       next if exists $addr{$ipv4};
                        push @out, $ipv4;
+                       $addr{$ipv4}++;
                        next;
                }
                my ($ipv6) = /^\[([:a-f\d]+)\]/;
-               push @out, $ipv6 if $ipv6;
+               if ($ipv6) {
+                       next if exists $addr{$ipv6};
+                       push @out, $ipv6;
+                       $addr{$ipv6}++;
+               }
        }
        return @out;
 }