From: Dirk Koopman Date: Sun, 31 May 2020 13:59:02 +0000 (+0100) Subject: initial removal of Storable X-Git-Url: http://www.dxcluster.org/gitweb/gitweb.cgi?p=spider.git;a=commitdiff_plain;h=6ddc03379ca50a7ecbc04aea34edc8edc1ce0f84 initial removal of Storable --- diff --git a/cmd/export_users.pl b/cmd/export_users.pl index 774d8384..45b03d75 100644 --- a/cmd/export_users.pl +++ b/cmd/export_users.pl @@ -7,9 +7,9 @@ my $self = shift; my $line = shift;; return (1, $self->msg('e5')) unless $self->priv >= 9; +my $line ||= 'user_json'; my ($fn, $flag) = split /\s+/, $line; -$fn ||= 'user_asc'; -unless ($fn && $fn eq 'user_asc') { +unless ($fn && $fn eq 'user_json') { $fn =~ s|[/\.]||g; $fn = "/tmp/$fn"; } diff --git a/perl/DXUser.pm b/perl/DXUser.pm index b33f8823..02ed86cd 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -17,6 +17,9 @@ use DXDebug; use DXUtil; use LRU; use File::Copy; +use Data::Structure::Util qw(unbless); +use Time::HiRes qw(gettimeofday tv_interval); +use IO::File; use strict; @@ -27,11 +30,13 @@ $dbm = undef; $filename = undef; $lastoperinterval = 60*24*60*60; $lasttime = 0; -$lrusize = 2000; +$lrusize = 3000; $tooold = 86400 * 365; # this marks an old user who hasn't given enough info to be useful $v3 = 0; our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs +my $json; + # hash of valid elements and a simple prompt %valid = ( call => '0,Callsign', @@ -127,73 +132,34 @@ sub init { my $mode = shift; - my $ufn; - my $convert; - - eval { - require Storable; - }; - + $json = JSON->new->canonical(1); my $fn = "users"; - - if ($@) { - $ufn = localdata("users.v2"); - $v3 = $convert = 0; - dbg("the module Storable appears to be missing!!"); - dbg("trying to continue in compatibility mode (this may fail)"); - dbg("please install Storable from CPAN as soon as possible"); - } else { - import Storable qw(nfreeze thaw); - - $ufn = localdata("users.v3"); - $v3 = 1; - $convert++ if -e localdata("users.v2") && !-e $ufn; - } - - if ($mode) { - $dbm = tie (%u, 'DB_File', $ufn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; - } else { - $dbm = tie (%u, 'DB_File', $ufn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; + $filename = localdata("$fn.v3j"); + unless (-e $filename || $mode == 2) { + LogDbg('DXUser', "New User File version $filename does not exist, running conversion from users.v3 or v2, please wait"); + system('/spider/perl/convert-users-v3-to-v3j.pl'); + init(1); + export(); + return; } - - die "Cannot open $ufn ($!)\n" unless $dbm; - - $lru = LRU->newbase("DXUser", $lrusize); - - # do a conversion if required - if ($dbm && $convert) { - my ($key, $val, $action, $count, $err) = ('','',0,0,0); - - my %oldu; - dbg("Converting the User File to V3 "); - dbg("This will take a while, I suggest you go and have cup of strong tea"); - my $odbm = tie (%oldu, 'DB_File', localdata("users.v2"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v2 ($!) [rebuild it from user_asc?]"; - for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { - my $ref; - eval { $ref = asc_decode($val) }; - unless ($@) { - if ($ref) { - $ref->put; - $count++; - } else { - $err++ - } - } else { - Log('err', "DXUser: error decoding $@"); - } - } - undef $odbm; - untie %oldu; - dbg("Conversion completed $count records $err errors"); + if (-e $filename || $mode == 2) { + $lru = LRU->newbase("DXUser", $lrusize); + if ($mode) { + $dbm = tie (%u, 'DB_File', $filename, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_json?]"; + } else { + $dbm = tie (%u, 'DB_File', $filename, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_json?]"; + } } - $filename = $ufn; + die "Cannot open $filename ($!)\n" unless $dbm || $mode == 2; + return; } +# delete files with extreme prejudice sub del_file { # with extreme prejudice - unlink "$main::data/users.v3"; - unlink "$main::local_data/users.v3"; + unlink "$main::data/users.v3j"; + unlink "$main::local_data/users.v3j"; } # @@ -330,60 +296,31 @@ sub put $dbm->put($call, $ref); } -# freeze the user -sub encode -{ - goto &asc_encode unless $v3; - my $self = shift; - return nfreeze($self); -} # thaw the user sub decode { - goto &asc_decode unless $v3; - my $ref; - $ref = thaw(shift); - return $ref; + my $s = shift; + my $ref; + eval { $ref = $json->decode($s) }; + if ($ref && !$@) { + return bless $ref, 'DXUser'; + } else { + LogDbg('DXUser', "DXUser::json_decode: on '$s' $@"); + } + return undef; } -# -# create a string from a user reference (in_ascii) -# -sub asc_encode +# freeze the user +sub encode { - my $self = shift; - my $strip = shift; - my $p; - - if ($strip) { - my $ref = bless {}, ref $self; - foreach my $k (qw(qth lat long qra sort call homenode node lastoper lastin)) { - $ref->{$k} = $self->{$k} if exists $self->{$k}; - } - $ref->{name} = $self->{name} if exists $self->{name} && $self->{name} !~ /selfspot/i; - $p = dd($ref); - } else { - $p = dd($self); - } - return $p; + my $ref = shift; + unbless($ref); + my $s = $json->encode($ref); + bless $ref, 'DXUser'; + return $s; } -# -# create a hash from a string (in ascii) -# -sub asc_decode -{ - my $s = shift; - my $ref; - $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; - eval '$ref = ' . $s; - if ($@) { - LogDbg('err', "DXUser::asc_decode: on '$s' $@"); - $ref = undef; - } - return $ref; -} # # del - delete a user @@ -440,10 +377,10 @@ sub fields sub export { - my $name = shift || 'user_asc'; + my $name = shift || 'user_json'; my $basic_info_only = shift; - my $fn = $name ne 'user_asc' ? $name : "$main::local_data/$name"; # force use of local + my $fn = $name ne 'user_json' ? $name : "$main::local_data/$name"; # force use of local # save old ones move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo"; @@ -452,6 +389,7 @@ sub export move "$fn.o", "$fn.oo" if -e "$fn.o"; move "$fn", "$fn.o" if -e "$fn"; + my $ta = [gettimeofday]; my $count = 0; my $err = 0; my $del = 0; @@ -495,35 +433,39 @@ BEGIN { } use SysVar; +use DXUtil; use DXUser; +use JSON; +use Time::HiRes qw(gettimeofday tv_interval); +package DXUser; -if (@ARGV) { - $main::userfn = shift @ARGV; - print "user filename now $userfn\n"; -} +our $json = JSON->new->canonical(1); -package DXUser; +my $ta = [gettimeofday]; +our $filename = "$main::local_data/users.v3j"; +my $exists = -e $filename ? "OVERWRITING" : "CREATING"; +print "perl user_json $exists $filename\n"; del_file(); -init(1); +init(2); %u = (); my $count = 0; my $err = 0; while () { chomp; my @f = split /\t/; - my $ref = asc_decode($f[1]); + my $ref = decode($f[1]); if ($ref) { $ref->put(); $count++; - DXUser::sync() unless $count % 10000; } else { print "# Error: $f[0]\t$f[1]\n"; $err++ } } DXUser::sync(); DXUser::finish(); -print "There are $count user records and $err errors\n"; +my $diff = _diffms($ta); +print "There are $count user records and $err errors in $diff mS\n"; }; print $fh "__DATA__\n"; @@ -553,7 +495,7 @@ print "There are $count user records and $err errors\n"; } } # only store users that are reasonably active or have useful information - print $fh "$key\t" . $ref->asc_encode($basic_info_only) . "\n"; + print $fh "$key\t" . encode($ref) . "\n"; ++$count; } else { LogDbg('DXCommand', "Export Error3: $key\t" . carp($val) ."\n$@"); @@ -564,7 +506,8 @@ print "There are $count user records and $err errors\n"; } $fh->close; } - my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors ('sh/log Export' for details)}; + my $diff = _diffms($ta); + my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $diff mS ('sh/log Export' for details)}; LogDbg('command', $s); return $s; } diff --git a/perl/convert-users-v3-to-v3j.pl b/perl/convert-users-v3-to-v3j.pl new file mode 100755 index 00000000..06fda097 --- /dev/null +++ b/perl/convert-users-v3-to-v3j.pl @@ -0,0 +1,148 @@ +#!/usr/bin/env perl +# +# Convert users.v2 or .v3 to JSON .v3j format +# +# It is believed that this can be run at any time... +# +# Copyright (c) 2020 Dirk Koopman G1TLH +# +# +# + +# make sure that modules are searched in the order local then perl + +BEGIN { + # root of directory tree for this system + $root = "/spider"; + $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; + unshift @INC, "$root/perl"; # this IS the right way round! + unshift @INC, "$root/local"; +} + +use strict; + +use SysVar; +use DXUser; +use DXUtil; +use JSON; +use Data::Structure::Util qw(unbless); +use Time::HiRes qw(gettimeofday tv_interval); +use IO::File; +use File::Copy; +use Carp; +use DB_File; + +use 5.10.1; + +my $ufn; +my $fn = "users"; + +my $json = JSON->new()->canonical(1); +my $ofn = localdata("$fn.v3j"); +my $convert; + +eval { + require Storable; +}; + +if ($@) { + if ( ! -e localdata("$fn.v3") && -e localdata("$fn.v2") ) { + $convert = 2; + } + LogDbg('',"the module Storable appears to be missing!!"); + LogDbg('',"trying to continue in compatibility mode (this may fail)"); + LogDbg('',"please install Storable from CPAN as soon as possible"); +} +else { + import Storable qw(nfreeze thaw); + $convert = 3 if -e localdata("users.v3") && !-e $ufn; +} + +die "need to have a $fn.v2 or (preferably) a $fn.v3 file in /spider/data or /spider/local_data\n" unless $convert; + +if (-e $ofn) { + my $nfn = localdata("$fn.v3j.new"); + say "You appear to have (or are using) $ofn, creating $nfn instead"; + $ofn = $nfn; +} else { + $ofn = $ofn; + say "using $ofn for output"; +} + + +# do a conversion if required +if ($convert) { + my ($key, $val, $action, $count, $err) = ('','',0,0,0); + my $ta = [gettimeofday]; + my $ofh = IO::File->new(">$ofn") or die "cannot open $ofn ($!)\n"; + + my %oldu; + my %newu; + + LogDbg('',"Converting the User from V$convert format to $fn.v3j "); + LogDbg('',"This will take a while, maybe as much as 10 secs"); + my $idbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]"; + my $odbm = tie (%newu, 'DB_File', $ofn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $ofn ($!)"; + for ($action = R_FIRST; !$idbm->seq($key, $val, $action); $action = R_NEXT) { + my $ref; + if ($convert == 3) { + eval { $ref = storable_decode($val) }; + } + else { + eval { $ref = asc_decode($val) }; + } + unless ($@) { + if ($ref) { + unbless $ref; + $newu{$ref->{call}} = $json->encode($ref); + $count++; + } + else { + $err++ + } + } + else { + Log('err', "DXUser: error decoding $@"); + } + } + untie %oldu; + undef $idbm; + untie %newu; + undef $odbm; + my $t = _diffms($ta); + LogDbg('',"Conversion from users.v$convert to $ofn completed $count records $err errors $t mS"); + $ofh->close; +} + +exit 0; + +sub asc_decode +{ + my $s = shift; + my $ref; + $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; + eval '$ref = ' . $s; + if ($@) { + LogDbg('err', "asc_decode: on '$s' $@"); + $ref = undef; + } + return $ref; +} + +sub storable_decode +{ + my $ref; + $ref = thaw(shift); + return $ref; +} + +sub LogDbg +{ + my (undef, $s) = @_; + say $s; +} + +sub Log +{ + say shift; +}