From: Dirk Koopman Date: Wed, 20 May 2020 12:30:07 +0000 (+0100) Subject: correct users conversion program name X-Git-Url: http://www.dxcluster.org/gitweb/gitweb.cgi?p=spider.git;a=commitdiff_plain;h=ff93c84f70d94087bfa178889b8dbbd465919f53 correct users conversion program name --- diff --git a/Changes b/Changes index a5188e22..c2d4e117 100644 --- a/Changes +++ b/Changes @@ -1,7 +1,7 @@ 20May20======================================================================= 1. Strip out conversion from users.v2 and v3 to new json format file into a - new program called convert-v3-to-v4.pl. In theory, this program *could* be - run at any time and is backported to mojo and master branches. + new program called convert-users-v3-to-v4.pl. In theory, this program + *could* be run at any time and is backported to mojo and master branches. 19May20======================================================================= 1. Convert all remaining commands and areas within the program that used the DB_File/Storable interface to DXUsers.pm to use the (hopefully) more stable diff --git a/perl/DXUser.pm b/perl/DXUser.pm index d9c16600..ee24c5d4 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -197,7 +197,7 @@ sub init $v4 = 1; readinjson(); } else { - die "User file $filename missing, please run users-v3-to-v4.pl or copy a user_json backup from somewhere\n"; + die "User file $filename missing, please run convert-users-v3-to-v4.pl or copy a user_json backup from somewhere\n"; } } diff --git a/perl/convert-users-v3-to-v4.pl b/perl/convert-users-v3-to-v4.pl new file mode 100755 index 00000000..f8ac9bf4 --- /dev/null +++ b/perl/convert-users-v3-to-v4.pl @@ -0,0 +1,141 @@ +#!/usr/bin/env perl +# +# Convert users.v2 or .v3 to JSON .v4 format +# +# It is believed that this can be run at any time... +# +# Copyright (c) 2020 Dirk Koopman G1TLH +# +# +# + +# make sure that modules are searched in the order local then perl + +BEGIN { + # root of directory tree for this system + $root = "/spider"; + $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; + unshift @INC, "$root/perl"; # this IS the right way round! + unshift @INC, "$root/local"; +} + +use strict; + +use SysVar; +use DXUser; +use DXUtil; +use JSON; +use Data::Structure::Util qw(unbless); +use Time::HiRes qw(gettimeofday tv_interval); +use IO::File; +use Carp; +use DB_File; + +use 5.10.1; + +my $ufn; +my $fn = "users"; + +my $json = JSON->new()->canonical(1); +my $ofn = localdata("$fn.v4"); +my $convert; + +eval { + require Storable; +}; + +if ($@) { + if ( ! -e localdata("$fn.v3") && -e localdata("$fn.v2") ) { + $convert = 2; + } + LogDbg('',"the module Storable appears to be missing!!"); + LogDbg('',"trying to continue in compatibility mode (this may fail)"); + LogDbg('',"please install Storable from CPAN as soon as possible"); +} +else { + import Storable qw(nfreeze thaw); + $convert = 3 if -e localdata("users.v3") && !-e $ufn; +} + +die "need to have a $fn.v2 or (preferably) a $fn.v3 file in /spider/data or /spider/local_data\n" unless $convert; + +if (-e $ofn || -e "$ofn.n") { + my $nfn = localdata("$fn.v4.json"); + say "You appear to have (or are using) $ofn, creating $nfn instead"; + $ofn = $nfn; +} else { + say "using $ofn for output"; +} + + +# do a conversion if required +if ($convert) { + my ($key, $val, $action, $count, $err) = ('','',0,0,0); + my $ta = [gettimeofday]; + my $ofh = IO::File->new(">$ofn") or die "cannot open $ofn ($!)\n"; + + my %oldu; + LogDbg('',"Converting the User File from V$convert to $fn.v4 "); + LogDbg('',"This will take a while, maybe as much as 10 secs"); + my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]"; + for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { + my $ref; + if ($convert == 3) { + eval { $ref = storable_decode($val) }; + } + else { + eval { $ref = asc_decode($val) }; + } + unless ($@) { + if ($ref) { + unbless $ref; + $ofh->print($json->encode($ref) . "\n"); + $count++; + } + else { + $err++ + } + } + else { + Log('err', "DXUser: error decoding $@"); + } + } + undef $odbm; + untie %oldu; + my $t = _diffms($ta); + LogDbg('',"Conversion from users.v$convert to $ofn completed $count records $err errors $t mS"); + $ofh->close; +} + +exit 0; + +sub asc_decode +{ + my $s = shift; + my $ref; + $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; + eval '$ref = ' . $s; + if ($@) { + LogDbg('err', "DXUser::asc_decode: on '$s' $@"); + $ref = undef; + } + return $ref; +} + +sub storable_decode +{ + my $ref; + $ref = thaw(shift); + return $ref; +} + +sub LogDbg +{ + my (undef, $s) = @_; + say $s; +} + +sub Log +{ + say shift; +} diff --git a/perl/users-v3-to-v4.pl b/perl/users-v3-to-v4.pl deleted file mode 100755 index f8ac9bf4..00000000 --- a/perl/users-v3-to-v4.pl +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env perl -# -# Convert users.v2 or .v3 to JSON .v4 format -# -# It is believed that this can be run at any time... -# -# Copyright (c) 2020 Dirk Koopman G1TLH -# -# -# - -# make sure that modules are searched in the order local then perl - -BEGIN { - # root of directory tree for this system - $root = "/spider"; - $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; - unshift @INC, "$root/perl"; # this IS the right way round! - unshift @INC, "$root/local"; -} - -use strict; - -use SysVar; -use DXUser; -use DXUtil; -use JSON; -use Data::Structure::Util qw(unbless); -use Time::HiRes qw(gettimeofday tv_interval); -use IO::File; -use Carp; -use DB_File; - -use 5.10.1; - -my $ufn; -my $fn = "users"; - -my $json = JSON->new()->canonical(1); -my $ofn = localdata("$fn.v4"); -my $convert; - -eval { - require Storable; -}; - -if ($@) { - if ( ! -e localdata("$fn.v3") && -e localdata("$fn.v2") ) { - $convert = 2; - } - LogDbg('',"the module Storable appears to be missing!!"); - LogDbg('',"trying to continue in compatibility mode (this may fail)"); - LogDbg('',"please install Storable from CPAN as soon as possible"); -} -else { - import Storable qw(nfreeze thaw); - $convert = 3 if -e localdata("users.v3") && !-e $ufn; -} - -die "need to have a $fn.v2 or (preferably) a $fn.v3 file in /spider/data or /spider/local_data\n" unless $convert; - -if (-e $ofn || -e "$ofn.n") { - my $nfn = localdata("$fn.v4.json"); - say "You appear to have (or are using) $ofn, creating $nfn instead"; - $ofn = $nfn; -} else { - say "using $ofn for output"; -} - - -# do a conversion if required -if ($convert) { - my ($key, $val, $action, $count, $err) = ('','',0,0,0); - my $ta = [gettimeofday]; - my $ofh = IO::File->new(">$ofn") or die "cannot open $ofn ($!)\n"; - - my %oldu; - LogDbg('',"Converting the User File from V$convert to $fn.v4 "); - LogDbg('',"This will take a while, maybe as much as 10 secs"); - my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]"; - for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { - my $ref; - if ($convert == 3) { - eval { $ref = storable_decode($val) }; - } - else { - eval { $ref = asc_decode($val) }; - } - unless ($@) { - if ($ref) { - unbless $ref; - $ofh->print($json->encode($ref) . "\n"); - $count++; - } - else { - $err++ - } - } - else { - Log('err', "DXUser: error decoding $@"); - } - } - undef $odbm; - untie %oldu; - my $t = _diffms($ta); - LogDbg('',"Conversion from users.v$convert to $ofn completed $count records $err errors $t mS"); - $ofh->close; -} - -exit 0; - -sub asc_decode -{ - my $s = shift; - my $ref; - $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; - eval '$ref = ' . $s; - if ($@) { - LogDbg('err', "DXUser::asc_decode: on '$s' $@"); - $ref = undef; - } - return $ref; -} - -sub storable_decode -{ - my $ref; - $ref = thaw(shift); - return $ref; -} - -sub LogDbg -{ - my (undef, $s) = @_; - say $s; -} - -sub Log -{ - say shift; -}