From e7a28420ced12c94812516b17194c997dcc759e9 Mon Sep 17 00:00:00 2001
From: Dirk Koopman <djk@tobit.co.uk>
Date: Sat, 30 May 2020 17:46:24 +0100
Subject: [PATCH] change sh/dx backend to the Filter engine

---
 Changes        |  20 +++
 cmd/show/dx.pl | 419 +++++++------------------------------------------
 perl/Filter.pm |  11 +-
 perl/Spot.pm   | 114 ++++++--------
 4 files changed, 133 insertions(+), 431 deletions(-)

diff --git a/Changes b/Changes
index 0acea0c7..50788fd9 100644
--- a/Changes
+++ b/Changes
@@ -1,3 +1,23 @@
+30May20=======================================================================
+1. Fix sh/dx! It appears that there is a long standing problem with sh/dx not
+   actually return all the answers it should. I have also had a request to
+   allow / enable the "not" keyword (just like the filtering system dones).
+
+   Now this was one of the earliest modules that I wrote and it took quite
+   a bit of work to tease out the important bits and then use the Filter 
+   module (as in acc/spot) to generate the sh/dx filtering expressions. This
+   seems to have been done. 
+ 
+   NB This will likely not work if you are using a SQL backend to do sh/dx
+   commands. This was never actually an official feature. It is now 
+   deprecated.   
+29May20=======================================================================
+1. Please install Math::Round and Data::Structure::Util. Instructions are
+   available in UPGRADE.mojo.
+2. Notwithstanding item 1 below. I have decided to keep DB_File but ditch
+   Storable. The current users.v3 will be converted, automatically, to 
+   users.v3j on startup. After this is done a 'user_json' file (the 
+   replacement for 'user_asc') will also be generated. 
 20May20=======================================================================
 1. Backport convert-users-v3-to-v4.pl to allow creation of the new json 
    formatted *text* based user file from the old v3 DB_File and Storable 
diff --git a/cmd/show/dx.pl b/cmd/show/dx.pl
index 71bb2e61..174071f8 100644
--- a/cmd/show/dx.pl
+++ b/cmd/show/dx.pl
@@ -14,425 +14,120 @@ sub handle
 	my @out;
 	my $f;
 	my $call = $self->call;
+	my $usesql = $main::dbh && $Spot::use_db_for_search;
 	my ($from, $to);
 	my ($fromday, $today);
-	my @freq;
-	my @ans;
-	my $pre;
-	my $spotter;
-	my $info;
+	my $exact;
+	my $real;
+	my $user;
 	my $expr;
-	my $hint;
+	my $dofilter;
+	my $pre;
 	my $dxcc;
-	my $real;
-	my $zone;
-	my $byzone;
-	my $state;
-	my $bystate;
-	my $itu;
-	my $byitu;
-	my $fromdxcc = 0;
-	my $exact;
-	my $origin;
-	my $ip;
-	my ($doqsl, $doiota, $doqra, $dofilter);
 
-	my $usesql = $main::dbh && $Spot::use_db_for_search;
+	my @flist;
 
 	while ($f = shift @list) {	# next field
-		dbg "arg: $f list: " . join(',', @list) if isdbg('sh/dx');
+		dbg "sh/dx arg: $f list: " . join(',', @list) if isdbg('sh/dx');
 		if (!$from && !$to) {
 			($from, $to) = $f =~ m|^(\d+)[-/](\d+)$|; # is it a from -> to count?
+			dbg("sh/dx from: $from to: $to") if isdbg('sh/dx');
 			next if $from && $to > $from;
 		}
 		if (!$to) {
 			($to) = $f =~ /^(\d+)$/o if !$to; # is it a to count?
+			dbg("sh/dx to: $to") if isdbg('sh/dx');
 			next if $to;
 		}
-		if (lc $f eq 'exact') {
-			$exact = 1;
-			next;
-		}
-		if (lc $f eq 'dxcc') {
-			$dxcc = 1;
-			next;
-		}
-		if (lc $f eq 'rt' || $f =~ /^real/i) {
-			$real = 1;
-			next;
-		}
-		if (lc $f eq 'on' && $list[0]) { # is it freq range?
-			dbg "freq $list[0]" if isdbg('sh/dx');
-			if (my ($from, $to) = $list[0] =~ m|^(\d+)(?:\.\d+)?(?:[-/](\d+)(?:\.\d+)?)?$|) {
-				$to = $from unless defined $to;
-				dbg "freq '$from' '$to'" if isdbg('sh/dx');
-				push @freq, $from, $to;
-				shift @list;
-				next;
-			}
-			else {
-				my @r = split '/', lc $list[0];
-				dbg "r0: $r[0] r1: $r[1]" if isdbg('sh/dx');
-				my @fr = Bands::get_freq($r[0], $r[1]);
-				if (@fr) {		# yup, get rid of extranous param
-					dbg "freq: " . join(',', @fr) if isdbg('sh/dx');
-					push @freq, @fr; # add these to the list
-					shift @list;
-					next;
-				}
-			}
-		}
 		if (lc $f eq 'day' && $list[0]) {
 			($fromday, $today) = split m|[-/]|, shift(@list);
-			dbg "got day $fromday/$today" if isdbg('sh/dx');
-			next;
-		}
-		if (lc $f eq 'info' && $list[0]) {
-			$info = shift @list;
-			dbg "got info $info" if isdbg('sh/dx');
-			next;
-		}
-		if (lc $f eq 'origin' && $list[0]) {
-			$origin = uc shift @list;
-			dbg "got origin $origin" if isdbg('sh/dx');
+			dbg "sh/dx got day $fromday/$today" if isdbg('sh/dx');
 			next;
 		}
-		if (lc $f eq 'ip' && $list[0]) {
-			$ip = shift @list;
-			dbg "got ip $ip" if isdbg('sh/dx');
+		if (lc $f eq 'exact') {
+			dbg("sh/dx exact") if isdbg('sh/dx');
+			$exact = 1;
 			next;
 		}
-
-		if ((lc $f eq 'spotter' || lc $f eq 'by') && $list[0]) {
-			$spotter = uc shift @list;
-			if ($list[0] && lc $list[0] eq 'dxcc') {
-				$fromdxcc = 1;
-				shift @list;
-			}
-			dbg "got spotter $spotter fromdxcc $fromdxcc" if isdbg('sh/dx');
+		if (lc $f eq 'rt' || $f =~ /^real/i) {
+			dbg("sh/dx real") if isdbg('sh/dx');
+			$real = 1;
 			next;
 		}
 		if (lc $f =~ /^filt/) {
+			dbg("sh/dx run spotfilter") if isdbg('sh/dx');
 			$dofilter = 1 if $self && $self->spotsfilter;
 			next;
 		}
+		if (lc $f eq 'spotter') {
+			dbg("sh/dx by") if isdbg('sh/dx');
+			push @flist, 'by';
+			next;
+		}
 		if (lc $f eq 'qsl') {
-			$doqsl = 1;
+			dbg("sh/dx qsl") if isdbg('sh/dx');
+			push @flist, "info {QSL|VIA}";
 			next;
 		}
 		if (lc $f eq 'iota') {
-			my ($a, $b);
-			#		$DB::single =1;
-		
+			my $doiota;
 			if (@list && $list[0] && (($a, $b) = $list[0] =~ /(AF|AN|NA|SA|EU|AS|OC)-?(\d?\d\d)/oi)) {
 				$a = uc $a;
 				$doiota = "\\b$a\[\-\ \]\?$b\\b";
 				shift @list;
 			}
 			$doiota = '\b(IOTA|(AF|AN|NA|SA|EU|AS|OC)[- ]?\d?\d\d)\b' unless $doiota;
+			push @flist, "info {$doiota}";
+			dbg("sh/dx iota") if isdbg('sh/dx');
 			next;
 		}
 		if (lc $f eq 'qra') {
-			$doqra = uc shift @list if @list && $list[0] =~ /[A-Z][A-Z]\d\d/oi;
+			my $doqra = uc shift @list if @list && $list[0] =~ /[A-Z][A-Z]\d\d/oi;
 			$doqra = '\b([A-Z][A-Z]\d\d|[A-Z][A-Z]\d\d[A-Z][A-Z])\b' unless $doqra;
+			push @flist, "info {$doqra}";
+			dbg("sh/dx qra") if isdbg('sh/dx');
 			next;
 		}
-		if (lc $f eq 'zone') {
-			$zone = shift @list if @list;
+		if (grep {lc $f eq $_} qw(zone byzone by_zone itu byitu by_itu state bystate by_state on) ) {
+			$f =~ s/^by(\w)/by_$1/;
+			push @flist, $f;
+			push @flist, shift @list if @list;
 			next;
 		}
-		if (lc $f =~ /^by_?zone/) {
-			$byzone = shift @list if @list;
+		unless ($pre) {
+			$pre = $f;
 			next;
 		}
-		if (lc $f eq 'itu') {
-			$itu = shift @list if @list;
-			next;
-		}
-		if (lc $f =~ /^by_?itu/) {
-			$byitu = shift @list if @list;
-			next;
-		}
-		if (lc $f eq 'state') {
-			$state = uc shift @list if @list;
-			next;
-		}
-		if (lc $f =~ /^by_?state/) {
-			$bystate = uc shift @list if @list;
-			next;
-		}
-		if (!$pre) {
-			$pre = uc $f;
-		}
-	}
-
-	#$DB::single = 1;
-
-	# check origin
-	if ($origin) {
-		$expr .= ' && ' if $expr;
-		$expr .= "\$f7 eq '$origin'";
-		$hint .= ' && ' if $hint;
-		$hint .= "m{$origin}";
-	}
-
-	# check (any) ip address
-	if ($ip) {
-		$expr .= ' && ' if $expr;
-		$expr .= "\$f14 && \$f14 =~ m{^$ip}";
-		$hint .= ' && ' if $hint;
-		$ip =~ s/\./\\./g;		# IPV4
-		$hint .= "m{$ip}";
-	}
-
-	#  deal with the prefix
-	if ($pre) {
-		my @ans;
-	
-		if ($dxcc) {
-			@ans = Prefix::extract($pre); # is it a callsign/prefix?
-		
-			if (@ans) {
-
-				# first deal with the prefix
-				my $pre = shift @ans;
-				my $a;
-				my $str = "Prefix: $pre";
-				my $l = length $str;
-				my @expr;
-				my @hint;
-			
-				# build up a search string for this dxcc country/countries
-				foreach $a (@ans) {
-					my $n = $a->dxcc();
-					push @expr, "\$f5 == $n";
-					push @hint, "m{$n}";
-					my $name = $a->name();
-					$str .= " Dxcc: $n ($name)";
-					push @out, $str;
-					$str = ' ' x $l;
-				}
-				$expr = @expr > 1 ? '(' . join(' || ', @expr) . ')' : $expr[0];
-				$hint = @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-			}
-		} 
-		unless (@ans) {
-			$pre .= '*' unless $pre =~ /[\*\?\[]$/o;
-			$pre = shellregex($pre);
-			if ($usesql) {
-				$pre =~ s/\.\*/%/g;
-			}
-			else {
-				$pre =~ s/\.\*\$$//;
-			}
-			$pre .= '$' if $exact;
-			$expr = "\$f1 =~ m{$pre}";
-			$pre =~ s/[\^\$]//g;
-			$hint = "m{\U$pre}";
-		}
-	}
-  
-	# now deal with any frequencies specified
-	if (@freq) {
-		$expr .= ($expr) ? ' && (' : "(";
-		#	$hint .= ($hint) ? ' && ' : "(";
-		#	$hint .= ' && ' if $hint;
-		my $i;
-		for ($i = 0; $i < @freq; $i += 2) {
-			$expr .= "(\$f0 >= $freq[$i] && \$f0 <= $freq[$i+1]) ||";
-			my $r = Spot::ftor($freq[$i], $freq[$i+1]);
-			#		$hint .= "m{$r\\.} ||" if $r;
-			#		$hint .= "m{\d+\.} ||";
-			#		$hint .= "1 ||";
-		}
-		chop $expr;	chop $expr;
-		#	chop $hint;	chop $hint;
-		$expr .= ")";
-		#	$hint .= ")";
+		push @flist, $f;
 	}
 
-	# any info
-	if ($info) {
-		$expr .= ' && ' if $expr;
-		#	$info =~ s{(.)}{"\Q$1"}ge;
-		$expr .= "\$f3 =~ m{$info}i";
-		$hint .= ' && ' if $hint;
-		$hint .= "m{$info}i";
-	}
-
-	# any spotter
-	if ($spotter) {
 	
-		if ($fromdxcc) {
-			@ans = Prefix::extract($spotter); # is it a callsign/prefix?
-		
-			if (@ans) {
-
-				# first deal with the prefix
-				my $pre = shift @ans;
-				my $a;
-				$expr .= ' && ' if $expr;
-				$hint .= ' && ' if $hint;
-				my $str = "Spotter: $pre";
-				my $l = length $str;
-				my @expr;
-				my @hint;
-			
-				# build up a search string for this dxcc country/countries
-				foreach $a (@ans) {
-					my $n = $a->dxcc();
-					push @expr, "\$f6 == $n";
-					push @hint, "m{$n}";
-					my $name = $a->name();
-					$str .= " Dxcc: $n ($name)";
-					push @out, $str;
-					$str = ' ' x $l;
-				}
-				$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : $expr[0];
-				$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-			}
-		} 
-		unless (@ans) {
-			$expr .= ' && ' if $expr;
-			$spotter .= '*' unless $spotter =~ /[\*\?\[]/o;
-			$spotter = shellregex($spotter);
-			if ($usesql) {
-				$spotter =~ s/\.\*/%/g;
-			}
-			else {
-				$spotter =~ s/\.\*\$$//;
-			}
-			$expr .= "\$f4 =~ m{\U$spotter}";
-			$hint .= ' && ' if $hint;
-			$spotter =~ s/[\^\$]//g;
-			$hint .= "m{\U$spotter}";
-		}
-	}
-
-	# zone requests
-	if ($zone) {
-		my @expr;
-		my @hint;
-		$expr .= ' && ' if $expr;
-		$hint .= ' && ' if $hint;
-		for (split /[:,]/, $zone) {
-			push @expr, "\$f9==$_";
-			push @hint, "m{$_}";
-		}
-		$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : $expr[0];
-		$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-	}
-	if ($byzone) {
-		my @expr;
-		my @hint;
-		$expr .= ' && ' if $expr;
-		$hint .= ' && ' if $hint;
-		for (split /[:,]/, $byzone) {
-			push @expr, "\$f11==$_";
-			push @hint, "m{$_}";
-		}
-		$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : $expr[0];
-		$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-	}
-
-	# itu requests
-	if ($itu) {
-		my @expr;
-		my @hint;
-		$expr .= ' && ' if $expr;
-		$hint .= ' && ' if $hint;
-		for (split /[:,]/, $itu) {
-			push @expr, "\$f8==$_";
-			push @hint, "m{$_}";
-		}
-		$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : $expr[0];
-		$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-	}
-	if ($byitu) {
-		my @expr;
-		my @hint;
-		$expr .= ' && ' if $expr;
-		$hint .= ' && ' if $hint;
-		for (split /[:,]/, $byitu) {
-			push @expr, "\$f10==$_";
-			push @hint, "m{$_}";
-		}
-		$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : $expr[0];
-		$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-	}
-
-	# state requests
-	if ($state) {
-		my @expr;
-		my @hint;
-		$expr .= ' && ' if $expr;
-		$hint .= ' && ' if $hint;
-		for (split /[:,]/, $state) {
-			push @expr, "\$f12 eq '$_'";
-			push @hint, "m{$_}";
-		}
-		if ($usesql) {
-			$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : "$expr[0]";
-		}
-		else {
-			$expr .= @expr > 1 ? '(\$f12 && (' . join(' || ', @expr) . '))' : "(\$f12 && $expr[0])";
-		}
-		$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-	}
-	if ($bystate) {
-		my @expr;
-		my @hint;
-		$expr .= ' && ' if $expr;
-		$hint .= ' && ' if $hint;
-		for (split /[:,]/, $bystate) {
-			push @expr, "\$f13 eq '$_'";
-			push @hint, "m{$_}";
-		}
+	if ($pre) {
+		$pre .= '*' unless $pre =~ /[\*\?\[]$/o;
+		$pre = shellregex($pre);
 		if ($usesql) {
-			$expr .= @expr > 1 ? '(' . join(' || ', @expr) . ')' : "$expr[0]";
-		}
-		else {
-			$expr .= @expr > 1 ? '(\$f13 && (' . join(' || ', @expr) . '))' : "(\$f13 && $expr[0])";
+			$pre =~ s/\.\*/%/g;
+		} else {
+			$pre =~ s/\.\*\$$//;
 		}
-		$hint .= @hint > 1 ? '(' . join(' || ', @hint) . ')' : $hint[0];
-	}
-
-	# qsl requests
-	if ($doqsl) {
-		$expr .= ' && ' if $expr;
-		$expr .= "\$f3 =~ m{QSL|VIA}i";
-		$hint .= ' && ' if $hint;
-		$hint .= "m{QSL|VIA}i";
-	}
-
-	# iota requests
-	if ($doiota) {
-		$expr .= ' && ' if $expr;
-		$expr .= "\$f3 =~ m{$doiota}i";
-		$hint .= ' && ' if $hint;
-		$hint .= "m{$doiota}i";
-	}
-
-	# iota requests
-	if ($doqra) {
-		$expr .= ' && ' if $expr;
-		$expr .= "\$f3 =~ m{$doqra}i";
-		$hint .= ' && ' if $hint;
-		$hint .= "m{$doqra}io";
+		$pre .= '$' if $exact;
+		$pre =~ s/\^//;
+		
+		push @flist, 'call', $pre;
 	}
+	
+    my $newline = join(' ', @flist);
+	dbg("sh/dx newline: $newline") if isdbg('sh/dx');
+	my ($r, $filter, $fno, $user, $expr) = $Spot::filterdef->parse($self, 'spots', $newline, 1);
 
-
-	$from ||= '';
-	$to ||= '';
-	$fromday ||= '';
-	$today ||= '';
-
-	dbg "expr: $expr from: $from to: $to fromday: $fromday today: $today" if isdbg('sh/dx');
+	return (0, "sh/dx parse error '$r' " . $filter) if $r;
+	
+	dbg "sh/dx user: $user expr: $expr from: $from to: $to fromday: $fromday today: $today" if isdbg('sh/dx');
   
 	# now do the search
 
 	if ($self->{_nospawn}) {
-		my @res = Spot::search($expr, $fromday, $today, $from, $to, $hint, $dofilter ? $self : undef);
+		my @res = Spot::search($expr, $fromday, $today, $from, $to, $user, $dofilter ? $self : undef);
 		my $ref;
 		my @dx;
 		foreach $ref (@res) {
@@ -451,7 +146,7 @@ sub handle
 	}
 	else {
 		push @out, $self->spawn_cmd("sh/dx $line", \&Spot::search, 
-									args => [$expr, $fromday, $today, $from, $to, $hint, $dofilter ? $self : undef],
+									args => [$expr, $fromday, $today, $from, $to, $filter, $dofilter ? $self : undef],
 									cb => sub {
 										my ($dxchan, @res) = @_; 
 										my $ref;
diff --git a/perl/Filter.pm b/perl/Filter.pm
index 6b5cad94..2be5cd5b 100644
--- a/perl/Filter.pm
+++ b/perl/Filter.pm
@@ -364,7 +364,7 @@ use vars qw(@ISA);
 # this is called as a subroutine not as a method
 sub parse
 {
-	my ($self, $dxchan, $sort, $line) = @_;
+	my ($self, $dxchan, $sort, $line, $forcenew) = @_;
 	my $ntoken = 0;
 	my $fno = 1;
 	my $filter;
@@ -373,7 +373,8 @@ sub parse
 	my $user;
 	
 	# check the line for non legal characters
-	return ('ill', $dxchan->msg('e19')) if $line !~ /{.*}/ && $line =~ /[^\s\w,_\-\*\/\(\)!]/;
+	dbg("Filter::parse line: '$line'") if isdbg('filter');
+	return ('ill', $dxchan->msg('e19')) if $line !~ /{.*}/ && $line =~ /[^\s\w,_\-\*\/\(\)\$!]/;
 	
 	# add some spaces for ease of parsing
 	$line =~ s/([\(\)])/ $1 /g;
@@ -385,7 +386,7 @@ sub parse
 	while (@f) {
 		if ($ntoken == 0) {
 			
-			if (@f && $dxchan->priv >= 8 && ((is_callsign(uc $f[0]) && DXUser::get(uc $f[0])) || $f[0] =~ /(?:node|user)_default/)) {
+			if (!$forcenew &&  @f && $dxchan->priv >= 8 && ((is_callsign(uc $f[0]) && DXUser::get(uc $f[0])) || $f[0] =~ /(?:node|user)_default/)) {
 				$call = shift @f;
 				if ($f[0] eq 'input') {
 					shift @f;
@@ -399,7 +400,7 @@ sub parse
 				$fno = shift @f;
 			}
 
-			$filter = Filter::read_in($sort, $call, $flag);
+			$filter = Filter::read_in($sort, $call, $flag) unless $forcenew;
 			$filter = Filter->new($sort, $call, $flag) if !$filter || $filter->isa('Filter::Old');
 			
 			$ntoken++;
@@ -537,7 +538,7 @@ sub parse
 	$user =~ s/\!/ not /g;
 	$user =~ s/\s+/ /g;
 	
-	return (0, $filter, $fno, $user, "$s");
+	return (0, $filter, $fno, $user, $s);
 }
 
 # a filter accept/reject command
diff --git a/perl/Spot.pm b/perl/Spot.pm
index 840913d1..185711b5 100644
--- a/perl/Spot.pm
+++ b/perl/Spot.pm
@@ -39,16 +39,23 @@ $filterdef = bless ([
 					 ['on', 'r', 0, 0, \&decodefreq],
 					 ['call', 'c', 1],
 					 ['info', 't', 3],
+					 ['spotter', 'c', 4],
 					 ['by', 'c', 4],
 					 ['call_dxcc', 'nc', 5],
 					 ['by_dxcc', 'nc', 6],
 					 ['origin', 'c', 7, 9],
 					 ['call_itu', 'ni', 8],
+					 ['itu', 'ni', 8],
 					 ['call_zone', 'nz', 9],
+					 ['zone', 'nz', 9],
 					 ['by_itu', 'ni', 10],
+					 ['byitu', 'ni', 10],
 					 ['by_zone', 'nz', 11],
+					 ['byzone', 'nz', 11],
 					 ['call_state', 'ns', 12],
+					 ['state', 'ns', 12],
 					 ['by_state', 'ns', 13],
+					 ['bystate', 'ns', 13],
 					 ['ip', 'c', 14],
 #					 ['channel', 'c', 15],
 #					 ['rbn', 'a', 4, 0, \&filterrbnspot],
@@ -56,7 +63,7 @@ $filterdef = bless ([
 $totalspots = $hfspots = $vhfspots = 0;
 $use_db_for_search = 0;
 
-our $usetac = 0;
+our $usetac = 1;
 our $readback;
 
 if ($usetac) {
@@ -176,6 +183,8 @@ sub prefix
 }
 
 # fix up the full spot data from the basic spot data
+# input is
+# freq, call, time, comment, spotter, origin[, ip_address]
 sub prepare
 {
 	# $freq, $call, $t, $comment, $spotter, node, ip address = @_
@@ -201,7 +210,7 @@ sub prepare
 	push @out, $_[6] if $_[6] && is_ipaddr($_[6]);
 
 	# thus we now have:
-	# freq, call, time, comment, spotter, call country code, call itu, call cqzone, spotter country code, spotter itu, spotter cqzone, call state, spotter state, node, spotter ip address
+	# freq, call, time, comment, spotter, call country code, spotter country code, origin, call itu, call cqzone, spotter itu, spotter cqzone, call state, spotter state, spotter ip address
 	return @out;
 }
 
@@ -240,7 +249,13 @@ sub add
 #   $f5 = spotted dxcc country
 #   $f6 = spotter dxcc country
 #   $f7 = origin
-#   $f8 = ip address
+#   $f8 = spotted itu
+#   $f9 = spotted cq zone
+#   $f10 = spotter itu
+#   $f11 = spotter cq zone
+#   $f12 = spotted us state
+#   $f13 = spotter us state
+#   $f14 = ip address
 #
 # In addition you can specify a range of days, this means that it will start searching
 # from <n> days less than today to <m> days less than today
@@ -254,7 +269,6 @@ sub add
 sub search
 {
 	my ($expr, $dayfrom, $dayto, $from, $to, $hint, $dxchan) = @_;
-	my $eval;
 	my @out;
 	my $ref;
 	my $i;
@@ -276,90 +290,62 @@ sub search
 	$to = $from + $maxspots if $to - $from > $maxspots || $to - $from <= 0;
 
 	if ($main::dbh && $use_db_for_search) {
-		return $main::dbh->spot_search($expr, $dayfrom, $dayto, $to-$from, $dxchan);
+		return $main::dbh->spot_search($hint, $dayfrom, $dayto, $to-$from, $dxchan);
 	}
 
-	$expr =~ s/\$f(\d\d?)/\$ref->[$1]/g; # swap the letter n for the correct field name
+#	$expr =~ s/\$f(\d\d?)/\$ref->[$1]/g; # swap the letter n for the correct field name
 	#  $expr =~ s/\$f(\d)/\$spots[$1]/g;               # swap the letter n for the correct field name
   
-	my $checkfilter;
-	$checkfilter = qq (
-                      if (\@s < 9) {
-                          my \@a = (Prefix::cty_data(\$s[1]))[1..3];
-                          my \@b = (Prefix::cty_data(\$s[4]))[1..3];
-                          push \@s, \@a[0,1], \@b[0,1], \$a[2], \$a[2];  
-                      } else {
-                          \$s[12] ||= ' ';
-                          \$s[13] ||= ' ';
-                      }
-	                  my (\$filter, \$hops) = \$dxchan->{spotsfilter}->it(\@s);
-	                  next unless (\$filter);
-                      ) if $dxchan;
-	$checkfilter ||= ' ';
-	
-	dbg("hint='$hint', expr='$expr', spotno=$from-$to, day=$dayfrom-$dayto\n") if isdbg('search');
+
+	dbg("Spot::search hint='$hint', expr='$expr', spotno=$from-$to, day=$dayfrom-$dayto\n") if isdbg('search');
   
 	# build up eval to execute
-	$eval = qq(
-			   while (<\$fh>) {
-				   $hint;
-				   chomp;
-				   my \@s = split /\\^/;
-                   $checkfilter;
-                   push \@spots, \\\@s;
-                   shift \@spots if \@spots > $to + 2; 
-			   }
-			   my \$c;
-			   my \$ref;
-               if (\$readback) {
-                   foreach \$ref (\@spots) {
-		               if ($expr) {
-						   \$count++;
-						   next if \$count < $from; # wait until from 
-						   push(\@out, \$ref);
-						   last if \$count >= $to; # stop after to
-			           }         
-                   }
-               } else {
-			      for (\$c = \$#spots; \$c >= 0; \$c--) {
-					   \$ref = \$spots[\$c];
-					   if ($expr) {
-						   \$count++;
-						   next if \$count < $from; # wait until from 
-						   push(\@out, \$ref);
-						   last if \$count >= $to; # stop after to
-					   }
-				   }
-               }
-			  );
-	         
+	dbg("Spot::search Spot eval: $expr") if isdbg('searcheval');
+	$expr =~ s/\$r/\$_[0]/g;
+	my $eval = qq{ sub { return $expr; } };
+	dbg("Spot::search Spot eval: $eval") if isdbg('searcheval');
+	my $ecode = eval $eval;
+	return ("Spot search error", $@) if $@;
 	
-    
-	dbg("Spot eval: $eval") if isdbg('searcheval');
 	
 	my $fh;
 	my $now = $fromdate;
-	for ($i = $count = 0; $i < $maxdays; ++$i) {	# look thru $maxdays worth of files only
-		my @spots;
+	my @spots;
+	my $recs;
+	
+	for ($i = $count = 0; $count < $to && $i < $maxdays; ++$i) {	# look thru $maxdays worth of files only
 		last if $now->cmp($todate) <= 0;
 		
+		my $fn = $fp->fn($now->sub($i));
 		if ($readback) {
-			my $fn = $fp->fn($now->sub($i));
-			dbg("search using tac fn: $fn $i") if isdbg('search');
+			dbg("Spot::search search using tac fn: $fn $i") if isdbg('search');
 			$fh = IO::File->new("$readback $fn |");
 		} else {
+			dbg("Spot::search search fn: $fp->{fn} $i") if isdbg('search');
 			$fh = $fp->open($now->sub($i));      # get the next file
-			dbg("search fn: $fp->{fn} $i") if isdbg('search');
 		}
 		if ($fh) {
+			my $rec = 0;
 			my $in;
-			eval $eval;			# do the search on this file
+			while (<$fh>) {
+				my @r = split /\^/;
+				++$rec;
+				if (&$ecode(\@r)) {
+					++$count;
+					next if $count < $from;
+					push @out, \@r;
+					last if $count >= $to;
+				}
+			}
+			dbg("Spot::search recs read: $rec") if isdbg('search');
 			last if $count >= $to; # stop after to
+			
 			return ("Spot search error", $@) if $@;
 		}
 	}
+   
 
-	return @out;
+	return $readback ? @out : reverse @out;
 }
 
 # change a freq range->regular expression
-- 
2.34.1