Files
agc_2-X/bin/VICIDIAL_international_DNC_parser_2.pl
T
mattf f9af96ebc0 Added International DNC scrub features
git-svn-id: svn://192.168.202.10@3273 3d104415-ff17-0410-8863-d5cf3c621b8a
2020-08-15 03:08:08 +00:00

572 lines
17 KiB
Perl

#!/usr/bin/perl
#
# VICIDIAL_international_DNC_parser_2.pl
#
# This is the second part of a two-step program in the DNC importing process
#
# It requires that a system settings container exists in your system settings
# named "INTERNATIONAL_DNC_IMPORT". Two run-time variables must be defined
# in said container - "--file-dir" and "--file-destination"
#
# It also requires a second settings container named "DNC_IMPORT_FORMATS",
# which it will use to determine how to parse the data in the file.
#
# This should run every minute to check the DNC queue table for files
# marked as PENDING, which it will then try to process based on the import
# format defined through the web interface.
#
# As the file is being processed, the web interface will reflect the progress
# of this script. Only one file will be processed at any time.
#
# Put an entry into the cron of of your asterisk machine to run this script
# every minute or however often you desire
#
# ### International DNC importing scripts
##* * * * * /usr/share/astguiclient/VICIDIAL_international_DNC_parser_2.pl
#
# Copyright (C) 2020 Matt Florell <vicidial@gmail.com>, Joe Johnson <joej@vicidial.com> LICENSE: AGPLv2
#
# CHANGE LOG:
# 200813-1230 - First Build
#
use DBI;
use Time::Local;
# default path to astguiclient configuration file:
$PATHconf = '/etc/astguiclient.conf';
open(conf, "$PATHconf") || die "can't open $PATHconf: $!\n";
@conf = <conf>;
close(conf);
$i=0;
foreach(@conf)
{
$line = $conf[$i];
$line =~ s/ |>|\n|\r|\t|\#.*|;.*//gi;
if ( ($line =~ /^PATHhome/) && ($CLIhome < 1) )
{$PATHhome = $line; $PATHhome =~ s/.*=//gi;}
if ( ($line =~ /^PATHlogs/) && ($CLIlogs < 1) )
{$PATHlogs = $line; $PATHlogs =~ s/.*=//gi;}
if ( ($line =~ /^PATHagi/) && ($CLIagi < 1) )
{$PATHagi = $line; $PATHagi =~ s/.*=//gi;}
if ( ($line =~ /^PATHweb/) && ($CLIweb < 1) )
{$PATHweb = $line; $PATHweb =~ s/.*=//gi;}
if ( ($line =~ /^PATHsounds/) && ($CLIsounds < 1) )
{$PATHsounds = $line; $PATHsounds =~ s/.*=//gi;}
if ( ($line =~ /^PATHmonitor/) && ($CLImonitor < 1) )
{$PATHmonitor = $line; $PATHmonitor =~ s/.*=//gi;}
if ( ($line =~ /^VARserver_ip/) && ($CLIserver_ip < 1) )
{$VARserver_ip = $line; $VARserver_ip =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_server/) && ($CLIDB_server < 1) )
{$VARDB_server = $line; $VARDB_server =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_database/) && ($CLIDB_database < 1) )
{$VARDB_database = $line; $VARDB_database =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_user/) && ($CLIDB_user < 1) )
{$VARDB_user = $line; $VARDB_user =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_pass/) && ($CLIDB_pass < 1) )
{$VARDB_pass = $line; $VARDB_pass =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_custom_user/) && ($CLIDB_custom_user < 1) )
{$VARDB_custom_user = $line; $VARDB_custom_user =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_custom_pass/) && ($CLIDB_custom_pass < 1) )
{$VARDB_custom_pass = $line; $VARDB_custom_pass =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_port/) && ($CLIDB_port < 1) )
{$VARDB_port = $line; $VARDB_port =~ s/.*=//gi;}
if ( ($line =~ /^VARREPORT_host/) && ($CLIREPORT_host < 1) )
{$VARREPORT_host = $line; $VARREPORT_host =~ s/.*=//gi;}
if ( ($line =~ /^VARREPORT_user/) && ($CLIREPORT_user < 1) )
{$VARREPORT_user = $line; $VARREPORT_user =~ s/.*=//gi;}
if ( ($line =~ /^VARREPORT_pass/) && ($CLIREPORT_pass < 1) )
{$VARREPORT_pass = $line; $VARREPORT_pass =~ s/.*=//gi;}
if ( ($line =~ /^VARREPORT_port/) && ($CLIREPORT_port < 1) )
{$VARREPORT_port = $line; $VARREPORT_port =~ s/.*=//gi;}
if ( ($line =~ /^VARREPORT_dir/) && ($CLIREPORT_dir < 1) )
{$VARREPORT_dir = $line; $VARREPORT_dir =~ s/.*=//gi;}
$i++;
}
$dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
or die "Couldn't connect to database: " . DBI->errstr;
$dbhB = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
or die "Couldn't connect to database: " . DBI->errstr;
$dbhC = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_custom_user", "$VARDB_custom_pass")
or die "Couldn't connect to database: " . DBI->errstr;
if ($run_check > 0)
{
my $grepout = `/bin/ps ax | grep $0 | grep -v grep | grep -v '/bin/sh'`;
my $grepnum=0;
$grepnum++ while ($grepout =~ m/\n/g);
if ($grepnum > 1)
{
if ($DB) {print "I am not alone! Another $0 is running! Exiting...\n";}
exit;
}
}
### begin parsing run-time options ###
if (length($ARGV[0])>1)
{
$i=0;
while ($#ARGV >= $i)
{
$args = "$args $ARGV[$i]";
$i++;
}
if ($args =~ /--help/i)
{
print "allowed run time options:\n";
print " [--help] = this screen\n";
print " [--debug] = debug\n";
print " [--debugX] = super debug\n";
print " [--FTPdb] = show FTP debug output\n";
print " [--test] = will execute script and transfer files, but not delete lists from dialer\n";
print "\n";
exit;
}
else
{
if ($args =~ /--debug/i)
{
$DB=1;
print "\n----- DEBUG -----\n\n";
}
if ($args =~ /--debugX/i)
{
$DB=1;
$DBX=1;
print "\n----- SUPER DEBUG -----\n\n";
}
if ($args =~ /--FTPdb/i)
{
$FTPdb=1;
print "\n----- FTP DEBUG -----\n\n";
}
if ($args =~ /--test/i)
{
$T=1;
print "\n----- TEST EXECUTION -----\n\n";
}
}
}
$stmt="SELECT enable_international_dncs FROM system_settings;";
$rslt=$dbhA->prepare($stmt);
if ($DB) {print "$stmt\n";}
$rslt->execute();
if ($rslt->rows>0)
{
@row=$rslt->fetchrow_array;
if ($row[0]==0 || !$row[0])
{
if ($DB) {print "Feature not enabled.\n"; exit;}
}
}
else
{
if ($DB) {print "Database error!\n"; exit;}
}
$secX = time();
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
$year = ($year + 1900);
$mon++;
$rowdate = "$mon/$mday/$year";
if ($mon < 10) {$mon = "0$mon";}
if ($mday < 10) {$mday = "0$mday";}
if ($min < 10) {$min = "0$min";}
if ($hour < 10) {$hour = "0$hour";}
if ($sec < 10) {$sec = "0$sec";}
$run_time="$year-$mon-$mday $hour:$min:$sec";
$stmt="select * from vicidial_settings_containers where container_id='INTERNATIONAL_DNC_IMPORT' and container_type='PERL_CLI'";
if ($DBX) {print "\n$stmt\n\n";}
$rslt=$dbhA->prepare($stmt);
$rslt->execute();
if ($rslt->rows>0)
{
@row=$rslt->fetchrow_array;
$row[4]=~s/\r//g;
@container_settings=split(/\n/, $row[4]);
for ($i=0; $i<scalar(@container_settings); $i++)
{
$args=$container_settings[$i];
$args=~s/^\s+|\s+$//;
if ($args!~/^#/)
{
if ($args =~ /--ftp-host/i)
{
my @data_in = split(/--ftp-host=/,$args);
$ftp_host = $data_in[1];
$ftp_host =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP HOST: $ftp_host -----\n\n";}
}
if ($args =~ /--ftp-user/i)
{
my @data_in = split(/--ftp-user=/,$args);
$ftp_user = $data_in[1];
$ftp_user =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP USER: $ftp_user -----\n\n";}
}
if ($args =~ /--ftp-pwd/i)
{
my @data_in = split(/--ftp-pwd=/,$args);
$ftp_pwd = $data_in[1];
$ftp_pwd =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP PASSWORD: $ftp_pwd -----\n\n";}
}
if ($args =~ /--ftp-port/i)
{
my @data_in = split(/--ftp-port=/,$args);
$ftp_port = $data_in[1];
$ftp_port =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP PORT: $ftp_port -----\n\n";}
}
if ($args =~ /--ftp-passive/i)
{
my @data_in = split(/--ftp-passive=/,$args);
$ftp_passive = $data_in[1];
$ftp_passive =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP PASSIVE: $ftp_passive -----\n\n";}
}
if ($args =~ /--file-dir/i)
{
my @data_in = split(/--file-dir=/,$args);
$file_dir = $data_in[1];
$file_dir =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP DIRECTORY: $file_dir -----\n\n";}
}
if ($args =~ /--file-destination/i)
{
my @data_in = split(/--file-destination=/,$args);
$file_destination = $data_in[1];
$file_destination =~ s/ .*//gi;
if ($DBX) {print "\n----- FTP DESTINATION: $file_destination -----\n\n";}
}
}
}
$rslt->finish();
$skip_ftp=0;
if (!$ftp_host)
{
if ($DB) {print "No FTP host given.\n";}
$skip_ftp++;
# exit;
}
if (!$ftp_user)
{
if ($DB) {print "No FTP user given.\n";}
$skip_ftp++;
# exit;
}
if (!$ftp_pwd)
{
if ($DB) {print "No FTP password given.\n";}
$skip_ftp++;
# exit;
}
if (!$ftp_port)
{
$ftp_port=21;
}
if (!$ftp_passive)
{
$ftp_passive=0;
}
if (!$file_dir)
{
print "Script failed - no directory to look up files.\n";
exit;
}
if (!-e $file_dir || (-e $file_dir && !-d $file_dir))
{
print "Script failed - file directory does not exist or is not a directory.\n";
exit;
}
if (!$file_destination)
{
print "Script failed - no destination to move completed files to.\n";
exit;
}
if (!-e $file_destination || (-e $file_destination && !-d $file_destination))
{
print "Script failed - file destination directory does not exist or is not a directory.\n";
exit;
}
if ($file_dir eq $file_destination)
{
print "Script failed - please choose a destination for the completed files that is different from where they're already located.\n";
exit;
}
# Get file layouts
$layout_stmt="select container_entry from vicidial_settings_containers where container_id='DNC_IMPORT_FORMATS'";
if ($DBX) {print "$layout_stmt\n\n";}
$layout_rslt=$dbhB->prepare($layout_stmt);
$layout_rslt->execute();
if ($layout_rslt->rows==0) {die "No import formats container found in settings containers";}
@layout_row=$layout_rslt->fetchrow_array;
@layout_array=split(/\n/, $layout_row[0]);
%layouts=();
for ($i=0; $i<scalar(@layout_array); $i++)
{
$entry=$layout_array[$i];
$entry=~s/^\s+//;
if ($entry!~/^#/ && $entry=~/\=\>/)
{
@current_layout=split(/\=\>/, $entry);
$current_layout[0]=~s/^\s+|\s+$//;
$current_layout[1]=~s/^\s+|\s+$//;
if (length($current_layout[0]))
{
$layouts{"$current_layout[0]"}=$current_layout[1];
}
}
}
$layout_rslt->finish;
# select from database where file is marked as ready to queue and country code is set
$process_stmt="select filename, file_layout, country_code, file_action, dnc_file_id from vicidial_country_dnc_queue where file_status='PENDING' and country_code is not null and file_layout is not null order by date_uploaded asc limit 1";
if ($DBX) {print "$process_stmt\n\n";}
$process_rslt=$dbhA->prepare($process_stmt);
$process_rslt->execute();
if ($process_rslt->rows==0) {exit;}
while(@process_row=$process_rslt->fetchrow_array)
{
$filename=$process_row[0];
$file_layout=$process_row[1];
$country_code=$process_row[2];
$file_action=$process_row[3];
$dnc_file_id=$process_row[4];
$process_count=0;
$records_processed=0;
$records_inserted=0;
if (!exists $layouts{"$file_layout"})
{
$upd_stmt="update vicidial_country_dnc_queue set file_status='INVALID LAYOUT' where dnc_file_id='$dnc_file_id'";
if ($DB) {print "Can't find file layout ($file_layout) - $upd_stmt\n\n";}
$upd_rslt=$dbhB->prepare($upd_stmt);
$upd_rslt->execute();
$upd_rslt->finish;
exit;
}
else
{
$create_stmt="CREATE TABLE IF NOT EXISTS vicidial_dnc_".$country_code."(phone_number varchar(20) primary key)";
if ($DB) {print "$create_stmt\n\n";}
$create_rslt=$dbhC->prepare($create_stmt);
$create_rslt->execute();
$create_rslt->finish;
if ($file_action eq "PURGE")
{
$delete_stmt="delete from vicidial_dnc_".$country_code;
if ($DBX) {print "$delete_stmt\n\n";}
$delete_rslt=$dbhB->prepare($delete_stmt);
$delete_rslt->execute();
$delete_rslt->finish;
$opt_stmt="optimize table vicidial_dnc_".$country_code;
if ($DBX) {print "$opt_stmt\n\n";}
$opt_rslt=$dbhB->prepare($opt_stmt);
$opt_rslt->execute();
$opt_rslt->finish;
}
@layout_stats=split(/,/, $layouts{"$file_layout"});
if ($DBX) {print $layouts{"$file_layout"}."\n";}
$txt_format=$layout_stats[0];
$delimiter=$layout_stats[1];
if ($filename=~/csv$/i)
{
$delimiter="comma";
}
$delimiter=~s/.*comma.*/,/gi;
$delimiter=~s/.*pipe.*/\\\|/gi;
$delimiter=~s/.*tab.*/\t/gi;
if ($txt_format ne "" && $DB) {print "Format: $txt_format\n\n";}
if ($delimiter ne "" && $DB) {print "Delimiter: $delimiter\n\n";}
@phone_fields=();
for($i=2; $i<scalar(@layout_stats); $i++)
{
push(@phone_fields, $layout_stats[$i]);
}
if (scalar(@phone_fields)==0 || ($txt_format=~/delimit/i && $delimiter eq "") || ($txt_format=~/fixed/i && $delimiter ne ""))
{
$upd_stmt="update vicidial_country_dnc_queue set file_status='INVALID LAYOUT' where dnc_file_id='$dnc_file_id'";
if ($DB) {print "No phone fields defined ($file_layout) - $upd_stmt\n\n";}
$upd_rslt=$dbhB->prepare($upd_stmt);
$upd_rslt->execute();
$upd_rslt->finish;
exit;
}
}
if (!$skip_ftp) # File is NOT local, so attempt to download.
{
use Net::Ping;
use Net::FTP;
$ftp = Net::FTP->new("$ftp_host", Port => $ftp_port, Passive => $ftp_passive, Debug => $FTPdb);
$ftp->login("$ftp_user","$ftp_pwd");
if ($file_dir) {$ftp->cwd("$file_dir");}
$ftp->get("$filename", "/tmp/$filename");
$ftp->quit;
$full_filename="/tmp/$filename";
}
else
{
$full_filename="$file_dir/$filename";
}
if ($filename=~/xlsx?$/)
{
use Spreadsheet::Read;
use Spreadsheet::XLSX;
use File::Basename;
use open ':std', ':encoding(UTF-8)';
# Gotta account for the fact that spreadsheets don't start with 0 for indices
for($k=0; $k<scalar(@phone_fields); $k++) {$phone_fields[$k]++;}
$datafile=Spreadsheet::Read->new("$full_filename");
$sheets=$datafile->[0]{sheets};
for ($i=1; $i<=$sheets; $i++)
{
$current_sheet=$datafile->[$i];
$sheet_rows=$datafile->[$i]{maxrow};
$sheet_cols=$datafile->[$i]{maxcol};
$upd_stmt="update vicidial_country_dnc_queue set file_status='PROCESSING', total_records='$sheet_rows' where dnc_file_id='$dnc_file_id'";
if ($DB) {print "$upd_stmt\n\n";}
$upd_rslt=$dbhB->prepare($upd_stmt);
$upd_rslt->execute();
$upd_rslt->finish;
for ($j=1; $j<=$sheet_rows; $j++)
{
$records_processed++;
$phone_number="";
for($k=0; $k<scalar(@phone_fields); $k++)
{
$phone_number.=$datafile->[$i]{cell}[$phone_fields[$k]][$j];
}
InsertPhoneNumber($records_processed, $records_inserted, $phone_number, $dnc_file_id);
}
}
}
else
{
$total_records_str=`wc -l $full_filename`;
@tr_str=split(/\s/, $total_records_str);
$total_records=$tr_str[0];
$upd_stmt="update vicidial_country_dnc_queue set file_status='PROCESSING', total_records='$total_records' where dnc_file_id='$dnc_file_id'";
if ($DB) {print "$upd_stmt\n\n";}
$upd_rslt=$dbhB->prepare($upd_stmt);
$upd_rslt->execute();
$upd_rslt->finish;
open(PHONE_FILE, "$full_filename");
while($buffer=<PHONE_FILE>)
{
$records_processed++;
$phone_number="";
if ($delimiter ne "")
{
@data=split(/$delimiter/, $buffer);
for($i=0; $i<scalar(@phone_fields); $i++)
{
$phone_number.=$data[$phone_fields[$i]];
}
}
else
{
for($i=0; $i<scalar(@phone_fields); $i++)
{
@data=split(/\|/, $phone_fields[$i]);
$phone_number.=substr($buffer, $data[0], $data[1]);
}
}
if ($records_processed%1000==0) {print "*$phone_number*\n";}
InsertPhoneNumber($records_processed, $records_inserted, $phone_number, $dnc_file_id);
}
}
close(PHONE_FILE);
$upd_stmt="update vicidial_country_dnc_queue set records_processed='$records_processed', records_inserted='$records_inserted', date_processed=now(), file_status='FINISHED' where dnc_file_id='$dnc_file_id'";
if ($DBX) {print "$upd_stmt\n\n";}
$upd_rslt=$dbhB->prepare($upd_stmt);
$upd_rslt->execute();
$upd_rslt->finish;
if (!$skip_ftp) # File is NOT local, so attempt to download.
{
use Net::Ping;
use Net::FTP;
$ftp = Net::FTP->new("$ftp_host", Port => $ftp_port, Passive => $ftp_passive, Debug => $FTPdb);
$ftp->login("$ftp_user","$ftp_pwd");
if ($file_dir) {$ftp->cwd("$file_dir");}
# $ftp->get("$filename", "/tmp/$filename");
# $ftp->quit;
# $full_filename="/tmp/$filename";
}
else
{
`mv $file_dir/$filename $file_destination/$filename`;
}
}
$dbhA->disconnect();
$dbhB->disconnect();
$dbhC->disconnect();
}
else
{
$dbhA->disconnect();
$dbhB->disconnect();
$dbhC->disconnect();
exit;
}
sub InsertPhoneNumber
{
my $rp=$_[0];
my $ri=$_[1];
my $phone=$_[2];
my $dfid=$_[3];
$phone=~s/[^0-9]//gi;
if (length($phone)>=5 && length($phone)<=18)
{
$ins_stmt="insert ignore into vicidial_dnc_".$country_code." values('$phone')";
$ins_rslt=$dbhB->prepare($ins_stmt);
$ins_rslt->execute();
if ($ins_rslt->rows>0) {$records_inserted++; $ri++;}
$ins_rslt->finish;
}
if ($rp%1000==0)
{
$upd_stmt="update vicidial_country_dnc_queue set records_processed='$rp', records_inserted='$ri' where dnc_file_id='$dfid'";
if ($DBX) {print "$ins_stmt\n$upd_stmt\n";}
$upd_rslt=$dbhB->prepare($upd_stmt);
$upd_rslt->execute();
$upd_rslt->finish;
}
}