From f9af96ebc053e1cbf031ddec640a5d0c492879a7 Mon Sep 17 00:00:00 2001 From: mattf Date: Sat, 15 Aug 2020 03:08:08 +0000 Subject: [PATCH] Added International DNC scrub features git-svn-id: svn://192.168.202.10@3273 3d104415-ff17-0410-8863-d5cf3c621b8a --- UPGRADE | 3 + bin/AST_VDhopper.pl | 43 +- bin/VICIDIAL_international_DNC_parser_1.pl | 510 ++++++++++++++++ bin/VICIDIAL_international_DNC_parser_2.pl | 571 ++++++++++++++++++ docs/INTERNATIONAL_DNC_SCRUB.txt | 53 ++ extras/MySQL_AST_CREATE_tables.sql | 36 +- extras/upgrade_2.14.sql | 24 + www/vicidial/AST_CMstats.php | 2 +- www/vicidial/AST_IVRstats.php | 2 +- www/vicidial/AST_LISTS_campaign_stats.php | 2 +- www/vicidial/AST_LISTS_pass_report.php | 2 +- www/vicidial/AST_LISTS_stats.php | 2 +- www/vicidial/AST_VDADstats.php | 2 +- www/vicidial/AST_admin_template_maker.php | 2 +- www/vicidial/AST_agent_timeclock_detail.php | 4 +- www/vicidial/AST_dialer_inventory_report.php | 2 +- www/vicidial/AST_timeonVDADall.php | 4 +- www/vicidial/AST_timeonVDADallREC.php | 3 +- www/vicidial/AST_timeonVDADallSUMMARY.php | 4 +- www/vicidial/AST_timeonVDADall_mobile.php | 4 +- www/vicidial/KHOMP_admin.php | 11 +- .../Vicidial_international_DNC_interface.php | 405 +++++++++++++ www/vicidial/admin.php | 344 ++++++----- www/vicidial/admin_NANPA_updater.php | 8 +- www/vicidial/admin_amm_multi.php | 5 +- www/vicidial/admin_bulk_tools.php | 28 +- www/vicidial/admin_header.php | 6 +- www/vicidial/admin_listloader_fourth_gen.php | 171 +++++- www/vicidial/help_documentation.txt | 8 +- www/vicidial/screen_colors.php | 11 +- www/vicidial/vicidial_stylesheet.php | 3 + 31 files changed, 2044 insertions(+), 231 deletions(-) create mode 100644 bin/VICIDIAL_international_DNC_parser_1.pl create mode 100644 bin/VICIDIAL_international_DNC_parser_2.pl create mode 100644 docs/INTERNATIONAL_DNC_SCRUB.txt create mode 100644 www/vicidial/Vicidial_international_DNC_interface.php diff --git a/UPGRADE b/UPGRADE index 584a0dc5..b51e89b5 100644 --- a/UPGRADE +++ b/UPGRADE @@ -516,6 +516,9 @@ OTHER CHANGES: 144. Added options for HTML formatting and UTF-8 charset in emails through the dispo_send_email.php script. +145. Added tools for International DNC phone number scrubbing. See the + INTERNATIONAL_DNC_SCRUB.txt doc for more information. + diff --git a/bin/AST_VDhopper.pl b/bin/AST_VDhopper.pl index 73dc1fb0..f348e510 100644 --- a/bin/AST_VDhopper.pl +++ b/bin/AST_VDhopper.pl @@ -21,7 +21,7 @@ # - R = Recycled leads # - S = Standard hopper load # -# Copyright (C) 2019 Matt Florell LICENSE: AGPLv2 +# Copyright (C) 2020 Matt Florell LICENSE: AGPLv2 # # CHANGELOG # 50810-1613 - Added database server variable definitions lookup @@ -95,6 +95,7 @@ # 190213-1207 - Added additional $VLforce_index flags, for high-volume dialing systems # 190524-1228 - Fix for lead filters with 'NONE' in the filter ID # 190703-1650 - Allow for single-quotes in state field +# 200814-2132 - Added support for Internation DNC scrubbing # # constants @@ -326,7 +327,7 @@ $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VA ### Grab system_settings values from the database $anyone_callback_inactive_lists='default'; -$stmtA = "SELECT anyone_callback_inactive_lists FROM system_settings;"; +$stmtA = "SELECT anyone_callback_inactive_lists,enable_international_dncs FROM system_settings;"; $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; $sthArows=$sthA->rows; @@ -334,6 +335,7 @@ if ($sthArows > 0) { @aryA = $sthA->fetchrow_array; $anyone_callback_inactive_lists = $aryA[0]; + $enable_international_dncs = $aryA[1]; } $sthA->finish(); @@ -354,6 +356,35 @@ if ($sthArows > 0) } $sthA->finish(); +### Grab "blocked" lists from settings container if international DNCs activated +if ($enable_international_dncs) + { + $dnc_list_stmt="select container_entry from vicidial_settings_containers where container_id='DNC_CURRENT_BLOCKED_LISTS' and container_entry!=''"; + if ($DBX) {print "$dnc_list_stmt\n";} + $dnc_list_rslt=$dbhA->prepare($dnc_list_stmt); + $dnc_list_rslt->execute(); + if ($dnc_list_rslt->rows>0) + { + $dnc_blocked_lists=""; + @dnc_list_row=$dnc_list_rslt->fetchrow_array; + + @blocked_dnc_lists=split(/\n/, $dnc_list_row[0]); + for ($i=0; $i/, $blocked_dnc_lists[$i]); + $dnc_list_id=$current_scrub[0]; + $dnc_list_id=~s/[^0-9]//g; + $dnc_blocked_lists.="'$dnc_list_id', "; + } + $dnc_blocked_lists=~s/, $//; + } + if ($DBX) {print "BLOCKED LISTS: $dnc_blocked_lists\n";} + if (length($dnc_blocked_lists)>0) + { + $dnc_blocked_lists_SQL="and list_id not in ($dnc_blocked_lists)"; + } + $dnc_list_rslt->finish; + } $secX = time(); ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($secX); @@ -2752,7 +2783,7 @@ foreach(@campaign_id) if ($hopper_vlc_dup_check[$i] =~ /Y/) {$vlc_dup_check_SQL = "and vendor_lead_code NOT IN($live_vlc$vlc_lists)";} - $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where $recycle_SQL[$i] and ($list_id_sql[$i]) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $order_stmt limit $hopper_level[$i];"; + $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where $recycle_SQL[$i] and ($list_id_sql[$i]) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $dnc_blocked_lists_SQL $order_stmt limit $hopper_level[$i];"; if ($DBX) {print " |$stmtA|\n";} $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; @@ -2808,7 +2839,7 @@ foreach(@campaign_id) if ($hopper_vlc_dup_check[$i] =~ /Y/) {$vlc_dup_check_SQL = "and vendor_lead_code NOT IN($live_vlc$vlc_lists)";} - $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where called_since_last_reset='N' and status IN('NEW') and ($list_id_sql[$i]) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $order_stmt limit $NEW_level;"; + $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where called_since_last_reset='N' and status IN('NEW') and ($list_id_sql[$i]) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $dnc_blocked_lists_SQL $order_stmt limit $NEW_level;"; if ($DBX) {print " |$stmtA|\n";} $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; @@ -2862,7 +2893,7 @@ foreach(@campaign_id) if ($list_order_mix[$i] =~ /DISABLED/) { - $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where called_since_last_reset='N' and status IN($STATUSsql[$i]) and ($list_id_sql[$i]) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $order_stmt limit $OTHER_level;"; + $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where called_since_last_reset='N' and status IN($STATUSsql[$i]) and ($list_id_sql[$i]) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $dnc_blocked_lists_SQL $order_stmt limit $OTHER_level;"; if ($DBX) {print " |$stmtA|\n";} $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; @@ -2965,7 +2996,7 @@ foreach(@campaign_id) if ($hopper_vlc_dup_check[$i] =~ /Y/) {$vlc_dup_check_SQL = "and vendor_lead_code NOT IN($live_vlc$vlc_lists)";} - $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where called_since_last_reset='N' and ($list_mix_dialableSQL) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $order_stmt limit $LM_step_goal[$x];"; + $stmtA = "SELECT lead_id,list_id,gmt_offset_now,phone_number,state,status,modify_date,user,vendor_lead_code FROM vicidial_list $VLforce_index where called_since_last_reset='N' and ($list_mix_dialableSQL) and lead_id NOT IN($lead_id_lists) $vlc_dup_check_SQL and ($all_gmtSQL[$i]) $lead_filter_sql[$i] $CCLsql[$i] $DLTsql[$i] $dnc_blocked_lists_SQL $order_stmt limit $LM_step_goal[$x];"; if ($DBX) {print " |$stmtA|\n";} $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; diff --git a/bin/VICIDIAL_international_DNC_parser_1.pl b/bin/VICIDIAL_international_DNC_parser_1.pl new file mode 100644 index 00000000..127e17b1 --- /dev/null +++ b/bin/VICIDIAL_international_DNC_parser_1.pl @@ -0,0 +1,510 @@ +#!/usr/bin/perl +# +# VICIDIAL_international_DNC_parser_1.pl +# +# This is the first part of a two-step program in the DNC importing process +# +# It requires that a system settings container exists in your system +# settings named "INTERNATIONAL_DNC_IMPORT". Two run-time variables +# must be defined in said container - "--file-dir" and "--file-destination" +# +# This should run every minute to check the directory defined in the +# "--file-dir" variable for any files to post and to insert into the DNC +# queue table in the database. It will also purge any CANCELLED files from +# both the queue table and the file folder. +# +# Put an entry into the cron of of your asterisk machine to run this script +# every minute or however often you desire +# +# ### International DNC importing scripts +##* * * * * /usr/share/astguiclient/VICIDIAL_international_DNC_parser_1.pl +# +# Copyright (C) 2020 Matt Florell , Joe Johnson LICENSE: AGPLv2 +# +# CHANGE LOG: +# 200813-1230 - First Build +# + +use DBI; +use Time::Local; + +# default path to astguiclient configuration file: +$PATHconf = '/etc/astguiclient.conf'; + +open(conf, "$PATHconf") || die "can't open $PATHconf: $!\n"; +@conf = ; +close(conf); +$i=0; +foreach(@conf) + { + $line = $conf[$i]; + $line =~ s/ |>|\n|\r|\t|\#.*|;.*//gi; + if ( ($line =~ /^PATHhome/) && ($CLIhome < 1) ) + {$PATHhome = $line; $PATHhome =~ s/.*=//gi;} + if ( ($line =~ /^PATHlogs/) && ($CLIlogs < 1) ) + {$PATHlogs = $line; $PATHlogs =~ s/.*=//gi;} + if ( ($line =~ /^PATHagi/) && ($CLIagi < 1) ) + {$PATHagi = $line; $PATHagi =~ s/.*=//gi;} + if ( ($line =~ /^PATHweb/) && ($CLIweb < 1) ) + {$PATHweb = $line; $PATHweb =~ s/.*=//gi;} + if ( ($line =~ /^PATHsounds/) && ($CLIsounds < 1) ) + {$PATHsounds = $line; $PATHsounds =~ s/.*=//gi;} + if ( ($line =~ /^PATHmonitor/) && ($CLImonitor < 1) ) + {$PATHmonitor = $line; $PATHmonitor =~ s/.*=//gi;} + if ( ($line =~ /^VARserver_ip/) && ($CLIserver_ip < 1) ) + {$VARserver_ip = $line; $VARserver_ip =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_server/) && ($CLIDB_server < 1) ) + {$VARDB_server = $line; $VARDB_server =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_database/) && ($CLIDB_database < 1) ) + {$VARDB_database = $line; $VARDB_database =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_user/) && ($CLIDB_user < 1) ) + {$VARDB_user = $line; $VARDB_user =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_pass/) && ($CLIDB_pass < 1) ) + {$VARDB_pass = $line; $VARDB_pass =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_port/) && ($CLIDB_port < 1) ) + {$VARDB_port = $line; $VARDB_port =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_host/) && ($CLIREPORT_host < 1) ) + {$VARREPORT_host = $line; $VARREPORT_host =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_user/) && ($CLIREPORT_user < 1) ) + {$VARREPORT_user = $line; $VARREPORT_user =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_pass/) && ($CLIREPORT_pass < 1) ) + {$VARREPORT_pass = $line; $VARREPORT_pass =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_port/) && ($CLIREPORT_port < 1) ) + {$VARREPORT_port = $line; $VARREPORT_port =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_dir/) && ($CLIREPORT_dir < 1) ) + {$VARREPORT_dir = $line; $VARREPORT_dir =~ s/.*=//gi;} + $i++; + } + +$dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; +$dbhB = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; +$dbhC = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; +$dbhD = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; +$dbhE = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; + +### begin parsing run-time options ### +if (length($ARGV[0])>1) + { + $i=0; + while ($#ARGV >= $i) + { + $args = "$args $ARGV[$i]"; + $i++; + } + + if ($args =~ /--help/i) + { + print "allowed run time options:\n"; + print " [--help] = this screen\n"; + print " [--debug] = debug\n"; + print " [--debugX] = super debug\n"; + print " [--FTPdb] = show FTP debug output\n"; + print " [--test] = will execute script and transfer files, but not delete lists from dialer\n"; + print "\n"; + exit; + } + else + { + if ($args =~ /--debug/i) + { + $DB=1; + print "\n----- DEBUG -----\n\n"; + } + if ($args =~ /--debugX/i) + { + $DB=1; + $DBX=1; + print "\n----- SUPER DEBUG -----\n\n"; + } + if ($args =~ /--FTPdb/i) + { + $FTPdb=1; + print "\n----- FTP DEBUG -----\n\n"; + } + if ($args =~ /--test/i) + { + $T=1; + print "\n----- TEST EXECUTION -----\n\n"; + } + if ($args =~ /--force-dnc-scrub/i) + { + $force_dnc_scrub = "Y"; + if ($DBX) {print "\n----- FORCING DNC SCRUB -----\n\n";} + } + } + } + +if ($run_check > 0) + { + my $grepout = `/bin/ps ax | grep $0 | grep -v grep | grep -v '/bin/sh'`; + my $grepnum=0; + $grepnum++ while ($grepout =~ m/\n/g); + if ($grepnum > 1) + { + if ($DB) {print "I am not alone! Another $0 is running! Exiting...\n";} + exit; + } + } + +$stmt="SELECT enable_international_dncs FROM system_settings;"; +$rslt=$dbhA->prepare($stmt); +if ($DB) {print "$stmt\n";} +$rslt->execute(); +if ($rslt->rows>0) + { + @row=$rslt->fetchrow_array; + if ($row[0]==0 || !$row[0]) + { + if ($DB) {print "Feature not enabled.\n"; exit;} + } + } +else + { + if ($DB) {print "Database error!\n"; exit;} + } + + +$secX = time(); + ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); +$year = ($year + 1900); +$mon++; +$rowdate = "$mon/$mday/$year"; +if ($mon < 10) {$mon = "0$mon";} +if ($mday < 10) {$mday = "0$mday";} +if ($min < 10) {$min = "0$min";} +if ($hour < 10) {$hour = "0$hour";} +if ($sec < 10) {$sec = "0$sec";} +$run_time="$year-$mon-$mday $hour:$min:$sec"; +$dnc_scrub_status="DNCI"; + +$stmt="select * from vicidial_settings_containers where container_id='INTERNATIONAL_DNC_IMPORT' and container_type='PERL_CLI'"; +if ($DBX) {print "\n$stmt\n\n";} +$rslt=$dbhA->prepare($stmt); +$rslt->execute(); +if ($rslt->rows>0) + { + @row=$rslt->fetchrow_array; + $row[4]=~s/\r//g; + @container_settings=split(/\n/, $row[4]); + for ($i=0; $ifinish(); + + if ($DBX) {print "\n----- DNC STATUS: $dnc_scrub_status -----\n\n";} + + if ($hour<1 || $force_dnc_scrub) { + $scrub_stmt="select container_entry from vicidial_settings_containers where container_id='DNC_CURRENT_BLOCKED_LISTS' and container_entry!=''"; + if ($DBX) {print "$scrub_stmt\n";} + $scrub_rslt=$dbhB->prepare($scrub_stmt); + $scrub_rslt->execute(); + if ($scrub_rslt->rows>0) + { + @scrub_row=$scrub_rslt->fetchrow_array; + $scrub_rslt->finish; + + + $dnc_entries=$scrub_row[0]; + + @dnc_entries=split(/\n/, $scrub_row[0]); + for ($i=0; $i/, $dnc_entries[$i]); + $scrub_list_id=$current_scrub[0]; + $scrub_country=$current_scrub[1]; + $scrub_list_id=~s/^\s+|\s+$//; + $scrub_country=~s/^\s+|\s+$//; + if (length($scrub_list_id)>0 && length($scrub_country)>0) + { + $check_table_stmt="show tables like 'vicidial_dnc_".$scrub_country."'"; + $check_table_rslt=$dbhB->prepare($check_table_stmt); + $check_table_rslt->execute(); + if ($check_table_rslt->rows>0) + { + $list_stmt="select phone_number from vicidial_list where list_id='$scrub_list_id'"; + $list_rslt=$dbhC->prepare($list_stmt); + $list_rslt->execute(); + while (@list_row=$list_rslt->fetchrow_array) + { + $dnc_check_stmt="select count(*) from vicidial_dnc_".$scrub_country." where phone_number='$list_row[0]'"; + $dnc_check_rslt=$dbhD->prepare($dnc_check_stmt); + $dnc_check_rslt->execute(); + @count_row=$dnc_check_rslt->fetchrow_array; + if ($count_row[0]>0) + { + $upd_dnc_stmt="update vicidial_list set status='$dnc_scrub_status' where list_id='$scrub_list_id' and phone_number='$list_row[0]'"; + $upd_dnc_rslt=$dbhE->prepare($upd_dnc_stmt); + $upd_dnc_rslt->execute(); + $upd_dnc_rslt->finish; + } + $dnc_check_rslt->finish; + } + $list_rslt->finish; + } + $check_table_rslt->finish; + } + } + } + else + { + $scrub_rslt->finish; + } + $upd_stmt="update vicidial_settings_containers set container_entry='' where container_id='DNC_CURRENT_BLOCKED_LISTS'"; + $upd_rslt=$dbhB->prepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + } + + $skip_ftp=0; + if (!$ftp_host) + { + if ($DB) {print "No FTP host given.\n";} + $skip_ftp++; + # exit; + } + if (!$ftp_user) + { + if ($DB) {print "No FTP user given.\n";} + $skip_ftp++; + # exit; + } + if (!$ftp_pwd) + { + if ($DB) {print "No FTP password given.\n";} + $skip_ftp++; + # exit; + } + if (!$ftp_port) + { + $ftp_port=21; + } + if (!$ftp_passive) + { + $ftp_passive=0; + } + if (!$file_dir) + { + print "Script failed - no directory to look up files.\n"; + exit; + } + if (!-e $file_dir || (-e $file_dir && !-d $file_dir)) + { + print "Script failed - file directory does not exist or is not a directory.\n"; + exit; + } + if (!$file_destination) + { + print "Script failed - no destination to move completed files to.\n"; + exit; + } + if (!-e $file_destination || (-e $file_destination && !-d $file_destination)) + { + print "Script failed - file destination directory does not exist or is not a directory.\n"; + exit; + } + if ($file_dir eq $file_destination) + { + print "Script failed - please choose a destination for the completed files that is different from where they're already located.\n"; + exit; + } + + $delete_stmt="select filename from vicidial_country_dnc_queue where file_status='CANCELLED'"; + if ($DBX) {print "$delete_stmt\n";} + $delete_rslt=$dbhA->prepare($delete_stmt); + $delete_rslt->execute(); + while (@delete_row=$delete_rslt->fetchrow_array) + { + if (-e "$file_dir/$delete_row[0]") + { + `rm $file_dir/$delete_row[0]`; + } + } + $delete_rslt->finish(); + + # Purge queue - current default 24 hours + $delete_stmt="delete from vicidial_country_dnc_queue where file_status='CANCELLED' or (file_status='FINISHED' and date_processed<=now()-INTERVAL 1 WEEK)"; + if ($DBX) {print "$delete_stmt\n";} + $delete_rslt=$dbhA->prepare($delete_stmt); + $delete_rslt->execute(); + $delete_rslt->finish(); + + # Get list of files already ready to be queued + $stmt="select filename from vicidial_country_dnc_queue where file_status in ('READY', 'PENDING', 'PROCESSING', 'INVALID LAYOUT')"; + $rslt=$dbhA->prepare($stmt); + $rslt->execute(); + @files_array=(); + while (@filename_row=$rslt->fetchrow_array) + { + push(@files_array, "$filename_row[0]"); + } + %active_files = map { $_ => 1 } @files_array; + $rslt->finish(); + + + if (!$skip_ftp) + { + # Get directory, insert and ignore + use Net::Ping; + use Net::FTP; + + $p = Net::Ping->new(); + $ping_good = $p->ping("$ftp_host"); + if (!$ping_good) + { + $p = Net::Ping->new("icmp"); + $ping_good = $p->ping("$ftp_host"); + } + + if ($ping_good) + { + $ftp = Net::FTP->new("$ftp_host", Port => $ftp_port, Passive => $ftp_passive, Debug => $FTPdb); + $ftp->login("$ftp_user","$ftp_pwd"); + if ($file_dir) {$ftp->cwd("$file_dir");} + + @FILES = $ftp->dir(".") or die "Can't get a list of files in $file_dir: $!"; + } + else + { + if($DB){print "ERROR: Could not ping server $ftp_host\n";} + exit; + } + $ftp->quit; + } + else + { + opendir(FILE, "$file_dir/"); + @FILES = readdir(FILE); + } + + $i=0; + $filename_str=""; + + foreach(@FILES) + { + $FILEsize1[$i] = 0; + if($FILES[$i]=~/(csv|xlsx?|txt)$/i && (length($FILES[$i]) > 4) ) + { + $FILEsize1[$i] = (-s "$file_dir/$FILES[$i]"); + if ($DBX) {print "$file_dir/$FILES[$i] $FILEsize1[$i]\n";} + } + $i++; + } + sleep(5); + + $i=0; + foreach(@FILES) + { + $FILEsize2[$i] = 0; + if($FILES[$i]=~/(csv|xlsx?|txt)$/i && (length($FILES[$i]) > 4) ) + { + $FILEsize2[$i] = (-s "$file_dir/$FILES[$i]"); + if ($DBX) {print "$file_dir/$FILES[$i] $FILEsize2[$i]\n";} + if ($FILEsize1[$i] ne $FILEsize2[$i]) + { + # do nothing + } + elsif(!exists($active_files{$FILES[$i]})) + { + $ins_stmt="insert into vicidial_country_dnc_queue(filename, date_uploaded, file_status) VALUES('$FILES[$i]', now(), 'READY')"; + if ($DBX) {print "$ins_stmt\n";} + $ins_rslt=$dbhA->prepare($ins_stmt); + $ins_rslt->execute(); + $ins_rslt->finish; + $filename_str.="'$FILES[$i]',"; + } + } + $i++; + } + + if (length($filename_str)>0) + { + # Clear records from queue table where the file is no longer found in the expected place + $filename_str=~s/,$//; + $delete_stmt="delete from vicidial_country_dnc_queue where file_status!='FINISHED' and filename not in ($filename_str)"; + if ($DBX) {print "$delete_stmt\n";} + $delete_rslt=$dbhA->prepare($delete_stmt); + $delete_rslt->execute(); + $delete_rslt->finish(); + } + + $dbhA->disconnect(); + $dbhB->disconnect(); + $dbhC->disconnect(); + $dbhD->disconnect(); + $dbhE->disconnect(); + } +else + { + $dbhA->disconnect(); + $dbhB->disconnect(); + $dbhC->disconnect(); + $dbhD->disconnect(); + $dbhE->disconnect(); + exit; + } +#### end 1st script diff --git a/bin/VICIDIAL_international_DNC_parser_2.pl b/bin/VICIDIAL_international_DNC_parser_2.pl new file mode 100644 index 00000000..1877746e --- /dev/null +++ b/bin/VICIDIAL_international_DNC_parser_2.pl @@ -0,0 +1,571 @@ +#!/usr/bin/perl +# +# VICIDIAL_international_DNC_parser_2.pl +# +# This is the second part of a two-step program in the DNC importing process +# +# It requires that a system settings container exists in your system settings +# named "INTERNATIONAL_DNC_IMPORT". Two run-time variables must be defined +# in said container - "--file-dir" and "--file-destination" +# +# It also requires a second settings container named "DNC_IMPORT_FORMATS", +# which it will use to determine how to parse the data in the file. +# +# This should run every minute to check the DNC queue table for files +# marked as PENDING, which it will then try to process based on the import +# format defined through the web interface. +# +# As the file is being processed, the web interface will reflect the progress +# of this script. Only one file will be processed at any time. +# +# Put an entry into the cron of of your asterisk machine to run this script +# every minute or however often you desire +# +# ### International DNC importing scripts +##* * * * * /usr/share/astguiclient/VICIDIAL_international_DNC_parser_2.pl +# +# Copyright (C) 2020 Matt Florell , Joe Johnson LICENSE: AGPLv2 +# +# CHANGE LOG: +# 200813-1230 - First Build +# + +use DBI; +use Time::Local; + +# default path to astguiclient configuration file: +$PATHconf = '/etc/astguiclient.conf'; + +open(conf, "$PATHconf") || die "can't open $PATHconf: $!\n"; +@conf = ; +close(conf); +$i=0; +foreach(@conf) + { + $line = $conf[$i]; + $line =~ s/ |>|\n|\r|\t|\#.*|;.*//gi; + if ( ($line =~ /^PATHhome/) && ($CLIhome < 1) ) + {$PATHhome = $line; $PATHhome =~ s/.*=//gi;} + if ( ($line =~ /^PATHlogs/) && ($CLIlogs < 1) ) + {$PATHlogs = $line; $PATHlogs =~ s/.*=//gi;} + if ( ($line =~ /^PATHagi/) && ($CLIagi < 1) ) + {$PATHagi = $line; $PATHagi =~ s/.*=//gi;} + if ( ($line =~ /^PATHweb/) && ($CLIweb < 1) ) + {$PATHweb = $line; $PATHweb =~ s/.*=//gi;} + if ( ($line =~ /^PATHsounds/) && ($CLIsounds < 1) ) + {$PATHsounds = $line; $PATHsounds =~ s/.*=//gi;} + if ( ($line =~ /^PATHmonitor/) && ($CLImonitor < 1) ) + {$PATHmonitor = $line; $PATHmonitor =~ s/.*=//gi;} + if ( ($line =~ /^VARserver_ip/) && ($CLIserver_ip < 1) ) + {$VARserver_ip = $line; $VARserver_ip =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_server/) && ($CLIDB_server < 1) ) + {$VARDB_server = $line; $VARDB_server =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_database/) && ($CLIDB_database < 1) ) + {$VARDB_database = $line; $VARDB_database =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_user/) && ($CLIDB_user < 1) ) + {$VARDB_user = $line; $VARDB_user =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_pass/) && ($CLIDB_pass < 1) ) + {$VARDB_pass = $line; $VARDB_pass =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_custom_user/) && ($CLIDB_custom_user < 1) ) + {$VARDB_custom_user = $line; $VARDB_custom_user =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_custom_pass/) && ($CLIDB_custom_pass < 1) ) + {$VARDB_custom_pass = $line; $VARDB_custom_pass =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_port/) && ($CLIDB_port < 1) ) + {$VARDB_port = $line; $VARDB_port =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_host/) && ($CLIREPORT_host < 1) ) + {$VARREPORT_host = $line; $VARREPORT_host =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_user/) && ($CLIREPORT_user < 1) ) + {$VARREPORT_user = $line; $VARREPORT_user =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_pass/) && ($CLIREPORT_pass < 1) ) + {$VARREPORT_pass = $line; $VARREPORT_pass =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_port/) && ($CLIREPORT_port < 1) ) + {$VARREPORT_port = $line; $VARREPORT_port =~ s/.*=//gi;} + if ( ($line =~ /^VARREPORT_dir/) && ($CLIREPORT_dir < 1) ) + {$VARREPORT_dir = $line; $VARREPORT_dir =~ s/.*=//gi;} + $i++; + } + +$dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; +$dbhB = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass") + or die "Couldn't connect to database: " . DBI->errstr; +$dbhC = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_custom_user", "$VARDB_custom_pass") + or die "Couldn't connect to database: " . DBI->errstr; + +if ($run_check > 0) + { + my $grepout = `/bin/ps ax | grep $0 | grep -v grep | grep -v '/bin/sh'`; + my $grepnum=0; + $grepnum++ while ($grepout =~ m/\n/g); + if ($grepnum > 1) + { + if ($DB) {print "I am not alone! Another $0 is running! Exiting...\n";} + exit; + } + } + + +### begin parsing run-time options ### +if (length($ARGV[0])>1) + { + $i=0; + while ($#ARGV >= $i) + { + $args = "$args $ARGV[$i]"; + $i++; + } + + if ($args =~ /--help/i) + { + print "allowed run time options:\n"; + print " [--help] = this screen\n"; + print " [--debug] = debug\n"; + print " [--debugX] = super debug\n"; + print " [--FTPdb] = show FTP debug output\n"; + print " [--test] = will execute script and transfer files, but not delete lists from dialer\n"; + print "\n"; + exit; + } + else + { + if ($args =~ /--debug/i) + { + $DB=1; + print "\n----- DEBUG -----\n\n"; + } + if ($args =~ /--debugX/i) + { + $DB=1; + $DBX=1; + print "\n----- SUPER DEBUG -----\n\n"; + } + if ($args =~ /--FTPdb/i) + { + $FTPdb=1; + print "\n----- FTP DEBUG -----\n\n"; + } + if ($args =~ /--test/i) + { + $T=1; + print "\n----- TEST EXECUTION -----\n\n"; + } + } + } + +$stmt="SELECT enable_international_dncs FROM system_settings;"; +$rslt=$dbhA->prepare($stmt); +if ($DB) {print "$stmt\n";} +$rslt->execute(); +if ($rslt->rows>0) + { + @row=$rslt->fetchrow_array; + if ($row[0]==0 || !$row[0]) + { + if ($DB) {print "Feature not enabled.\n"; exit;} + } + } +else + { + if ($DB) {print "Database error!\n"; exit;} + } + +$secX = time(); + ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); +$year = ($year + 1900); +$mon++; +$rowdate = "$mon/$mday/$year"; +if ($mon < 10) {$mon = "0$mon";} +if ($mday < 10) {$mday = "0$mday";} +if ($min < 10) {$min = "0$min";} +if ($hour < 10) {$hour = "0$hour";} +if ($sec < 10) {$sec = "0$sec";} +$run_time="$year-$mon-$mday $hour:$min:$sec"; + +$stmt="select * from vicidial_settings_containers where container_id='INTERNATIONAL_DNC_IMPORT' and container_type='PERL_CLI'"; +if ($DBX) {print "\n$stmt\n\n";} +$rslt=$dbhA->prepare($stmt); +$rslt->execute(); +if ($rslt->rows>0) + { + @row=$rslt->fetchrow_array; + $row[4]=~s/\r//g; + @container_settings=split(/\n/, $row[4]); + + for ($i=0; $ifinish(); + + $skip_ftp=0; + if (!$ftp_host) + { + if ($DB) {print "No FTP host given.\n";} + $skip_ftp++; + # exit; + } + if (!$ftp_user) + { + if ($DB) {print "No FTP user given.\n";} + $skip_ftp++; + # exit; + } + if (!$ftp_pwd) + { + if ($DB) {print "No FTP password given.\n";} + $skip_ftp++; + # exit; + } + if (!$ftp_port) + { + $ftp_port=21; + } + if (!$ftp_passive) + { + $ftp_passive=0; + } + if (!$file_dir) + { + print "Script failed - no directory to look up files.\n"; + exit; + } + if (!-e $file_dir || (-e $file_dir && !-d $file_dir)) + { + print "Script failed - file directory does not exist or is not a directory.\n"; + exit; + } + if (!$file_destination) + { + print "Script failed - no destination to move completed files to.\n"; + exit; + } + if (!-e $file_destination || (-e $file_destination && !-d $file_destination)) + { + print "Script failed - file destination directory does not exist or is not a directory.\n"; + exit; + } + if ($file_dir eq $file_destination) + { + print "Script failed - please choose a destination for the completed files that is different from where they're already located.\n"; + exit; + } + # Get file layouts + $layout_stmt="select container_entry from vicidial_settings_containers where container_id='DNC_IMPORT_FORMATS'"; + if ($DBX) {print "$layout_stmt\n\n";} + $layout_rslt=$dbhB->prepare($layout_stmt); + $layout_rslt->execute(); + if ($layout_rslt->rows==0) {die "No import formats container found in settings containers";} + @layout_row=$layout_rslt->fetchrow_array; + @layout_array=split(/\n/, $layout_row[0]); + %layouts=(); + for ($i=0; $i/) + { + @current_layout=split(/\=\>/, $entry); + $current_layout[0]=~s/^\s+|\s+$//; + $current_layout[1]=~s/^\s+|\s+$//; + if (length($current_layout[0])) + { + $layouts{"$current_layout[0]"}=$current_layout[1]; + } + } + } + $layout_rslt->finish; + + # select from database where file is marked as ready to queue and country code is set + $process_stmt="select filename, file_layout, country_code, file_action, dnc_file_id from vicidial_country_dnc_queue where file_status='PENDING' and country_code is not null and file_layout is not null order by date_uploaded asc limit 1"; + if ($DBX) {print "$process_stmt\n\n";} + $process_rslt=$dbhA->prepare($process_stmt); + $process_rslt->execute(); + if ($process_rslt->rows==0) {exit;} + + while(@process_row=$process_rslt->fetchrow_array) + { + $filename=$process_row[0]; + $file_layout=$process_row[1]; + $country_code=$process_row[2]; + $file_action=$process_row[3]; + $dnc_file_id=$process_row[4]; + $process_count=0; + $records_processed=0; + $records_inserted=0; + + if (!exists $layouts{"$file_layout"}) + { + $upd_stmt="update vicidial_country_dnc_queue set file_status='INVALID LAYOUT' where dnc_file_id='$dnc_file_id'"; + if ($DB) {print "Can't find file layout ($file_layout) - $upd_stmt\n\n";} + $upd_rslt=$dbhB->prepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + exit; + } + else + { + $create_stmt="CREATE TABLE IF NOT EXISTS vicidial_dnc_".$country_code."(phone_number varchar(20) primary key)"; + if ($DB) {print "$create_stmt\n\n";} + $create_rslt=$dbhC->prepare($create_stmt); + $create_rslt->execute(); + $create_rslt->finish; + + if ($file_action eq "PURGE") + { + $delete_stmt="delete from vicidial_dnc_".$country_code; + if ($DBX) {print "$delete_stmt\n\n";} + $delete_rslt=$dbhB->prepare($delete_stmt); + $delete_rslt->execute(); + $delete_rslt->finish; + + $opt_stmt="optimize table vicidial_dnc_".$country_code; + if ($DBX) {print "$opt_stmt\n\n";} + $opt_rslt=$dbhB->prepare($opt_stmt); + $opt_rslt->execute(); + $opt_rslt->finish; + } + + + @layout_stats=split(/,/, $layouts{"$file_layout"}); + if ($DBX) {print $layouts{"$file_layout"}."\n";} + $txt_format=$layout_stats[0]; + $delimiter=$layout_stats[1]; + if ($filename=~/csv$/i) + { + $delimiter="comma"; + } + + $delimiter=~s/.*comma.*/,/gi; + $delimiter=~s/.*pipe.*/\\\|/gi; + $delimiter=~s/.*tab.*/\t/gi; + + if ($txt_format ne "" && $DB) {print "Format: $txt_format\n\n";} + if ($delimiter ne "" && $DB) {print "Delimiter: $delimiter\n\n";} + + @phone_fields=(); + for($i=2; $iprepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + exit; + } + } + + if (!$skip_ftp) # File is NOT local, so attempt to download. + { + use Net::Ping; + use Net::FTP; + + $ftp = Net::FTP->new("$ftp_host", Port => $ftp_port, Passive => $ftp_passive, Debug => $FTPdb); + $ftp->login("$ftp_user","$ftp_pwd"); + if ($file_dir) {$ftp->cwd("$file_dir");} + $ftp->get("$filename", "/tmp/$filename"); + $ftp->quit; + $full_filename="/tmp/$filename"; + } + else + { + $full_filename="$file_dir/$filename"; + } + + if ($filename=~/xlsx?$/) + { + use Spreadsheet::Read; + use Spreadsheet::XLSX; + use File::Basename; + use open ':std', ':encoding(UTF-8)'; + + # Gotta account for the fact that spreadsheets don't start with 0 for indices + for($k=0; $knew("$full_filename"); + $sheets=$datafile->[0]{sheets}; + for ($i=1; $i<=$sheets; $i++) + { + $current_sheet=$datafile->[$i]; + $sheet_rows=$datafile->[$i]{maxrow}; + $sheet_cols=$datafile->[$i]{maxcol}; + + $upd_stmt="update vicidial_country_dnc_queue set file_status='PROCESSING', total_records='$sheet_rows' where dnc_file_id='$dnc_file_id'"; + if ($DB) {print "$upd_stmt\n\n";} + $upd_rslt=$dbhB->prepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + + for ($j=1; $j<=$sheet_rows; $j++) + { + $records_processed++; + $phone_number=""; + for($k=0; $k[$i]{cell}[$phone_fields[$k]][$j]; + } + InsertPhoneNumber($records_processed, $records_inserted, $phone_number, $dnc_file_id); + } + } + } + else + { + $total_records_str=`wc -l $full_filename`; + @tr_str=split(/\s/, $total_records_str); + $total_records=$tr_str[0]; + + $upd_stmt="update vicidial_country_dnc_queue set file_status='PROCESSING', total_records='$total_records' where dnc_file_id='$dnc_file_id'"; + if ($DB) {print "$upd_stmt\n\n";} + $upd_rslt=$dbhB->prepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + + open(PHONE_FILE, "$full_filename"); + while($buffer=) + { + $records_processed++; + $phone_number=""; + if ($delimiter ne "") + { + @data=split(/$delimiter/, $buffer); + for($i=0; $iprepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + + if (!$skip_ftp) # File is NOT local, so attempt to download. + { + use Net::Ping; + use Net::FTP; + + $ftp = Net::FTP->new("$ftp_host", Port => $ftp_port, Passive => $ftp_passive, Debug => $FTPdb); + $ftp->login("$ftp_user","$ftp_pwd"); + if ($file_dir) {$ftp->cwd("$file_dir");} +# $ftp->get("$filename", "/tmp/$filename"); +# $ftp->quit; +# $full_filename="/tmp/$filename"; + } + else + { + `mv $file_dir/$filename $file_destination/$filename`; + } + + + } + $dbhA->disconnect(); + $dbhB->disconnect(); + $dbhC->disconnect(); + } +else + { + $dbhA->disconnect(); + $dbhB->disconnect(); + $dbhC->disconnect(); + exit; + } + + +sub InsertPhoneNumber + { + my $rp=$_[0]; + my $ri=$_[1]; + my $phone=$_[2]; + my $dfid=$_[3]; + + $phone=~s/[^0-9]//gi; + + if (length($phone)>=5 && length($phone)<=18) + { + $ins_stmt="insert ignore into vicidial_dnc_".$country_code." values('$phone')"; + $ins_rslt=$dbhB->prepare($ins_stmt); + $ins_rslt->execute(); + if ($ins_rslt->rows>0) {$records_inserted++; $ri++;} + $ins_rslt->finish; + } + + if ($rp%1000==0) + { + $upd_stmt="update vicidial_country_dnc_queue set records_processed='$rp', records_inserted='$ri' where dnc_file_id='$dfid'"; + if ($DBX) {print "$ins_stmt\n$upd_stmt\n";} + $upd_rslt=$dbhB->prepare($upd_stmt); + $upd_rslt->execute(); + $upd_rslt->finish; + } + + } diff --git a/docs/INTERNATIONAL_DNC_SCRUB.txt b/docs/INTERNATIONAL_DNC_SCRUB.txt new file mode 100644 index 00000000..085a0bb2 --- /dev/null +++ b/docs/INTERNATIONAL_DNC_SCRUB.txt @@ -0,0 +1,53 @@ +INTERNATIONAL DNC SCRUB DOC Started: 2020-07-20 Updated: 2020-08-14 + +NOTE: Requires svn/trunk code revision 3273 or higher to work properly!!! + +NOTE: THIS FEATURE IS OPTIONAL, AND SHOULD ONLY BE CONFIGURED AFTER YOUR SYSTEM + HAS BEEN INSTALLED AND TESTED! + + +Instructions for using the international DNC scrub. + +### SETTING UP DNC LISTS TO BE PROCESSED ### +- The international DNC scrub can be enabled via a system setting "Enable International DNCs". Set the value to "1" in order to get access to the DNC web interface as well as cause the option to scrub against international DNCs to appear in the lead loader. + +- To start loading a file, you must have a settings container in your system by the name of "INTERNATIONAL_DNC_IMPORT". The container outlines the necessary values it must have, which consists of a "--file-dir" variable and a "--file-destination" variable. + +- There is a Perl script that must be put into your cron, "Vicidial_international_DNC_parser_1.pl". This script will check that the system setting is enabled and that the "INTERNATIONAL_DNC_IMPORT" exists and contains the two necessary variables. It will confirm that the file directories given by the variables both exist, are directories, and are not the same. It will also do some other checks depending on what variables you have active in that container. If these conditions are met, the script will proceed to check for any .txt, .csv, .xls, or .xlsx files in the "--file-dir" directory. For any files found in this directory, the script will check the queue table to see if the file is currently in the database with a status other than "FINISHED" or "CANCELLED". Finding no records will add the file to the queue table with a status of "READY". + +- Additionally, this script will also take the list of files it finds in the folder and check it against any unfinished file currently in the queue. Unfinished files in the queue table not on the list will be automatically purged. Also, any filenames marked as "CANCELLED" will be purged from the "--file-dir" folder if they are still there. + +- Once files are loaded into the queue table, they will be viewable through the DNC web interface. + + +### USING THE WEB INTERFACE TO PROCESS DNC LISTS ### +- The link to the DNC web interface can be found in the "Admin Utilities" page under the "International DNC Loader" link as long as the "Enable International DNCs" setting is active. + +- Users must have the privilege to load leads in order to view the DNC web interface. + +- The DNC Web Interface page will show all files currently in the queue table, including ones that have been processed ("FINISHED") for up to 1 week. + +- If a file's current status is "READY" or "INVALID_LAYOUT" (more on "INVALID_LAYOUT" below), the user can assign to the file a country code, a file action of either "PURGE" or "APPEND", and a file layout that the file follows. Selecting a file layout will cause a brief description of the layout to appear below the corresponding drop-down menu. Once all three parameters have been defined and if the file's current status makes it eligible for processing, the user will have the option to "START FILE" or "CANCEL" it. "START FILE" will set the file status to "PENDING" so it can be picked up by an automated script that will handle it. "CANCEL" will set the file status to "CANCELLED" and the file will shortly be purged from the queue table. + +- If a file is "FINISHED" or "PROCESSING", it can no longer be modified or set to "PENDING" or "CANCELLED". If a file is currently "PENDING" or "CANCELLED", you have a narrow window of time until the processing script automatically runs to change it to the other value. + +- Once a file is marked as "PENDING", there is a second automated script that should be placed in your cron, "Vicidial_international_DNC_parser_2.pl". This script will constantly check for records in the queue table where a file is marked as "PENDING", and will handle it per the parameters defined by the user in the web interface when the file was submitted. It will check that the file still exists in the folder given by the "--file-dir" variable and do a count on the number of records in that file. Using the "country_code" value, the script will create a table named "vicidial_dnc_" if no such table currently exists. If the "file_action" value is set to "PURGE", the script will empty out and optimize the table prior to attempting to insert the records from the pending file. + +- The script will then open the file and begin reading from it. Using the "file_layout" value, as well as the type of file gleaned from the filename, the script will parse out the phone number from . If the file_layout is not found in the "DNC_IMPORT_FORMATS" setting or if the layout doesn't give adequate information to parse the file (i.e. no phone field is given), then the file status is changed to "INVALID LAYOUT" and the script terminates. Otherwise the status of the file is changed to "PROCESSING". As the script reads through the file, every 1,000 records it will update the queue table to reflect how many records were processed and how many were inserted. This is so the user can track the progress of the file through the web interface. + +- When the file is finished, it is moved to the location given by the "--file-destination" variable from the "INTERNATIONAL_DNC_IMPORT" settings container and is marked as "FINISHED" in the queue table. The DNC list can now be used to dedupe against from the lead loader. + + +### USING THE DNC LISTS IN THE LEAD LOADER ### +- To dedupe against a country's DNC list when new leads are loaded, simply select a country from the "DNC Scrub by Country" drop-down menu from the main lead loader page. Only countries that have had DNC lists loaded will appear here. When the new lead file is being processed, phone numbers matching a phone number in the selected country's DNC list will not be loaded and will be individually listed, similar to other invalid leads, with a "DNC" code instead of "INV". + +- If a lead file is loaded into a specific list using a country's DNC list as a suppression file, that list will become "locked" ***AND WILL BE COMPLETELY UNDIALABLE UNTIL THE REST OF THE LIST IS CHECKED AGAINST THE SELECTED COUNTRY'S DNC LIST***. This will also cause any leads from that list to be deleted from the hopper immediately. + +- Lists that are currently locked can be viewed in the settings container named "DNC_CURRENT_BLOCKED_LISTS". This is a read-only settings container and cannot be modified once a list is placed into it. Entries are stored here in the format " => ". The hopper script will use this settings container to block any list ID currently in it (provided that the "Enable International DNC" system setting is still active). + +- The list(s) will remain locked for the remainder of the day. DNC scrubs for existing lists are handled by the "Vicidial_international_DNC_parser_1.pl" cron job. This script has a two-hour window between midnight and 2am wherein it will take any list ID/country pairings it finds in the "DNC_CURRENT_BLOCKED_LISTS" settings container and scrub the dialer list against the country DNC list. Any phone number found in the country's DNC list will be marked as "DNCI", regardless of it's current status in the dialer. If an alternate DNC code is desired instead of "DNCI", that can be set via the "--dnc-status-override" variable in the "INTERNATIONAL_DNC_IMPORT". The settings container has instructions on how to use this variable. + +- If an admin wishes to force the "Vicidial_international_DNC_parser_1.pl" script to perform the DNC scrub regardless of the time of day, it can be run with the command line option "--force-dnc-scrub". + +- Once the DNC scrub is completed for all lists, the "DNC_CURRENT_BLOCKED_LISTS" container will be cleared and the hopper script will no longer block the lists from having leads loaded into the hopper. + diff --git a/extras/MySQL_AST_CREATE_tables.sql b/extras/MySQL_AST_CREATE_tables.sql index efa54533..00d4adb7 100644 --- a/extras/MySQL_AST_CREATE_tables.sql +++ b/extras/MySQL_AST_CREATE_tables.sql @@ -1864,7 +1864,8 @@ outbound_cid_any ENUM('DISABLED','API_ONLY') DEFAULT 'DISABLED', entries_per_page SMALLINT(5) UNSIGNED DEFAULT '0', browser_call_alerts ENUM('0','1','2') DEFAULT '0', queuemetrics_pausereason ENUM('STANDARD','EVERY_NEW','EVERY_NEW_ADMINCALL','EVERY_NEW_ALLCALL') default 'STANDARD', -inbound_answer_config ENUM('0','1','2','3','4','5') DEFAULT '0' +inbound_answer_config ENUM('0','1','2','3','4','5') DEFAULT '0', +enable_international_dncs ENUM('0','1') default '0' ) ENGINE=MyISAM; CREATE TABLE vicidial_campaigns_list_mix ( @@ -3365,7 +3366,7 @@ unique index vdweb (webserver, hostname) CREATE TABLE vicidial_urls ( url_id INT(9) UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL, -url VARCHAR(255) default '', +url VARCHAR(250) default '', unique index (url) ) ENGINE=MyISAM; @@ -3398,7 +3399,7 @@ index (avatar_id) ) ENGINE=MyISAM; CREATE TABLE audio_store_details ( -audio_filename VARCHAR(255) NOT NULL, +audio_filename VARCHAR(250) NOT NULL, audio_format VARCHAR(10) default 'unknown', audio_filesize BIGINT(20) UNSIGNED default '0', audio_epoch BIGINT(20) UNSIGNED default '0', @@ -3752,7 +3753,8 @@ alt_row1_background VARCHAR(6) default 'BDFFBD', alt_row2_background VARCHAR(6) default '99FF99', alt_row3_background VARCHAR(6) default 'CCFFCC', user_group VARCHAR(20) default '---ALL---', -web_logo VARCHAR(100) default 'default_new' +web_logo VARCHAR(100) default 'default_new', +button_color VARCHAR(6) default 'EFEFEF' ) ENGINE=MyISAM; CREATE TABLE vicidial_user_list_new_lead ( @@ -4318,6 +4320,22 @@ index (lead_id), index (call_date) ) ENGINE=MyISAM; +CREATE TABLE vicidial_country_dnc_queue ( +dnc_file_id int(10) UNSIGNED NOT NULL AUTO_INCREMENT, +filename varchar(250) DEFAULT NULL, +country_code varchar(3) DEFAULT NULL, +file_layout varchar(30) DEFAULT NULL, +file_status enum('UPLOADING','READY','PENDING','INVALID LAYOUT','PROCESSING','FINISHED','CANCELLED') DEFAULT NULL, +file_action enum('PURGE','APPEND') DEFAULT NULL, +date_uploaded DATETIME DEFAULT NULL, +total_records int(10) UNSIGNED DEFAULT NULL, +records_processed int(10) UNSIGNED DEFAULT NULL, +records_inserted int(10) UNSIGNED DEFAULT NULL, +date_processed DATETIME DEFAULT NULL, +PRIMARY KEY (dnc_file_id), +KEY vicidial_country_dnc_queue_filename_key (filename) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + ALTER TABLE vicidial_email_list MODIFY message text character set utf8; @@ -4348,7 +4366,7 @@ INSERT INTO vicidial_music_on_hold SET moh_id='default',moh_name='Default Music INSERT INTO vicidial_music_on_hold_files SET moh_id='default',filename='conf',rank='1'; INSERT INTO vicidial_inbound_groups(group_id,group_name,group_color,active,queue_priority) values('AGENTDIRECT','Single Agent Direct Queue','white','Y','99'); -INSERT INTO vicidial_inbound_groups(group_id,group_name,group_color,active,web_form_address,voicemail_ext,next_agent_call,fronter_display,ingroup_script,get_call_launch,xferconf_a_dtmf,xferconf_a_number,xferconf_b_dtmf,xferconf_b_number,drop_call_seconds,drop_action,drop_exten,call_time_id,after_hours_action,after_hours_message_filename,after_hours_exten,after_hours_voicemail,welcome_message_filename,moh_context,onhold_prompt_filename,prompt_interval,agent_alert_exten,agent_alert_delay,default_xfer_group,queue_priority,drop_inbound_group,ingroup_recording_override,ingroup_rec_filename,afterhours_xfer_group,qc_enabled,qc_statuses,qc_shift_id,qc_get_record_launch,qc_show_recording,qc_web_form_address,qc_script,play_place_in_line,play_estimate_hold_time,hold_time_option,hold_time_option_seconds,hold_time_option_exten,hold_time_option_voicemail,hold_time_option_xfer_group,hold_time_option_callback_filename,hold_time_option_callback_list_id,hold_recall_xfer_group,no_delay_call_route,play_welcome_message,answer_sec_pct_rt_stat_one,answer_sec_pct_rt_stat_two,default_group_alias,no_agent_no_queue,no_agent_action,no_agent_action_value,web_form_address_two,timer_action,timer_action_message,timer_action_seconds,start_call_url,dispo_call_url,xferconf_c_number,xferconf_d_number,xferconf_e_number,ignore_list_script_override,extension_appended_cidname,uniqueid_status_display,uniqueid_status_prefix,hold_time_option_minimum,hold_time_option_press_filename,hold_time_option_callmenu,hold_time_option_no_block,hold_time_option_prompt_seconds,onhold_prompt_no_block,onhold_prompt_seconds,hold_time_second_option,hold_time_third_option,wait_hold_option_priority,wait_time_option,wait_time_second_option,wait_time_third_option,wait_time_option_seconds,wait_time_option_exten,wait_time_option_voicemail,wait_time_option_xfer_group,wait_time_option_callmenu,wait_time_option_callback_filename,wait_time_option_callback_list_id,wait_time_option_press_filename,wait_time_option_no_block,wait_time_option_prompt_seconds,timer_action_destination,calculate_estimated_hold_seconds,add_lead_url,eht_minimum_prompt_filename,eht_minimum_prompt_no_block,eht_minimum_prompt_seconds,on_hook_ring_time,na_call_url,on_hook_cid,group_calldate,action_xfer_cid,drop_callmenu,after_hours_callmenu,user_group,max_calls_method,max_calls_count,max_calls_action,dial_ingroup_cid,group_handling,web_form_address_three,populate_lead_ingroup,drop_lead_reset,after_hours_lead_reset,nanq_lead_reset,wait_time_lead_reset,hold_time_lead_reset,status_group_id,routing_initiated_recordings,on_hook_cid_number) VALUES ('AGENTDIRECT_CHAT','Agent Direct Queue for Chats','#FFFFFF','Y','','','longest_wait_time','Y','','',NULL,NULL,NULL,NULL,360,'MESSAGE','8307','24hours','MESSAGE','vm-goodbye','8300',NULL,'---NONE---','default','generic_hold',60,'ding',1000,'---NONE---',99,'---NONE---','DISABLED','NONE','---NONE---','N',NULL,'24HRMIDNIGHT','NONE','Y',NULL,NULL,'N','N','NONE',360,'8300','','---NONE---','vm-hangup',0,'---NONE---','N','ALWAYS',20,30,'','N','MESSAGE','nbdy-avail-to-take-call|vm-goodbye','','NONE','',-1,'','','','','','N','N','DISABLED','',0,'to-be-called-back|digits/1','','N',10,'N',10,'NONE','NONE','WAIT','NONE','NONE','NONE',120,'8300','','---NONE---','','vm-hangup',999,'to-be-called-back|digits/1','N',10,'',0,'','','N',10,15,'','GENERIC',NULL,'CUSTOMER','','','---ALL---','DISABLED',0,'DROP','','CHAT','','ENABLED','N','N','N','N','N','','N',''); +INSERT INTO vicidial_inbound_groups(group_id,group_name,group_color,active,web_form_address,voicemail_ext,next_agent_call,fronter_display,ingroup_script,get_call_launch,xferconf_a_dtmf,xferconf_a_number,xferconf_b_dtmf,xferconf_b_number,drop_call_seconds,drop_action,drop_exten,call_time_id,after_hours_action,after_hours_message_filename,after_hours_exten,after_hours_voicemail,welcome_message_filename,moh_context,onhold_prompt_filename,prompt_interval,agent_alert_exten,agent_alert_delay,default_xfer_group,queue_priority,drop_inbound_group,ingroup_recording_override,ingroup_rec_filename,afterhours_xfer_group,qc_enabled,qc_statuses,qc_shift_id,qc_get_record_launch,qc_show_recording,qc_web_form_address,qc_script,play_place_in_line,play_estimate_hold_time,hold_time_option,hold_time_option_seconds,hold_time_option_exten,hold_time_option_voicemail,hold_time_option_xfer_group,hold_time_option_callback_filename,hold_time_option_callback_list_id,hold_recall_xfer_group,no_delay_call_route,play_welcome_message,answer_sec_pct_rt_stat_one,answer_sec_pct_rt_stat_two,default_group_alias,no_agent_no_queue,no_agent_action,no_agent_action_value,web_form_address_two,timer_action,timer_action_message,timer_action_seconds,start_call_url,dispo_call_url,xferconf_c_number,xferconf_d_number,xferconf_e_number,ignore_list_script_override,extension_appended_cidname,uniqueid_status_display,uniqueid_status_prefix,hold_time_option_minimum,hold_time_option_press_filename,hold_time_option_callmenu,hold_time_option_no_block,hold_time_option_prompt_seconds,onhold_prompt_no_block,onhold_prompt_seconds,hold_time_second_option,hold_time_third_option,wait_hold_option_priority,wait_time_option,wait_time_second_option,wait_time_third_option,wait_time_option_seconds,wait_time_option_exten,wait_time_option_voicemail,wait_time_option_xfer_group,wait_time_option_callmenu,wait_time_option_callback_filename,wait_time_option_callback_list_id,wait_time_option_press_filename,wait_time_option_no_block,wait_time_option_prompt_seconds,timer_action_destination,calculate_estimated_hold_seconds,add_lead_url,eht_minimum_prompt_filename,eht_minimum_prompt_no_block,eht_minimum_prompt_seconds,on_hook_ring_time,na_call_url,on_hook_cid,group_calldate,action_xfer_cid,drop_callmenu,after_hours_callmenu,user_group,max_calls_method,max_calls_count,max_calls_action,dial_ingroup_cid,group_handling,web_form_address_three,populate_lead_ingroup,drop_lead_reset,after_hours_lead_reset,nanq_lead_reset,wait_time_lead_reset,hold_time_lead_reset,status_group_id,routing_initiated_recordings,on_hook_cid_number) VALUES ('AGENTDIRECT_CHAT','Agent Direct Queue for Chats','#FFFFFF','Y','','','longest_wait_time','Y','','NONE',NULL,NULL,NULL,NULL,360,'MESSAGE','8307','24hours','MESSAGE','vm-goodbye','8300',NULL,'---NONE---','default','generic_hold',60,'ding',1000,'---NONE---',99,'---NONE---','DISABLED','NONE','---NONE---','N',NULL,'24HRMIDNIGHT','NONE','Y',NULL,NULL,'N','N','NONE',360,'8300','','---NONE---','vm-hangup',0,'---NONE---','N','ALWAYS',20,30,'','N','MESSAGE','nbdy-avail-to-take-call|vm-goodbye','','NONE','',-1,'','','','','','N','N','DISABLED','',0,'to-be-called-back|digits/1','','N',10,'N',10,'NONE','NONE','WAIT','NONE','NONE','NONE',120,'8300','','---NONE---','','vm-hangup',999,'to-be-called-back|digits/1','N',10,'',0,'','','N',10,15,'','GENERIC',NULL,'CUSTOMER','','','---ALL---','DISABLED',0,'DROP','','CHAT','','ENABLED','N','N','N','N','N','','N',''); INSERT INTO vicidial_lists SET list_id='999',list_name='Default inbound list',campaign_id='TESTCAMP',active='N'; INSERT INTO vicidial_lists SET list_id='998',list_name='Default Manual list',campaign_id='TESTCAMP',active='N'; @@ -4430,8 +4448,8 @@ INSERT INTO vicidial_scripts (script_id,script_name,script_comments,active,scrip INSERT INTO vicidial_custom_leadloader_templates (template_id, template_name, template_description, list_id, standard_variables, custom_table, custom_variables, template_statuses) values ('SAMPLE_TEMPLATE','Sample template','',999,'phone_number,9|first_name,0|last_name,1|address1,3|address2,4|address3,5|city,6|state,7|postal_code,8|','custom_999','appointment_date,2|appointment_notes,9|nearest_city,2|',''); -INSERT INTO vicidial_screen_colors VALUES ('red_rust','dark red rust','Y','804435','E7D0C2','C68C71','D9B39F','D9B49F','C68C72','C68C73','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'),('pale_green','pale green','Y','738035','E0E7C2','B6C572','C4CF8B','B6C572','C4CF8B','C4CF8B','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'),('alt_green','alternate green','Y','333333','D6E3B2','AEC866','BCD180','BCD180','AEC866','AEC866','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'),('default_blue_test','default blue test','Y','015B91','D9E6FE','9BB9FB','B9CBFD','8EBCFD','B6D3FC','A3C3D6','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'),('basic_orange','basic orange','Y','804d00','ffebcc','ffcc80','ffd699','ffcc80','ffd699','ffcc80','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'),('basic_purple','basic purple','Y','660066','ffccff','ff99ff','ffb3ff','ff99ff','ffb3ff','ff99ff','BDFFBD','99FF99','CCFFCC','---ALL---','SAMPLE.png'),('basic_yellow','basic yellow','Y','666600','ffffcc','ffff66','ffff99','ffff66','ffff99','ffff66','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'),('basic_red','basic red','Y','800000','ffe6e6','ff9999','ffb3b3','ff9999','ffb3b3','ff9999','BDFFBD','99FF99','CCFFCC','---ALL---','default_new'); -INSERT INTO vicidial_screen_colors VALUES ('default_grey_agent','default grey agent','Y','FFFFFF','cccccc','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','---ALL---','DEFAULTAGENT.png'); +INSERT INTO vicidial_screen_colors VALUES ('red_rust','dark red rust','Y','804435','E7D0C2','C68C71','D9B39F','D9B49F','C68C72','C68C73','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'),('pale_green','pale green','Y','738035','E0E7C2','B6C572','C4CF8B','B6C572','C4CF8B','C4CF8B','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'),('alt_green','alternate green','Y','333333','D6E3B2','AEC866','BCD180','BCD180','AEC866','AEC866','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'),('default_blue_test','default blue test','Y','015B91','D9E6FE','9BB9FB','B9CBFD','8EBCFD','B6D3FC','A3C3D6','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'),('basic_orange','basic orange','Y','804d00','ffebcc','ffcc80','ffd699','ffcc80','ffd699','ffcc80','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'),('basic_purple','basic purple','Y','660066','ffccff','ff99ff','ffb3ff','ff99ff','ffb3ff','ff99ff','BDFFBD','99FF99','CCFFCC','---ALL---','SAMPLE.png','EFEFEF'),('basic_yellow','basic yellow','Y','666600','ffffcc','ffff66','ffff99','ffff66','ffff99','ffff66','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'),('basic_red','basic red','Y','800000','ffe6e6','ff9999','ffb3b3','ff9999','ffb3b3','ff9999','BDFFBD','99FF99','CCFFCC','---ALL---','default_new','EFEFEF'); +INSERT INTO vicidial_screen_colors VALUES ('default_grey_agent','default grey agent','Y','FFFFFF','cccccc','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','E6E6E6','---ALL---','DEFAULTAGENT.png','EFEFEF'); UPDATE system_settings SET qc_last_pull_time=NOW(); @@ -4644,6 +4662,8 @@ INSERT INTO vicidial_settings_containers(container_id,container_notes,container_ INSERT INTO vicidial_settings_containers(container_id,container_notes,container_type,user_group,container_entry) VALUES ('TIMEZONES_CANADA','Canadian Timezone List','TIMEZONE_LIST','---ALL---','CAN,NST,Y,Newfoundland Time Zone\nCAN,AST,Y,Atlantic Time Zone\nCAN,EST,Y,Eastern Time Zone\nCAN,CST,Y,Central Time Zone\nCAN,CST,N,Saskatchewan Time Zone\nCAN,MST,Y,Mountain Time Zone\nCAN,PST,Y,Pacific Time Zone\n'); INSERT INTO vicidial_settings_containers(container_id,container_notes,container_type,user_group,container_entry) VALUES ('TIMEZONES_AUSTRALIA','Australian Timezone List','TIMEZONE_LIST','---ALL---','AUS,AEST,Y,Eastern Australia Time Zone\nAUS,AEST,N,Queensland Time Zone\nAUS,ACST,Y,Central Australia Time Zone\nAUS,ACST,N,Northern Territory Time Zone\nAUS,AWST,N,Western Australia Time Zone\n'); +INSERT INTO vicidial_settings_containers VALUES ('INTERNATIONAL_DNC_IMPORT','Process DNC lists of various countries from FTP site','PERL_CLI','---ALL---','# This setting container is used for the international DNC system. \r\n# The below two settings are mandatory for importing suppression lists\r\n# and tell the import process where to look for new files and where to\r\n# move them when handled. These settings cannot have the same value. \r\n--file-dir=/root/ftp\r\n--file-destination=/root/ftp/DONE\r\n\r\n# Uncomment below and set the status to whatever custom disposition you \r\n# would like already-loaded leads to be set to when they dedupe against\r\n# a country\'s DNC list (default is \"DNCI\")\r\n# --dnc-status-override=BMNR\r\n\r\n# The below settings are optional for when files are stored on a remote\r\n# server. It is strongly recommended these settings are not used and\r\n# that the processing scripts and files are stored locally on the same\r\n# server. \r\n# --ftp-host=localhost\r\n# --ftp-user=user\r\n# --ftp-pwd=pwd\r\n# --ftp-port=21\r\n# --ftp-passive=1\r\n'),('DNC_IMPORT_FORMATS','Import formats for DNC files','OTHER','---ALL---','# This setting container is used for storing file formats used when \r\n# loading DNC suppression lists into the dialer. \r\n#\r\n# import template => (delimited|fixed),delimiter,phone1(,phone2,phone3)\r\n#\r\n# For delimited files, the phone1 value should be the index value of\r\n# the field where the phone appears. The first array index is 0 and\r\n# indexes continue through the natural numbers.\r\n\r\n# In delimited files, acceptable values for the \"delimiter\" field are:\r\n# - \"tab\", \"pipe\", \"comma\", \"quote-comma\"\r\nBASIC_DELIMITED_FORMAT => delimited,pipe,0\r\n\r\n# If the phone number is split into multiple fields (ex: area code in\r\n# one field, rest of the number in another), simply list additional \r\n# indices of the phone number fields separated by commas in the order \r\n# in which the data should be combined to make the complete phone \r\n# number \r\nDELIMITED_WITH_AC_AND_EXCHANGE_SPLIT => delimited,tab,0,1\r\n\r\n# For fixed-length files, the phone field values should be of the type:\r\n# - \"starting_position|length\"\r\nBASIC_FIXED_FORMAT => fixed,,0|10\r\n\r\n# (delimited|fixed) is not used for CSV/Excel files, so all that needs \r\n# providing for those is the index field value(s) of the phone number\r\nBASIC_CSV_OR_EXCEL_FORMAT => ,,0'),('DNC_CURRENT_BLOCKED_LISTS','Lists currently blocked due to pending DNC scrub','READ_ONLY','---ALL---',''); + UPDATE system_settings set vdc_agent_api_active='1'; -UPDATE system_settings SET db_schema_version='1602',db_schema_update_date=NOW(),reload_timestamp=NOW(); +UPDATE system_settings SET db_schema_version='1603',db_schema_update_date=NOW(),reload_timestamp=NOW(); diff --git a/extras/upgrade_2.14.sql b/extras/upgrade_2.14.sql index d94785e1..81ade9e8 100644 --- a/extras/upgrade_2.14.sql +++ b/extras/upgrade_2.14.sql @@ -1279,3 +1279,27 @@ UPDATE system_settings SET db_schema_version='1601',db_schema_update_date=NOW() ALTER TABLE system_settings MODIFY queuemetrics_pausereason ENUM('STANDARD','EVERY_NEW','EVERY_NEW_ADMINCALL','EVERY_NEW_ALLCALL') default 'STANDARD'; UPDATE system_settings SET db_schema_version='1602',db_schema_update_date=NOW() where db_schema_version < 1602; + +ALTER TABLE vicidial_screen_colors ADD button_color VARCHAR(6) default 'EFEFEF'; + +ALTER TABLE system_settings ADD enable_international_dncs ENUM('0','1') default '0'; + +CREATE TABLE vicidial_country_dnc_queue ( +dnc_file_id int(10) UNSIGNED NOT NULL AUTO_INCREMENT, +filename varchar(250) DEFAULT NULL, +country_code varchar(3) DEFAULT NULL, +file_layout varchar(30) DEFAULT NULL, +file_status enum('UPLOADING','READY','PENDING','INVALID LAYOUT','PROCESSING','FINISHED','CANCELLED') DEFAULT NULL, +file_action enum('PURGE','APPEND') DEFAULT NULL, +date_uploaded DATETIME DEFAULT NULL, +total_records int(10) UNSIGNED DEFAULT NULL, +records_processed int(10) UNSIGNED DEFAULT NULL, +records_inserted int(10) UNSIGNED DEFAULT NULL, +date_processed DATETIME DEFAULT NULL, +PRIMARY KEY (dnc_file_id), +KEY vicidial_country_dnc_queue_filename_key (filename) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + +INSERT INTO vicidial_settings_containers VALUES ('INTERNATIONAL_DNC_IMPORT','Process DNC lists of various countries from FTP site','PERL_CLI','---ALL---','# This setting container is used for the international DNC system. \r\n# The below two settings are mandatory for importing suppression lists\r\n# and tell the import process where to look for new files and where to\r\n# move them when handled. These settings cannot have the same value. \r\n--file-dir=/root/ftp\r\n--file-destination=/root/ftp/DONE\r\n\r\n# Uncomment below and set the status to whatever custom disposition you \r\n# would like already-loaded leads to be set to when they dedupe against\r\n# a country\'s DNC list (default is \"DNCI\")\r\n# --dnc-status-override=BMNR\r\n\r\n# The below settings are optional for when files are stored on a remote\r\n# server. It is strongly recommended these settings are not used and\r\n# that the processing scripts and files are stored locally on the same\r\n# server. \r\n# --ftp-host=localhost\r\n# --ftp-user=user\r\n# --ftp-pwd=pwd\r\n# --ftp-port=21\r\n# --ftp-passive=1\r\n'),('DNC_IMPORT_FORMATS','Import formats for DNC files','OTHER','---ALL---','# This setting container is used for storing file formats used when \r\n# loading DNC suppression lists into the dialer. \r\n#\r\n# import template => (delimited|fixed),delimiter,phone1(,phone2,phone3)\r\n#\r\n# For delimited files, the phone1 value should be the index value of\r\n# the field where the phone appears. The first array index is 0 and\r\n# indexes continue through the natural numbers.\r\n\r\n# In delimited files, acceptable values for the \"delimiter\" field are:\r\n# - \"tab\", \"pipe\", \"comma\", \"quote-comma\"\r\nBASIC_DELIMITED_FORMAT => delimited,pipe,0\r\n\r\n# If the phone number is split into multiple fields (ex: area code in\r\n# one field, rest of the number in another), simply list additional \r\n# indices of the phone number fields separated by commas in the order \r\n# in which the data should be combined to make the complete phone \r\n# number \r\nDELIMITED_WITH_AC_AND_EXCHANGE_SPLIT => delimited,tab,0,1\r\n\r\n# For fixed-length files, the phone field values should be of the type:\r\n# - \"starting_position|length\"\r\nBASIC_FIXED_FORMAT => fixed,,0|10\r\n\r\n# (delimited|fixed) is not used for CSV/Excel files, so all that needs \r\n# providing for those is the index field value(s) of the phone number\r\nBASIC_CSV_OR_EXCEL_FORMAT => ,,0'),('DNC_CURRENT_BLOCKED_LISTS','Lists currently blocked due to pending DNC scrub','READ_ONLY','---ALL---',''); + +UPDATE system_settings SET db_schema_version='1603',db_schema_update_date=NOW() where db_schema_version < 1603; diff --git a/www/vicidial/AST_CMstats.php b/www/vicidial/AST_CMstats.php index 70c5d0c4..7a08a5b7 100644 --- a/www/vicidial/AST_CMstats.php +++ b/www/vicidial/AST_CMstats.php @@ -584,7 +584,7 @@ if ($archives_available=="Y") { $MAIN.=""._QXZ("Search archived data")."

\n"; } -$MAIN.="    \n"; +$MAIN.="    \n"; $MAIN.="\n"; $MAIN.="\n"; $MAIN.="\n\n"; diff --git a/www/vicidial/AST_IVRstats.php b/www/vicidial/AST_IVRstats.php index 5126e7ce..fa08f51b 100644 --- a/www/vicidial/AST_IVRstats.php +++ b/www/vicidial/AST_IVRstats.php @@ -661,7 +661,7 @@ if ($archives_available=="Y") { $MAIN.="
"._QXZ("Search archived data")."
\n"; } -$MAIN.="    \n"; +$MAIN.="    \n"; $MAIN.="\n"; $MAIN.="\n\n"; diff --git a/www/vicidial/AST_LISTS_campaign_stats.php b/www/vicidial/AST_LISTS_campaign_stats.php index 52f58c52..04a88fd4 100644 --- a/www/vicidial/AST_LISTS_campaign_stats.php +++ b/www/vicidial/AST_LISTS_campaign_stats.php @@ -506,7 +506,7 @@ if ($archives_available=="Y") $MAIN.="

"._QXZ("Search archived data")."\n"; } -$MAIN.="

\n"; +$MAIN.="

\n"; $MAIN.="         "; $MAIN.=""; if (strlen($group[0]) > 1) diff --git a/www/vicidial/AST_LISTS_pass_report.php b/www/vicidial/AST_LISTS_pass_report.php index 716a7faa..89f76964 100644 --- a/www/vicidial/AST_LISTS_pass_report.php +++ b/www/vicidial/AST_LISTS_pass_report.php @@ -524,7 +524,7 @@ if ($archives_available=="Y") $MAIN.="
"._QXZ("Search archived data")."\n"; } $MAIN.="

\n"; -$MAIN.="\n"; +$MAIN.="\n"; $MAIN.="         "; $MAIN.=""; if ($use_lists > 0) diff --git a/www/vicidial/AST_LISTS_stats.php b/www/vicidial/AST_LISTS_stats.php index 461b56ef..223fc9ec 100644 --- a/www/vicidial/AST_LISTS_stats.php +++ b/www/vicidial/AST_LISTS_stats.php @@ -452,7 +452,7 @@ $MAIN.=""; #if ($report_display_type) {$MAIN.="";} #$MAIN.="  "; $MAIN.="


\n"; -$MAIN.="\n"; +$MAIN.="\n"; $MAIN.="         "; $MAIN.=""; if (strlen($group[0]) > 1) diff --git a/www/vicidial/AST_VDADstats.php b/www/vicidial/AST_VDADstats.php index 74d02592..4ca6eecb 100644 --- a/www/vicidial/AST_VDADstats.php +++ b/www/vicidial/AST_VDADstats.php @@ -742,7 +742,7 @@ $MAIN.="\n"; $MAIN.="\n"; $MAIN.="\n"; $MAIN.="

\n"; -$MAIN.="\n"; +$MAIN.="\n"; $MAIN.="         "; $MAIN.=""; $MAIN.=""._QXZ("DOWNLOAD")." |"; diff --git a/www/vicidial/AST_admin_template_maker.php b/www/vicidial/AST_admin_template_maker.php index 8ad7db8f..94f95021 100644 --- a/www/vicidial/AST_admin_template_maker.php +++ b/www/vicidial/AST_admin_template_maker.php @@ -514,7 +514,7 @@ if (mysqli_num_rows($template_rslt)>0) { - + type='submit' name='submit_template' onClick="return checkForm(this.form)" value=''> diff --git a/www/vicidial/AST_agent_timeclock_detail.php b/www/vicidial/AST_agent_timeclock_detail.php index 43477f86..ed4b4cbf 100644 --- a/www/vicidial/AST_agent_timeclock_detail.php +++ b/www/vicidial/AST_agent_timeclock_detail.php @@ -64,6 +64,8 @@ if (isset($_GET["report_display_type"])) {$report_display_type=$_GET["report_d if (strlen($shift)<2) {$shift='ALL';} if (strlen($stage)<2) {$stage='ID';} +require("screen_colors.php"); + $report_name = 'User Timeclock Detail Report'; $db_source = 'M'; @@ -519,7 +521,7 @@ window.open(url,"",'width=620,height=300,scrollbars=yes,menubar=yes,address=yes' - " name=smt id=smt onClick="submit_form()"> + type=button value="" name=smt id=smt onClick="submit_form()">
        "; diff --git a/www/vicidial/AST_dialer_inventory_report.php b/www/vicidial/AST_dialer_inventory_report.php index f48d3dfc..09d4ae04 100644 --- a/www/vicidial/AST_dialer_inventory_report.php +++ b/www/vicidial/AST_dialer_inventory_report.php @@ -981,7 +981,7 @@ $HTML_text.="
\n"; $HTML_text.="\n"; $HTML_text.="\n"; $HTML_text.="           "._QXZ("DOWNLOAD")." | "._QXZ("REPORTS")."

\n"; -$HTML_text.="\n"; +$HTML_text.="\n"; $HTML_text.="\n"; $HTML_text.="\n"; $HTML_text.="\n"; diff --git a/www/vicidial/AST_timeonVDADall.php b/www/vicidial/AST_timeonVDADall.php index 807725a0..c9ec7ba3 100644 --- a/www/vicidial/AST_timeonVDADall.php +++ b/www/vicidial/AST_timeonVDADall.php @@ -782,6 +782,8 @@ if (isset($RS_CUSTINFOminUL)) else {$CUSTINFOminUL = 9;} if ($LOGuser_level < $CUSTINFOminUL) {$CUSTINFOdisplay=0;} +require("screen_colors.php"); + $NFB = ''; $NFE = ''; $F=''; $FG=''; $B=''; $BG=''; @@ -986,7 +988,7 @@ $select_list .= ""; $select_list .= "
"; $select_list .= ""; -$select_list .= "         "; +$select_list .= "         "; $select_list .= ""; $select_list .= ""; $select_list .= "   "; diff --git a/www/vicidial/AST_timeonVDADallREC.php b/www/vicidial/AST_timeonVDADallREC.php index ab4d6a17..eeab0849 100644 --- a/www/vicidial/AST_timeonVDADallREC.php +++ b/www/vicidial/AST_timeonVDADallREC.php @@ -277,6 +277,7 @@ $NFB = ''; $NFE = ''; $F=''; $FG=''; $B=''; $BG=''; +require("screen_colors.php"); ?> @@ -412,7 +413,7 @@ if ($UGdisplay > 0) } echo "\n"; } -echo "         \n"; +echo "         \n"; echo ""._QXZ("STOP")." | "; echo ""._QXZ("SLOW")." | "; echo ""._QXZ("GO").""; diff --git a/www/vicidial/AST_timeonVDADallSUMMARY.php b/www/vicidial/AST_timeonVDADallSUMMARY.php index 0e52ad9f..979ddd19 100644 --- a/www/vicidial/AST_timeonVDADallSUMMARY.php +++ b/www/vicidial/AST_timeonVDADallSUMMARY.php @@ -222,6 +222,8 @@ while ($i < $groups_to_print) if (!isset($RR)) {$RR=4;} +require("screen_colors.php"); + $HEADER.="\n"; $HEADER.="\n"; $HEADER.="