diff --git a/bin/ADMIN_purge_log_tables.pl b/bin/ADMIN_purge_log_tables.pl new file mode 100644 index 00000000..1cb35ca6 --- /dev/null +++ b/bin/ADMIN_purge_log_tables.pl @@ -0,0 +1,439 @@ +#!/usr/bin/perl +# +# ADMIN_purge_log_tables.pl version 2.14 +# +# This script is designed to purge records from the archive log tables and +# other log tables that have a tendency to grow unchecked. This should ONLY be +# ran after the ADMIN_archive_log_tables.pl script is ran. This will render most +# reports ineffective for dates older then 2 years. If you need to retain this +# data for compliance reasons then you should create a backup and retain that data +# in cold storage for later reference. +# +# Place in the crontab and have run 24 hours after the ADMIN_archive_log_tables.pl +# runs. The below crontab entry matches the recommended settings from the archive +# log table. This should be a time when the server is not in production or busy +# with other tasks +# 30 1 2 * * /usr/share/astguiclient/ADMIN_purge_log_tables.pl +# +# NOTE: On a high-load outbound dialing system, this script can take hours to +# run. While the script is running the system is unusable. Please schedule to +# run this script at a time when the system will not be used for several hours. +# +# original author: James Pearson +# Based on perl scripts in ViciDial from Matt Florell and initial groundwork by Michael Cargile +# +# Copyright (C) 2017 James Pearson , Matt Florell LICENSE: AGPLv2 +# +# +# CHANGES +# 170117-1316 - First version +# + +# All my lovely little modules +use warnings; +use POSIX qw(strftime); +use DBI; +use IO::Handle; +use File::Basename; + +### Set-up some useful defaults +$DB=0; +$DBX=0; +$TEST=0; +$OPTIMIZE=0; +$VERBOSE=1; +$STARTTIME=time(); # Give output on how long it took to run +my $DBHOST = 'localhost'; +my $DBNAME = 'asterisk'; +my $DBUSER = 'cron'; +my $DBPASS = '1234'; +my $DBPORT = '3306'; +my $clidbhost=0; +my $clidbname=0; +my $clidbuser=0; +my $clidbpass=0; +my $clidbport=0; + +# Array of ages and table to check. Age first then table second. Line feeds are for readability +my @LOGTABLEAGE=('0','call_log_archive', +'30','call_log', +'30','twoday_call_log', +'30','twoday_recording_log', +'30','twoday_vicidial_agent_log', +'30','twoday_vicidial_closer_log', +'30','twoday_vicidial_log', +'30','twoday_vicidial_xfer_log', +'30','vicidial_cpd_log', +'30','vicidial_process_trigger_log', +'180','callcard_log', +'180','vicidial_api_log', +'180','vicidial_carrier_log_archive', +'180','vicidial_dial_log_archive', +'180','vicidial_list_update_log', +'180','vicidial_log_extended', +'180','vicidial_log_noanswer_archive', +'180','vicidial_outbound_ivr_log_archive', +'180','vicidial_url_log', +'730','park_log', +'730','vicidial_agent_log_archive', +'730','vicidial_did_agent_log_archive', +'730','vicidial_lead_search_log_archive', +'730','vicidial_log_extended_archive', +'730','vicidial_qc_agent_log', +'730','vicidial_remote_agent_log', +'730','vicidial_xfer_log'); + +# Lookup hash of the WHERE clause field for all the above tables. Basically just the field name to do the date comparison +my %TIMEFIELD = (call_log_archive => 'start_time', +call_log => 'start_time', +park_log => 'parked_time', +twoday_call_log => 'start_time', +twoday_recording_log => 'start_time', +twoday_vicidial_agent_log => 'event_time', +twoday_vicidial_closer_log => 'call_date', +twoday_vicidial_log => 'call_date', +twoday_vicidial_xfer_log => 'call_date', +vicidial_api_log => 'api_date', +vicidial_cpd_log => 'event_date', +vicidial_log_extended => 'call_date', +vicidial_process_trigger_log => 'trigger_time', +vicidial_url_log => 'url_date', +vicidial_carrier_log_archive => 'call_date', +callcard_log => 'call_time', +vicidial_dial_log_archive => 'call_date', +vicidial_did_agent_log_archive => 'call_date', +vicidial_lead_search_log_archive => 'event_date', +vicidial_list_update_log => 'event_date', +vicidial_log_extended_archive => 'call_date', +vicidial_log_noanswer_archive => 'call_date', +vicidial_outbound_ivr_log_archive => 'event_date', +vicidial_agent_log_archive => 'event_time', +vicidial_qc_agent_log => 'view_datetime', +vicidial_remote_agent_log => 'call_time', +vicidial_xfer_log => 'call_date'); + + +sub trim($) { + # Perl trim function to remove whitespace from the start and end of the string, stolen from google cause i'm lazy + my $string = shift; + $string =~ s/^\s+//; + $string =~ s/\s+$//; + return $string; +} + +sub debugoutput { + # I got tired of repeating this code snippet, so this gives debug output, with optional critical die + my $debugline = shift; + my $debugdie = 0; + $debugdie = shift; + if ($DB==1 and $debugdie==0) { + # We are just giving output, nothing more + print "$debugline\n"; + } elsif ($DB==1 and $debugdie==1) { + # Evidently it was a critical error, so we die on output + die("$debugline\n"); + } +} + +sub debugxoutput { + # I got tired of repeating this code snippet, so this gives verbose debug output, with optional critical die + my $debugline = shift; + my $debugdie = 0; + $debugdie = shift; + if ($DBX==1 and $debugdie==0) { + # We are just giving output, nothing more + print "$debugline\n"; + } elsif ($DBX==1 and $debugdie==1) { + # Evidently it was a critical error, so we die on output + die("$debugline\n"); + } +} + +sub mysqlolddate { + # Subtracts the number of days from epoch and returns it in a MySQL format + debugxoutput("---Begin mysqlolddate subroutine",0); + my $subtract_days=shift; + my $now=time(); + debugxoutput(" Subtract Days: $subtract_days",0); + $subtract_days=$subtract_days*24*60*60; + $now=$now-$subtract_days; + $now=strftime("%Y-%m-%d 00:00:00",localtime($now)); + debugxoutput(" Calculated date: $now",0); + debugxoutput("---End mysqlolddate subroutine",0); + return $now; +} + +sub mysqltablecheck { + debugxoutput("---Begin mysqltablecheck subroutine",0); + my $databasename=trim(shift); + my $tablename=trim(shift); + debugxoutput(" Database Name to Check: $databasename",0); + my $stmtVDCHK="SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '$databasename' AND table_name = '$tablename';"; + debugxoutput(" Statement check SQL: $stmtVDCHK",0); + my $sthVDCHK=$dbhDB->prepare($stmtVDCHK) or die "Preparing stmtVDCHK: ",$dbhDB->errstr; + $sthVDCHK->execute or die "Executing sthVDCHK: ",$dbhDB->errstr; + my @sthVDCHKROW=$sthVDCHK->fetchrow_array; + if ($sthVDCHKROW[0]>=1) { + debugxoutput(" Database appears to be ViciDial",0); + } else { + debugxoutput(" Database does not appear to be ViciDial",0); + $databasename='X'; + } + $sthVDCHK->finish; + debugxoutput("---End mysqltablecheck subroutine",0); + return $databasename; +} + +### concurrency checking so we don't loop ourselves +$RUNNING_FILE=basename($0); +debugxoutput(" Running Agent Script : $RUNNING_FILE",0); +my $grepout = `/bin/ps ax | grep $RUNNING_FILE | grep -v grep | grep -v /bin/sh`; +my $grepnum=0; +$grepnum++ while ($grepout =~ m/\n/g); +if ($grepnum > 1) { die("I am not alone! Another $0 is running! Exiting...\n"); } + +### run-time CLI args parsing +$args = ""; +if ( defined $ARGV[0] && length($ARGV[0])>1 ) { + $i=0; + while ($#ARGV >= $i) { + $args = "$args $ARGV[$i]"; + $i++; + } + if ($args =~ /--help/i) { + print "\n\nViciDial database log purge\n\n"; + print "allowed run time options:\n"; + print " [--debug] = Debug; So show more output\n"; + print " [--debugX] = Debug extended; Lots of output!\n"; + print " [--test] = Test run only, no DB modifications; Disabled by Default\n"; + print " [--optimize] = Optimize table after purge; Disabled by default\n"; + print " [--quiet] = Be quiet instead of normal verbosity\n"; + print " [--dbhost=$DBHOST] = Database Host or IP\n"; + print " [--dbname=$DBNAME] = Database Name; 'all' by default or comma separated list\n"; + print " [--dbuser=$DBUSER] = Database User\n"; + print " [--dbpass=$DBPASS] = Database Password\n"; + print " [--dbport=$DBPORT] = Database Port\n"; + + # Output the table purge info for reference + print "\n\nConfigured Tables and Ages to purge:\n"; + my $numtables=@LOGTABLEAGE/2; + if ($numtables>0) { + my $i=0; + while ($i < @LOGTABLEAGE) { + my $j = $i + 1; + my $tablename = $LOGTABLEAGE[$j]; + my $tableage = $LOGTABLEAGE[$i]; + my $sqldate = mysqlolddate($tableage); + print " $tablename - $tableage days - Older then $sqldate\n"; + $i=$i+2; + } + } + print "Found $numtables tables to check\n"; + + exit; + } + else + { + if ($args =~ /--debugX/i) { + $DB=1; + $DBX=1; + $VERBOSE=1; + print "\n\nViciDial database log purge\n\n"; + print "\n----- DEBUG Extended -----\n\n"; + } + if ($args =~ /--debug/i && $args !~ /--debugX/i) { + if ($DBX==0) { $DB=1; } + if ($VERBOSE==0) { $VERBOSE=1; } + print "\n\nViciDial database log purge\n\n"; + print "\n----- DEBUG -----\n\n"; + } + if ($args =~ /--test/i) { + $TEST=1; + debugoutput(" CLI Test Run : Enabled",0); + } + if ($args =~ /--optimize/i) { + $OPTIMIZE=1; + debugoutput(" CLI Optimize after : Enabled",0); + } + if ($args =~ /--quiet/i) { + if ($VERBOSE==1) { $VERBOSE=0; } + debugoutput(" CLI verbose : Disabled",0); + } + if ($args =~ /--dbhost=/i) { + @CLIdbhostARY = split(/--dbhost=/,$args); + @CLIdbhostARX = split(/ /,$CLIdbhostARY[1]); + if (length($CLIdbhostARX[0])>0) { + $DBHOST = $CLIdbhostARX[0]; + $DBHOST =~ s/\/$| |\r|\n|\t//gi; + $clidbhost=1; + debugoutput(" CLI Database Host : $DBHOST",0); + } + } + if ($args =~ /--dbname=/i) { + @CLIdbnameARY = split(/--dbname=/,$args); + @CLIdbnameARX = split(/ /,$CLIdbnameARY[1]); + if (length($CLIdbnameARX[0])>0) { + $DBNAME = $CLIdbnameARX[0]; + $DBNAME =~ s/\/$| |\r|\n|\t//gi; + $clidbname=1; + debugoutput(" CLI Database Name(s) : $DBNAME",0); + } + } + if ($args =~ /--dbuser=/i) { + @CLIdbuserARY = split(/--dbuser=/,$args); + @CLIdbuserARX = split(/ /,$CLIdbuserARY[1]); + if (length($CLIdbuserARX[0])>0) { + $DBUSER = $CLIdbuserARX[0]; + $DBUSER =~ s/\/$| |\r|\n|\t//gi; + $clidbuser=1; + debugoutput(" CLI Database User : $DBUSER",0); + } + } + if ($args =~ /--dbpass=/i) { + @CLIdbpassARY = split(/--dbpass=/,$args); + @CLIdbpassARX = split(/ /,$CLIdbpassARY[1]); + if (length($CLIdbpassARX[0])>0) { + $DBPASS = $CLIdbpassARX[0]; + $DBPASS =~ s/\/$| |\r|\n|\t//gi; + $clidbpass=1; + debugoutput(" CLI Database Password: $DBPASS",0); + } + } + if ($args =~ /--dbport=/i) { + @CLIdbportARY = split(/--dbport=/,$args); + @CLIdbportARX = split(/ /,$CLIdbportARY[1]); + if (length($CLIdbportARX[0])>0) { + $DBPORT = $CLIdbportARX[0]; + $DBPORT =~ s/\/$| |\r|\n|\t//gi; + $clidbport=1; + debugoutput(" CLI Database Port : $DBPORT",0); + } + } + } +} + +# default path to astguiclient configuration file: +$PATHconf = '/etc/astguiclient.conf'; + +# If we can read the conf file, then load it's settings +if ( -f $PATHconf ) + { + open('conffile', "$PATHconf") || die "can't open $PATHconf: $!\n"; + @conf = ; + close('conffile'); + $i=0; + foreach(@conf) + { + $line = $conf[$i]; + $line =~ s/ |>|\n|\r|\t|\#.*|;.*//gi; + if ( ($line =~ /^VARDB_server/) && ($clidbhost < 1) ) + {$DBHOST = $line; $DBHOST =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_database/) && ($clidbname < 1) ) + {$DBNAME = $line; $DBNAME =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_user/) && ($clidbuser < 1) ) + {$DBUSER = $line; $DBUSER =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_pass/) && ($clidbpass < 1) ) + {$DBPASS = $line; $DBPASS =~ s/.*=//gi;} + if ( ($line =~ /^VARDB_port/) && ($clidbport < 1) ) + {$DBPORT = $line; $DBPORT =~ s/.*=//gi;} + $i++; + } + } else { + if ($VERBOSE==1) { print " Missing Conf File : No $PATHconf, using defaults\n"; } + } + + +### Give feedback in debug mode on default runtime options +if ($DB==1) { + if ($TEST==0) { debugoutput(" Test Run : Disabled",0); } + if ($OPTIMIZE==0) { debugoutput(" Table optimize : Disabled",0); } + if ($clidbhost==0) { debugoutput(" Database Host : $DBHOST",0); } + if ($clidbname==0) { debugoutput(" Database Name : $DBNAME",0); } + if ($clidbuser==0) { debugoutput(" Database User : $DBUSER",0); } + if ($clidbpass==0) { debugoutput(" Database Password : $DBPASS",0); } + if ($clidbport==0) { debugoutput(" Database Port : $DBPORT",0); } +} + +### Connect to database schema +$dbhDB = DBI->connect("DBI:mysql:information_schema:$DBHOST:$DBPORT", "$DBUSER", "$DBPASS") or die "Couldn't connect to ViciDial database: " . DBI->errstr; + +### Parse through our databases, make sure they're vicidial ones +if ($VERBOSE==1) { print "Checking for ViciDial Databases...\n"; } +if ($DBNAME eq 'all') { + my $stmtALLDB="show databases;"; + my $sthALLDB=$dbhDB->prepare($stmtALLDB) or die "Preparing stmtALLDB: ",$dbhDB->errstr; + $sthALLDB->execute or die "Executing sthALLDB: ",$dbhDB->errstr; + while (@sthALLDB=$sthALLDB->fetchrow_array) { + my $databasename=$sthALLDB[0]; + debugxoutput(" Checking database $databasename",0); + if (mysqltablecheck($databasename,'call_log') ne 'X') { + if ($DB==1) { print " Adding $databasename to processing list\n"; } + push(@DBTOPROC,$databasename); + } + } + $sthALLDB->finish; + } else { + my @tempDB=split(',',$DBNAME); + foreach (@tempDB) { + my $databasename=trim($_); + debugoutput(" Checking database $databasename",0); + if (mysqltablecheck($databasename,'call_log') ne 'X') { + if ($DB==1) { print " Adding $databasename to process list\n"; } + push(@DBTOPROC,$databasename); + } + } +} + +# Begin purging tables +$numofdb=@DBTOPROC; +debugoutput("Found $numofdb databases to process\n",0); +my $i=0; +while ( $i < @DBTOPROC ) { + my $vicidb=$DBTOPROC[$i]; + if ($VERBOSE==1) { print "Processing database $vicidb\n"; } + my $j=0; + my $tableprocessed=0; + while ($j < @LOGTABLEAGE) { + my $tabletime=time(); + my $k = $j + 1; + my $tablename = $LOGTABLEAGE[$k]; + my $tableage = $LOGTABLEAGE[$j]; + my $whereclause = 'X'; + my $sqldate = mysqlolddate($tableage); + if (mysqltablecheck($vicidb,$tablename) ne 'X') { + if (exists $TIMEFIELD{$tablename} ) { + $whereclause=$TIMEFIELD{$tablename}; + my $stmtDBPURGE = "delete from $vicidb.$tablename where $whereclause <= '$sqldate';"; + my $stmtOPTIMIZE = "optimize table $vicidb.$tablename;"; + if ($VERBOSE==1 && $DB==0) { print " Purging table $tablename older then $tableage days - "; } + elsif ($DB==1) { print " Purging table $tablename, $tableage days, SQL: $stmtDBPURGE - "; } + $tableprocessed++; + my $affectedrows=0; + if ($TEST==0) { + my $sthDBPURGE=$dbhDB->prepare($stmtDBPURGE) or die "Preparing stmtDBPURGE: ",$dbhDB->errstr; + $affectedrows = $sthDBPURGE->execute or die "Executing sthDBPURGE: ",$dbhDB->errstr; + $affectedrows = $affectedrows+0; + } + if ($OPTIMIZE==1) { + if ($VERBOSE==1) { print "$affectedrows rows, optimizing,"; } + if ($TEST==0) { $dbhDB->do($stmtOPTIMIZE) or debugoutput(" Cannot optimize table $tablename",0); } + my $runtime = time() - $tabletime; + if ($VERBOSE==1 || $DB==1) { print " $runtime secs\n"; } + } else { + if ($VERBOSE==1) { + my $runtime = time() - $tabletime; + print "$affectedrows rows, $runtime secs\n"; + + } + } + } else { + debugoutput(" No where clause found for table $tablename, skipping.",0); + } + } + $j=$j+2; + } + debugoutput(" Processed $tableprocessed tables",0); + $i++; +} +$RUNTIME = time() - $STARTTIME; +if ($VERBOSE==1) { print "Done in $RUNTIME seconds.\n"; }