440 lines
15 KiB
Perl
440 lines
15 KiB
Perl
#!/usr/bin/perl
|
|
#
|
|
# ADMIN_purge_log_tables.pl version 2.14
|
|
#
|
|
# This script is designed to purge records from the archive log tables and
|
|
# other log tables that have a tendency to grow unchecked. This should ONLY be
|
|
# ran after the ADMIN_archive_log_tables.pl script is ran. This will render most
|
|
# reports ineffective for dates older then 2 years. If you need to retain this
|
|
# data for compliance reasons then you should create a backup and retain that data
|
|
# in cold storage for later reference.
|
|
#
|
|
# Place in the crontab and have run 24 hours after the ADMIN_archive_log_tables.pl
|
|
# runs. The below crontab entry matches the recommended settings from the archive
|
|
# log table. This should be a time when the server is not in production or busy
|
|
# with other tasks
|
|
# 30 1 2 * * /usr/share/astguiclient/ADMIN_purge_log_tables.pl
|
|
#
|
|
# NOTE: On a high-load outbound dialing system, this script can take hours to
|
|
# run. While the script is running the system is unusable. Please schedule to
|
|
# run this script at a time when the system will not be used for several hours.
|
|
#
|
|
# original author: James Pearson <jamesp@vicidial.com>
|
|
# Based on perl scripts in ViciDial from Matt Florell and initial groundwork by Michael Cargile
|
|
#
|
|
# Copyright (C) 2017 James Pearson <jamesp@vicidial.com>, Matt Florell <vicidial@gmail.com> LICENSE: AGPLv2
|
|
#
|
|
#
|
|
# CHANGES
|
|
# 170117-1316 - First version
|
|
#
|
|
|
|
# All my lovely little modules
|
|
use warnings;
|
|
use POSIX qw(strftime);
|
|
use DBI;
|
|
use IO::Handle;
|
|
use File::Basename;
|
|
|
|
### Set-up some useful defaults
|
|
$DB=0;
|
|
$DBX=0;
|
|
$TEST=0;
|
|
$OPTIMIZE=0;
|
|
$VERBOSE=1;
|
|
$STARTTIME=time(); # Give output on how long it took to run
|
|
my $DBHOST = 'localhost';
|
|
my $DBNAME = 'asterisk';
|
|
my $DBUSER = 'cron';
|
|
my $DBPASS = '1234';
|
|
my $DBPORT = '3306';
|
|
my $clidbhost=0;
|
|
my $clidbname=0;
|
|
my $clidbuser=0;
|
|
my $clidbpass=0;
|
|
my $clidbport=0;
|
|
|
|
# Array of ages and table to check. Age first then table second. Line feeds are for readability
|
|
my @LOGTABLEAGE=('0','call_log_archive',
|
|
'30','call_log',
|
|
'30','twoday_call_log',
|
|
'30','twoday_recording_log',
|
|
'30','twoday_vicidial_agent_log',
|
|
'30','twoday_vicidial_closer_log',
|
|
'30','twoday_vicidial_log',
|
|
'30','twoday_vicidial_xfer_log',
|
|
'30','vicidial_cpd_log',
|
|
'30','vicidial_process_trigger_log',
|
|
'180','callcard_log',
|
|
'180','vicidial_api_log',
|
|
'180','vicidial_carrier_log_archive',
|
|
'180','vicidial_dial_log_archive',
|
|
'180','vicidial_list_update_log',
|
|
'180','vicidial_log_extended',
|
|
'180','vicidial_log_noanswer_archive',
|
|
'180','vicidial_outbound_ivr_log_archive',
|
|
'180','vicidial_url_log',
|
|
'730','park_log',
|
|
'730','vicidial_agent_log_archive',
|
|
'730','vicidial_did_agent_log_archive',
|
|
'730','vicidial_lead_search_log_archive',
|
|
'730','vicidial_log_extended_archive',
|
|
'730','vicidial_qc_agent_log',
|
|
'730','vicidial_remote_agent_log',
|
|
'730','vicidial_xfer_log');
|
|
|
|
# Lookup hash of the WHERE clause field for all the above tables. Basically just the field name to do the date comparison
|
|
my %TIMEFIELD = (call_log_archive => 'start_time',
|
|
call_log => 'start_time',
|
|
park_log => 'parked_time',
|
|
twoday_call_log => 'start_time',
|
|
twoday_recording_log => 'start_time',
|
|
twoday_vicidial_agent_log => 'event_time',
|
|
twoday_vicidial_closer_log => 'call_date',
|
|
twoday_vicidial_log => 'call_date',
|
|
twoday_vicidial_xfer_log => 'call_date',
|
|
vicidial_api_log => 'api_date',
|
|
vicidial_cpd_log => 'event_date',
|
|
vicidial_log_extended => 'call_date',
|
|
vicidial_process_trigger_log => 'trigger_time',
|
|
vicidial_url_log => 'url_date',
|
|
vicidial_carrier_log_archive => 'call_date',
|
|
callcard_log => 'call_time',
|
|
vicidial_dial_log_archive => 'call_date',
|
|
vicidial_did_agent_log_archive => 'call_date',
|
|
vicidial_lead_search_log_archive => 'event_date',
|
|
vicidial_list_update_log => 'event_date',
|
|
vicidial_log_extended_archive => 'call_date',
|
|
vicidial_log_noanswer_archive => 'call_date',
|
|
vicidial_outbound_ivr_log_archive => 'event_date',
|
|
vicidial_agent_log_archive => 'event_time',
|
|
vicidial_qc_agent_log => 'view_datetime',
|
|
vicidial_remote_agent_log => 'call_time',
|
|
vicidial_xfer_log => 'call_date');
|
|
|
|
|
|
sub trim($) {
|
|
# Perl trim function to remove whitespace from the start and end of the string, stolen from google cause i'm lazy
|
|
my $string = shift;
|
|
$string =~ s/^\s+//;
|
|
$string =~ s/\s+$//;
|
|
return $string;
|
|
}
|
|
|
|
sub debugoutput {
|
|
# I got tired of repeating this code snippet, so this gives debug output, with optional critical die
|
|
my $debugline = shift;
|
|
my $debugdie = 0;
|
|
$debugdie = shift;
|
|
if ($DB==1 and $debugdie==0) {
|
|
# We are just giving output, nothing more
|
|
print "$debugline\n";
|
|
} elsif ($DB==1 and $debugdie==1) {
|
|
# Evidently it was a critical error, so we die on output
|
|
die("$debugline\n");
|
|
}
|
|
}
|
|
|
|
sub debugxoutput {
|
|
# I got tired of repeating this code snippet, so this gives verbose debug output, with optional critical die
|
|
my $debugline = shift;
|
|
my $debugdie = 0;
|
|
$debugdie = shift;
|
|
if ($DBX==1 and $debugdie==0) {
|
|
# We are just giving output, nothing more
|
|
print "$debugline\n";
|
|
} elsif ($DBX==1 and $debugdie==1) {
|
|
# Evidently it was a critical error, so we die on output
|
|
die("$debugline\n");
|
|
}
|
|
}
|
|
|
|
sub mysqlolddate {
|
|
# Subtracts the number of days from epoch and returns it in a MySQL format
|
|
debugxoutput("---Begin mysqlolddate subroutine",0);
|
|
my $subtract_days=shift;
|
|
my $now=time();
|
|
debugxoutput(" Subtract Days: $subtract_days",0);
|
|
$subtract_days=$subtract_days*24*60*60;
|
|
$now=$now-$subtract_days;
|
|
$now=strftime("%Y-%m-%d 00:00:00",localtime($now));
|
|
debugxoutput(" Calculated date: $now",0);
|
|
debugxoutput("---End mysqlolddate subroutine",0);
|
|
return $now;
|
|
}
|
|
|
|
sub mysqltablecheck {
|
|
debugxoutput("---Begin mysqltablecheck subroutine",0);
|
|
my $databasename=trim(shift);
|
|
my $tablename=trim(shift);
|
|
debugxoutput(" Database Name to Check: $databasename",0);
|
|
my $stmtVDCHK="SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '$databasename' AND table_name = '$tablename';";
|
|
debugxoutput(" Statement check SQL: $stmtVDCHK",0);
|
|
my $sthVDCHK=$dbhDB->prepare($stmtVDCHK) or die "Preparing stmtVDCHK: ",$dbhDB->errstr;
|
|
$sthVDCHK->execute or die "Executing sthVDCHK: ",$dbhDB->errstr;
|
|
my @sthVDCHKROW=$sthVDCHK->fetchrow_array;
|
|
if ($sthVDCHKROW[0]>=1) {
|
|
debugxoutput(" Database appears to be ViciDial",0);
|
|
} else {
|
|
debugxoutput(" Database does not appear to be ViciDial",0);
|
|
$databasename='X';
|
|
}
|
|
$sthVDCHK->finish;
|
|
debugxoutput("---End mysqltablecheck subroutine",0);
|
|
return $databasename;
|
|
}
|
|
|
|
### concurrency checking so we don't loop ourselves
|
|
$RUNNING_FILE=basename($0);
|
|
debugxoutput(" Running Agent Script : $RUNNING_FILE",0);
|
|
my $grepout = `/bin/ps ax | grep $RUNNING_FILE | grep -v grep | grep -v /bin/sh`;
|
|
my $grepnum=0;
|
|
$grepnum++ while ($grepout =~ m/\n/g);
|
|
if ($grepnum > 1) { die("I am not alone! Another $0 is running! Exiting...\n"); }
|
|
|
|
### run-time CLI args parsing
|
|
$args = "";
|
|
if ( defined $ARGV[0] && length($ARGV[0])>1 ) {
|
|
$i=0;
|
|
while ($#ARGV >= $i) {
|
|
$args = "$args $ARGV[$i]";
|
|
$i++;
|
|
}
|
|
if ($args =~ /--help/i) {
|
|
print "\n\nViciDial database log purge\n\n";
|
|
print "allowed run time options:\n";
|
|
print " [--debug] = Debug; So show more output\n";
|
|
print " [--debugX] = Debug extended; Lots of output!\n";
|
|
print " [--test] = Test run only, no DB modifications; Disabled by Default\n";
|
|
print " [--optimize] = Optimize table after purge; Disabled by default\n";
|
|
print " [--quiet] = Be quiet instead of normal verbosity\n";
|
|
print " [--dbhost=$DBHOST] = Database Host or IP\n";
|
|
print " [--dbname=$DBNAME] = Database Name; 'all' by default or comma separated list\n";
|
|
print " [--dbuser=$DBUSER] = Database User\n";
|
|
print " [--dbpass=$DBPASS] = Database Password\n";
|
|
print " [--dbport=$DBPORT] = Database Port\n";
|
|
|
|
# Output the table purge info for reference
|
|
print "\n\nConfigured Tables and Ages to purge:\n";
|
|
my $numtables=@LOGTABLEAGE/2;
|
|
if ($numtables>0) {
|
|
my $i=0;
|
|
while ($i < @LOGTABLEAGE) {
|
|
my $j = $i + 1;
|
|
my $tablename = $LOGTABLEAGE[$j];
|
|
my $tableage = $LOGTABLEAGE[$i];
|
|
my $sqldate = mysqlolddate($tableage);
|
|
print " $tablename - $tableage days - Older then $sqldate\n";
|
|
$i=$i+2;
|
|
}
|
|
}
|
|
print "Found $numtables tables to check\n";
|
|
|
|
exit;
|
|
}
|
|
else
|
|
{
|
|
if ($args =~ /--debugX/i) {
|
|
$DB=1;
|
|
$DBX=1;
|
|
$VERBOSE=1;
|
|
print "\n\nViciDial database log purge\n\n";
|
|
print "\n----- DEBUG Extended -----\n\n";
|
|
}
|
|
if ($args =~ /--debug/i && $args !~ /--debugX/i) {
|
|
if ($DBX==0) { $DB=1; }
|
|
if ($VERBOSE==0) { $VERBOSE=1; }
|
|
print "\n\nViciDial database log purge\n\n";
|
|
print "\n----- DEBUG -----\n\n";
|
|
}
|
|
if ($args =~ /--test/i) {
|
|
$TEST=1;
|
|
debugoutput(" CLI Test Run : Enabled",0);
|
|
}
|
|
if ($args =~ /--optimize/i) {
|
|
$OPTIMIZE=1;
|
|
debugoutput(" CLI Optimize after : Enabled",0);
|
|
}
|
|
if ($args =~ /--quiet/i) {
|
|
if ($VERBOSE==1) { $VERBOSE=0; }
|
|
debugoutput(" CLI verbose : Disabled",0);
|
|
}
|
|
if ($args =~ /--dbhost=/i) {
|
|
@CLIdbhostARY = split(/--dbhost=/,$args);
|
|
@CLIdbhostARX = split(/ /,$CLIdbhostARY[1]);
|
|
if (length($CLIdbhostARX[0])>0) {
|
|
$DBHOST = $CLIdbhostARX[0];
|
|
$DBHOST =~ s/\/$| |\r|\n|\t//gi;
|
|
$clidbhost=1;
|
|
debugoutput(" CLI Database Host : $DBHOST",0);
|
|
}
|
|
}
|
|
if ($args =~ /--dbname=/i) {
|
|
@CLIdbnameARY = split(/--dbname=/,$args);
|
|
@CLIdbnameARX = split(/ /,$CLIdbnameARY[1]);
|
|
if (length($CLIdbnameARX[0])>0) {
|
|
$DBNAME = $CLIdbnameARX[0];
|
|
$DBNAME =~ s/\/$| |\r|\n|\t//gi;
|
|
$clidbname=1;
|
|
debugoutput(" CLI Database Name(s) : $DBNAME",0);
|
|
}
|
|
}
|
|
if ($args =~ /--dbuser=/i) {
|
|
@CLIdbuserARY = split(/--dbuser=/,$args);
|
|
@CLIdbuserARX = split(/ /,$CLIdbuserARY[1]);
|
|
if (length($CLIdbuserARX[0])>0) {
|
|
$DBUSER = $CLIdbuserARX[0];
|
|
$DBUSER =~ s/\/$| |\r|\n|\t//gi;
|
|
$clidbuser=1;
|
|
debugoutput(" CLI Database User : $DBUSER",0);
|
|
}
|
|
}
|
|
if ($args =~ /--dbpass=/i) {
|
|
@CLIdbpassARY = split(/--dbpass=/,$args);
|
|
@CLIdbpassARX = split(/ /,$CLIdbpassARY[1]);
|
|
if (length($CLIdbpassARX[0])>0) {
|
|
$DBPASS = $CLIdbpassARX[0];
|
|
$DBPASS =~ s/\/$| |\r|\n|\t//gi;
|
|
$clidbpass=1;
|
|
debugoutput(" CLI Database Password: $DBPASS",0);
|
|
}
|
|
}
|
|
if ($args =~ /--dbport=/i) {
|
|
@CLIdbportARY = split(/--dbport=/,$args);
|
|
@CLIdbportARX = split(/ /,$CLIdbportARY[1]);
|
|
if (length($CLIdbportARX[0])>0) {
|
|
$DBPORT = $CLIdbportARX[0];
|
|
$DBPORT =~ s/\/$| |\r|\n|\t//gi;
|
|
$clidbport=1;
|
|
debugoutput(" CLI Database Port : $DBPORT",0);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# default path to astguiclient configuration file:
|
|
$PATHconf = '/etc/astguiclient.conf';
|
|
|
|
# If we can read the conf file, then load it's settings
|
|
if ( -f $PATHconf )
|
|
{
|
|
open('conffile', "$PATHconf") || die "can't open $PATHconf: $!\n";
|
|
@conf = <conffile>;
|
|
close('conffile');
|
|
$i=0;
|
|
foreach(@conf)
|
|
{
|
|
$line = $conf[$i];
|
|
$line =~ s/ |>|\n|\r|\t|\#.*|;.*//gi;
|
|
if ( ($line =~ /^VARDB_server/) && ($clidbhost < 1) )
|
|
{$DBHOST = $line; $DBHOST =~ s/.*=//gi;}
|
|
if ( ($line =~ /^VARDB_database/) && ($clidbname < 1) )
|
|
{$DBNAME = $line; $DBNAME =~ s/.*=//gi;}
|
|
if ( ($line =~ /^VARDB_user/) && ($clidbuser < 1) )
|
|
{$DBUSER = $line; $DBUSER =~ s/.*=//gi;}
|
|
if ( ($line =~ /^VARDB_pass/) && ($clidbpass < 1) )
|
|
{$DBPASS = $line; $DBPASS =~ s/.*=//gi;}
|
|
if ( ($line =~ /^VARDB_port/) && ($clidbport < 1) )
|
|
{$DBPORT = $line; $DBPORT =~ s/.*=//gi;}
|
|
$i++;
|
|
}
|
|
} else {
|
|
if ($VERBOSE==1) { print " Missing Conf File : No $PATHconf, using defaults\n"; }
|
|
}
|
|
|
|
|
|
### Give feedback in debug mode on default runtime options
|
|
if ($DB==1) {
|
|
if ($TEST==0) { debugoutput(" Test Run : Disabled",0); }
|
|
if ($OPTIMIZE==0) { debugoutput(" Table optimize : Disabled",0); }
|
|
if ($clidbhost==0) { debugoutput(" Database Host : $DBHOST",0); }
|
|
if ($clidbname==0) { debugoutput(" Database Name : $DBNAME",0); }
|
|
if ($clidbuser==0) { debugoutput(" Database User : $DBUSER",0); }
|
|
if ($clidbpass==0) { debugoutput(" Database Password : $DBPASS",0); }
|
|
if ($clidbport==0) { debugoutput(" Database Port : $DBPORT",0); }
|
|
}
|
|
|
|
### Connect to database schema
|
|
$dbhDB = DBI->connect("DBI:mysql:information_schema:$DBHOST:$DBPORT", "$DBUSER", "$DBPASS") or die "Couldn't connect to ViciDial database: " . DBI->errstr;
|
|
|
|
### Parse through our databases, make sure they're vicidial ones
|
|
if ($VERBOSE==1) { print "Checking for ViciDial Databases...\n"; }
|
|
if ($DBNAME eq 'all') {
|
|
my $stmtALLDB="show databases;";
|
|
my $sthALLDB=$dbhDB->prepare($stmtALLDB) or die "Preparing stmtALLDB: ",$dbhDB->errstr;
|
|
$sthALLDB->execute or die "Executing sthALLDB: ",$dbhDB->errstr;
|
|
while (@sthALLDB=$sthALLDB->fetchrow_array) {
|
|
my $databasename=$sthALLDB[0];
|
|
debugxoutput(" Checking database $databasename",0);
|
|
if (mysqltablecheck($databasename,'call_log') ne 'X') {
|
|
if ($DB==1) { print " Adding $databasename to processing list\n"; }
|
|
push(@DBTOPROC,$databasename);
|
|
}
|
|
}
|
|
$sthALLDB->finish;
|
|
} else {
|
|
my @tempDB=split(',',$DBNAME);
|
|
foreach (@tempDB) {
|
|
my $databasename=trim($_);
|
|
debugoutput(" Checking database $databasename",0);
|
|
if (mysqltablecheck($databasename,'call_log') ne 'X') {
|
|
if ($DB==1) { print " Adding $databasename to process list\n"; }
|
|
push(@DBTOPROC,$databasename);
|
|
}
|
|
}
|
|
}
|
|
|
|
# Begin purging tables
|
|
$numofdb=@DBTOPROC;
|
|
debugoutput("Found $numofdb databases to process\n",0);
|
|
my $i=0;
|
|
while ( $i < @DBTOPROC ) {
|
|
my $vicidb=$DBTOPROC[$i];
|
|
if ($VERBOSE==1) { print "Processing database $vicidb\n"; }
|
|
my $j=0;
|
|
my $tableprocessed=0;
|
|
while ($j < @LOGTABLEAGE) {
|
|
my $tabletime=time();
|
|
my $k = $j + 1;
|
|
my $tablename = $LOGTABLEAGE[$k];
|
|
my $tableage = $LOGTABLEAGE[$j];
|
|
my $whereclause = 'X';
|
|
my $sqldate = mysqlolddate($tableage);
|
|
if (mysqltablecheck($vicidb,$tablename) ne 'X') {
|
|
if (exists $TIMEFIELD{$tablename} ) {
|
|
$whereclause=$TIMEFIELD{$tablename};
|
|
my $stmtDBPURGE = "delete from $vicidb.$tablename where $whereclause <= '$sqldate';";
|
|
my $stmtOPTIMIZE = "optimize table $vicidb.$tablename;";
|
|
if ($VERBOSE==1 && $DB==0) { print " Purging table $tablename older then $tableage days - "; }
|
|
elsif ($DB==1) { print " Purging table $tablename, $tableage days, SQL: $stmtDBPURGE - "; }
|
|
$tableprocessed++;
|
|
my $affectedrows=0;
|
|
if ($TEST==0) {
|
|
my $sthDBPURGE=$dbhDB->prepare($stmtDBPURGE) or die "Preparing stmtDBPURGE: ",$dbhDB->errstr;
|
|
$affectedrows = $sthDBPURGE->execute or die "Executing sthDBPURGE: ",$dbhDB->errstr;
|
|
$affectedrows = $affectedrows+0;
|
|
}
|
|
if ($OPTIMIZE==1) {
|
|
if ($VERBOSE==1) { print "$affectedrows rows, optimizing,"; }
|
|
if ($TEST==0) { $dbhDB->do($stmtOPTIMIZE) or debugoutput(" Cannot optimize table $tablename",0); }
|
|
my $runtime = time() - $tabletime;
|
|
if ($VERBOSE==1 || $DB==1) { print " $runtime secs\n"; }
|
|
} else {
|
|
if ($VERBOSE==1) {
|
|
my $runtime = time() - $tabletime;
|
|
print "$affectedrows rows, $runtime secs\n";
|
|
|
|
}
|
|
}
|
|
} else {
|
|
debugoutput(" No where clause found for table $tablename, skipping.",0);
|
|
}
|
|
}
|
|
$j=$j+2;
|
|
}
|
|
debugoutput(" Processed $tableprocessed tables",0);
|
|
$i++;
|
|
}
|
|
$RUNTIME = time() - $STARTTIME;
|
|
if ($VERBOSE==1) { print "Done in $RUNTIME seconds.\n"; }
|