diff --git a/extras/CRM_log_manage.pl b/extras/CRM_log_manage.pl index 4430c448..ca6a1e9e 100644 --- a/extras/CRM_log_manage.pl +++ b/extras/CRM_log_manage.pl @@ -29,10 +29,17 @@ # 10 0 * * * /usr/share/astguiclient/CRM_log_manage.pl --populate-crm-only --days=2 # 30 1 * * 0 /usr/share/astguiclient/CRM_log_manage.pl --purge-crm-only --days=91 # +# OPTIONAL SECONDARY SERVER DURING-THE-DAY INSERTS, DO NOT RUN ON MASTER DB!!! +# 1,16,31,46 7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 * * * /usr/share/astguiclient/CRM_log_manage.pl --pop-replace-crm-only=2 --skip-optimizes --quiet +# 10 6 * * * /usr/share/astguiclient/CRM_log_manage.pl --pop-replace-crm-only=6 --debugX +# 10 0 * * * /usr/share/astguiclient/CRM_log_manage.pl --pop-replace-crm-only=26 --debugX +# 30 1 * * 0 /usr/share/astguiclient/CRM_log_manage.pl --purge-crm-only --days=91 --debugX +# # Copyright (C) 2026 Matt Florell LICENSE: AGPLv2 # # CHANGES # 260117-0756 - First version, based on parts of VCA_log_manage script +# 260211-1552 - Added --pop-replace-crm-only=XX for SECONDARY DB USE ONLY # $CALC_TEST=0; @@ -41,6 +48,8 @@ $DB=0; $DBX=0; $populate_crm_only=0; $archive_tables=0; $purge_crm_only=0; +$skip_optimizes=0; +$POPreplaceHOURS=0; $POP_table[0] = 'vicidial_agent_log'; $POP_table[1] = 'vicidial_log'; @@ -76,6 +85,8 @@ if (length($ARGV[0])>1) print " [--populate-crm-only] = OPTIONAL, only populates crm_..._log tables then exits\n"; print " [--archive-tables] = OPTIONAL with --populate-crm-only only, use _archive tables for populate\n"; print " [--purge-crm-only] = OPTIONAL, only deletes crm_..._log tables records past number of days then exit\n"; + print " [--skip-optimizes] = OPTIONAL do not optimize tables after inserts/deletes \n"; + print " [--pop-replace-crm-only=XX] = OPTIONAL, only populates and replace crm_..._log tables XX hours then exits\n"; print " [--quiet] = quiet\n"; print " [--calc-test] = date calculation test only\n"; print " [--test] = test\n"; @@ -109,6 +120,12 @@ if (length($ARGV[0])>1) $CALC_TEST=1; print "\n-----DATE CALCULATION TESTING ONLY-----\n\n"; } + if ($args =~ /--skip-optimizes/i) + { + $skip_optimizes=1; + if ($Q < 1) + {print "\n----- SKIP TABLE OPTIMIZES ENABLED: $skip_optimizes -----\n\n";} + } if ($args =~ /--days=/i) { @data_in = split(/--days=/,$args); @@ -120,6 +137,17 @@ if (length($ARGV[0])>1) if ($Q < 1) {print "\n----- DAYS OVERRIDE: $CLIdays -----\n\n";} } + if ($args =~ /--pop-replace-crm-only=/i) + { + @data_in = split(/--pop-replace-crm-only=/,$args); + $POPreplaceHOURS = $data_in[1]; + $POPreplaceHOURS =~ s/ .*$//gi; + $POPreplaceHOURS =~ s/\D//gi; + if ($POPreplaceHOURS > 999999) + {$POPreplaceHOURS=25;} + if ($Q < 1) + {print "\n----- POP-REPLACE HOURS: $POPreplaceHOURS -----\n\n";} + } if ($args =~ /--populate-crm-only/i) { @@ -154,6 +182,17 @@ if (length($CLIdays)<1) $secX = time(); ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); +($NTsec,$NTmin,$NThour,$NTmday,$NTmon,$NTyear,$NTwday,$NTyday,$NTisdst) = localtime($secX); +$NTyear = ($NTyear + 1900); +$NTmon++; +if ($NTmon < 10) {$NTmon = "0$NTmon";} +if ($NTmday < 10) {$NTmday = "0$NTmday";} +if ($NThour < 10) {$NThour = "0$NThour";} +if ($NTmin < 10) {$NTmin = "0$NTmin";} +if ($NTsec < 10) {$NTsec = "0$NTsec";} +$now_time = "$NTyear-$NTmon-$NTmday $NThour:$NTmin:$NTsec"; +$now_date = "$NTyear-$NTmon-$NTmday"; + $del_epoch = ($secX - (86400 * $CLIdays)); # X days ago ($RMsec,$RMmin,$RMhour,$RMmday,$RMmon,$RMyear,$RMwday,$RMyday,$RMisdst) = localtime($del_epoch); $RMyear = ($RMyear + 1900); @@ -166,6 +205,20 @@ if ($RMsec < 10) {$RMsec = "0$RMsec";} $del_time = "$RMyear-$RMmon-$RMmday $RMhour:$RMmin:$RMsec"; $del_date = "$RMyear-$RMmon-$RMmday"; +if ($POPreplaceHOURS > 0) + { + $del_epoch = ($secX - (3600 * $POPreplaceHOURS)); # X hours ago + ($RMsec,$RMmin,$RMhour,$RMmday,$RMmon,$RMyear,$RMwday,$RMyday,$RMisdst) = localtime($del_epoch); + $RMyear = ($RMyear + 1900); + $RMmon++; + if ($RMmon < 10) {$RMmon = "0$RMmon";} + if ($RMmday < 10) {$RMmday = "0$RMmday";} + if ($RMhour < 10) {$RMhour = "0$RMhour";} + if ($RMmin < 10) {$RMmin = "0$RMmin";} + if ($RMsec < 10) {$RMsec = "0$RMsec";} + $del_time = "$RMyear-$RMmon-$RMmday $RMhour:$RMmin:$RMsec"; + $del_date = "$RMyear-$RMmon-$RMmday"; + } if (!$Q) {print "\n\n-- CRM_log_manage.pl --\n\n";} if (!$Q) {print "This program is designed manage the crm_..._log tables records, with \n";} @@ -225,6 +278,98 @@ $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VA if (!$T) { + ########## BEGIN --pop-replace-crm-only=XX flag processing ########## + if ($POPreplaceHOURS > 0) + { + $i=0; + foreach(@POP_table) + { + $source_table = $POP_table[$i]; + if ($archive_tables > 0) + {$source_table = $POP_table[$i]."_archive";} + if (!$Q) {print "\nStarting processing of populating $source_table table... $i\n";} + $stmtA = "SELECT count(*) from $source_table;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + $sthArows=$sthA->rows; + if ($sthArows > 0) + { + @aryA = $sthA->fetchrow_array; + $live_table_log_count = $aryA[0]; + } + $sthA->finish(); + + $stmtA = "SELECT count(*) from crm_$POP_table[$i];"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + $sthArows=$sthA->rows; + if ($sthArows > 0) + { + @aryA = $sthA->fetchrow_array; + $crm_table_log_count = $aryA[0]; + } + $sthA->finish(); + + if (!$Q) {print "POP-REPLACE Processing population from $source_table table... ($live_table_log_count|$crm_table_log_count)\n";} + + $stmtA = "LOCK TABLES crm_$POP_table[$i] WRITE, $POP_table[$i] READ;"; + my $LOCKaffected_rows = $dbhA->do($stmtA); + if ($DBX > 0) {print " DEBUG: |$LOCKaffected_rows|$stmtA|\n";} + + $stmtA = "DELETE FROM crm_$POP_table[$i] where $POP_date[$i] > \"$del_time\" and $POP_date[$i] <= \"$now_time\";"; + if ($DBX > 0) {print " DEBUG: |$stmtA|\n";} + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + $sthArows = $sthA->rows; + if (!$Q) {print "$sthArows rows deleted from crm_$POP_table[$i] table \n";} + + $stmtA = "INSERT IGNORE INTO crm_$POP_table[$i] SELECT * from $source_table where $POP_date[$i] > \"$del_time\" and $POP_date[$i] <= \"$now_time\";"; + if ($DBX > 0) {print " DEBUG: |$stmtA|\n";} + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + $sthArows = $sthA->rows; + if (!$Q) {print "$sthArows rows inserted into crm_$POP_table[$i] table \n";} + + $rv = $sthA->err(); + if (!$rv) + { + $stmtA = "UNLOCK TABLES;"; + my $LOCKaffected_rows = $dbhA->do($stmtA); + if ($DBX > 0) {print " DEBUG: |$LOCKaffected_rows|$stmtA|\n";} + + if ($skip_optimizes < 1) + { + if ($DBX > 0) {print " DEBUG: optimizing table crm_$POP_table[$i]... \n";} + $stmtA = "optimize table crm_$POP_table[$i];"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + } + } + else + { + if (!$Q) {print "ERROR for insert into crm_$POP_table[$i] table!!! \n";} + + $stmtA = "UNLOCK TABLES;"; + my $LOCKaffected_rows = $dbhA->do($stmtA); + if ($DBX > 0) {print " DEBUG: |$LOCKaffected_rows|$stmtA|\n";} + } + + if (!$Q) {print "POP-REPLACE Processing $source_table table finished: ($sthArows rows inserted into crm_$POP_table[$i]) \n";} + + $i++; + } + + ### calculate time to run script ### + $secY = time(); + $secZ = ($secY - $secX); + $secZm = ($secZ /60); + if (!$Q) {print "\nscript execution time in seconds: $secZ minutes: $secZm\n";} + + exit; + } + ########## END --pop-replace-crm-only=XX flag processing ########## + + ########## BEGIN --populate-crm-only flag processing ########## if ($populate_crm_only > 0) { @@ -270,9 +415,12 @@ if (!$T) $rv = $sthA->err(); if (!$rv) { - $stmtA = "optimize table crm_$POP_table[$i];"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if ($skip_optimizes < 1) + { + $stmtA = "optimize table crm_$POP_table[$i];"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + } } if (!$Q) {print "\nProcessing $source_table table finished: ($sthArows rows inserted into crm_$POP_table[$i]) \n";} @@ -316,9 +464,12 @@ if (!$T) $rv = $sthA->err(); if (!$rv) { - $stmtA = "optimize table crm_$POP_table[$i];"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if ($skip_optimizes < 1) + { + $stmtA = "optimize table crm_$POP_table[$i];"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + } } if (!$Q) {print "\nProcessing $POP_table[$i] table finished: ($sthArows rows deleted from crm_$POP_table[$i]) \n";}