From d7441c85d303fedfd50114af4197c445a75385b8 Mon Sep 17 00:00:00 2001 From: mattf Date: Fri, 12 Jul 2024 00:11:02 +0000 Subject: [PATCH] Added --no-optimize-tables flag to cold-storage script git-svn-id: svn://192.168.202.10@3851 3d104415-ff17-0410-8863-d5cf3c621b8a --- bin/ADMIN_cold_storage_log_tables.pl | 190 ++++++++++++++++++------ docs/LOG_ARCHIVING_AND_COLD_STORAGE.txt | 3 +- 2 files changed, 143 insertions(+), 50 deletions(-) diff --git a/bin/ADMIN_cold_storage_log_tables.pl b/bin/ADMIN_cold_storage_log_tables.pl index f179510c..38091bea 100644 --- a/bin/ADMIN_cold_storage_log_tables.pl +++ b/bin/ADMIN_cold_storage_log_tables.pl @@ -20,13 +20,15 @@ # # CHANGES # 240704-1815 - First version -# 250705-0954 - Added --query-count-test flag option +# 240705-0954 - Added --query-count-test flag option +# 240711-1938 - Added --no-optimize-tables flag option # $DB=0; $DBX=0; $CALC_TEST=0; $QUERY_COUNT_TEST=0; +$NO_OPTIMIZE_TABLES=0; $T=0; $TEST=0; $only_trim_archive=0; $recording_log_archive=0; @@ -55,6 +57,7 @@ if (length($ARGV[0])>1) print " [--quiet] = quiet\n"; print " [--calc-test] = date calculation test only\n"; print " [--query-count-test] = run archive counts test only\n"; + print " [--no-optimize-tables] = do not optimize the archive tables after deletion of rows \n"; print " [--test] = test\n"; print " [--debug] = debug output for some options\n"; print " [--debugX] = extra debug output for some options\n"; @@ -92,6 +95,11 @@ if (length($ARGV[0])>1) $QUERY_COUNT_TEST=1; print "\n-----ARCHIVE TABLES QUERY COUNT TESTING ONLY: $QUERY_COUNT_TEST-----\n\n"; } + if ($args =~ /--no-optimize-tables/i) + { + $NO_OPTIMIZE_TABLES=1; + print "\n-----DO NOT OPTIMIZE ARCHIVE TABLES AFTER ROWS DELETION: $NO_OPTIMIZE_TABLES-----\n\n"; + } if ($args =~ /--days=/i) { @data_in = split(/--days=/,$args); @@ -391,10 +399,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from call_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE call_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "call_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE call_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "call_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "call_log_archive table skip optimize\n";} + } } } } @@ -546,10 +561,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_log_archive table skip optimize\n";} + } } } } @@ -701,10 +723,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_log_extended_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_log_extended_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_log_extended_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_log_extended_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_log_extended_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_log_extended_archive table skip optimize\n";} + } } } } @@ -856,10 +885,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_dial_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_dial_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_dial_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_dial_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_dial_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_dial_log_archive table skip optimize\n";} + } } } } @@ -1011,10 +1047,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_carrier_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_carrier_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_carrier_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_carrier_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_carrier_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_carrier_log_archive table skip optimize\n";} + } } } } @@ -1166,10 +1209,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_dial_cid_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_dial_cid_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_dial_cid_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_dial_cid_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_dial_cid_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_dial_cid_log_archive table skip optimize\n";} + } } } } @@ -1321,10 +1371,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_amd_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_amd_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_amd_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_amd_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_amd_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_amd_log_archive table skip optimize\n";} + } } } } @@ -1476,10 +1533,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_agent_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_agent_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_agent_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_agent_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_agent_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_agent_log_archive table skip optimize\n";} + } } } } @@ -1631,10 +1695,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_api_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_api_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_api_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_api_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_api_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_api_log_archive table skip optimize\n";} + } } } } @@ -1786,10 +1857,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_api_urls_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_api_urls_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_api_urls_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_api_urls_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_api_urls_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_api_urls_archive table skip optimize\n";} + } } } } @@ -1941,10 +2019,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_agent_visibility_log_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_agent_visibility_log_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_agent_visibility_log_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_agent_visibility_log_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_agent_visibility_log_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_agent_visibility_log_archive table skip optimize\n";} + } } } } @@ -2100,10 +2185,17 @@ if (!$T) $affected_rowsA = $dbhA->do($stmtA); if (!$Q) {print STDERR "records deleted from vicidial_log_extended_sip_archive: $affected_rowsA|$stmtA|\n";} - $stmtA = "OPTIMIZE TABLE vicidial_log_extended_sip_archive;"; - $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; - $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; - if (!$Q) {print STDERR "vicidial_log_extended_sip_archive table optimized, cold-storage process is complete for this table!\n";} + if ($NO_OPTIMIZE_TABLES < 1) + { + $stmtA = "OPTIMIZE TABLE vicidial_log_extended_sip_archive;"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + if (!$Q) {print STDERR "vicidial_log_extended_sip_archive table optimized, cold-storage process is complete for this table!\n";} + } + else + { + if (!$Q) {print STDERR "vicidial_log_extended_sip_archive table skip optimize\n";} + } } } } diff --git a/docs/LOG_ARCHIVING_AND_COLD_STORAGE.txt b/docs/LOG_ARCHIVING_AND_COLD_STORAGE.txt index fdaa455c..88ea01e1 100644 --- a/docs/LOG_ARCHIVING_AND_COLD_STORAGE.txt +++ b/docs/LOG_ARCHIVING_AND_COLD_STORAGE.txt @@ -1,4 +1,4 @@ -LOG ARCHIVING AND COLD STORAGE DOC Started: 2024-07-04 Updated: 2024-07-05 +LOG ARCHIVING AND COLD STORAGE DOC Started: 2024-07-04 Updated: 2024-07-11 This document covers the options for archiving of specific log tables and the new "Cold Storage" features @@ -126,6 +126,7 @@ allowed run time options: [--quiet] = quiet [--calc-test] = date calculation test only [--query-count-test] = run archive counts test only + [--no-optimize-tables] = do not optimize the archive tables after deletion of rows [--test] = test [--debug] = debug output for some options [--debugX] = extra debug output for some options