Added Internal Process Logs generation and admin display

git-svn-id: svn://192.168.202.10@3994 3d104415-ff17-0410-8863-d5cf3c621b8a

agc_2-X/trunk/bin/ADMIN_archive_log_tables.pl

@@ -20,7 +20,7 @@
 # Based on perl scripts in ViciDial from Matt Florell and post: 
 # http://www.vicidial.org/VICIDIALforum/viewtopic.php?p=22506&sid=ca5347cffa6f6382f56ce3db9fb3d068#22506
 #
-# Copyright (C) 2025  I. Taushanov, Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  I. Taushanov, Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGES
 # 90615-1701 - First version
@@ -77,6 +77,7 @@
 # 251011-2048 - Added archiving of recording_live_log table
 # 251024-1518 - Added --vicidial-dial-log-only flag
 # 260111-2138 - Added --agent-log-only flag
+# 260516-2149 - Added internal logging
 #
 
 $CALC_TEST=0;
@@ -91,6 +92,7 @@ $api_archive_only=0;
 $url_log_only=0;
 $url_log_archive=0;
 $agent_log_only=0;
+$script_name = 'ADMIN_archive_log_tables.pl';
 
 
 ### begin parsing run-time options ###
@@ -625,6 +627,8 @@ use DBI;
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
  or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 #############################################
 ##### Gather system_settings #####
 $stmtA = "SELECT sip_event_logging FROM system_settings;";
@@ -4480,4 +4484,17 @@ $secZ = ($secY - $secX);
 $secZm = ($secZ /60);
 if (!$Q) {print "\nscript execution time in seconds: $secZ     minutes: $secZm\n";}
 
+$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='finished', stage='Run seconds: $secZ' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+if($DB){print STDERR "|$stmtA|";}
+my $affected_rows = $dbhA->do($stmtA);
+if($DB){print STDERR "$affected_rows|\n";}
+
 exit;
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/ADMIN_keepalive_ALL.pl

@@ -182,9 +182,10 @@
 # 260327-0846 - Added check of empty phone dialplan extensions when building conf files
 # 260402-1440 - Added reset of vicidial_max_inbound_cache table entries
 # 260515-1610 - Changed multi-listen-process kill section to only run if listen process is supposed to run on this server
+# 260515-2121 - Added truncating of vicidial_internal_log entries after 7 days, updating of some records
 #
 
-$build = '260515-1610';
+$build = '260515-2121';
 
 $DB=0; # Debug flag
 $teodDB=0; # flag to log Timeclock End of Day processes to log file
@@ -801,6 +802,14 @@ else
 		$i++;
 		}
 
+	# if $runningFastAGI_log running, update internal process log
+	if ( ($runningFastAGI_log > 0) && ($reset_test =~ /0$|5$/) )
+		{
+		$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage=CONCAT((UNIX_TIMESTAMP(NOW())-UNIX_TIMESTAMP(db_time)),' seconds runtime') WHERE process='FastAGI_log.pl' and server_ip='$server_ip' order by db_time desc limit 1;";
+		if($DB){print STDERR "|$stmtA|";}
+		my $affected_rows = $dbhA->do($stmtA);
+		if($DB){print STDERR "$affected_rows|\n";}
+		}
 
 
 
@@ -866,8 +875,6 @@ else
 
 
 
-
-
 	##### Third, double-check that non-running scripts are not running #####
 	@psline=@MT;
 	@psoutput=@MT;
@@ -2401,7 +2408,25 @@ if ($timeclock_end_of_day_NOW > 0)
 		@aryA = $sthA->fetchrow_array;
 		if ($DB) {print "|",$aryA[0],"|",$aryA[1],"|",$aryA[2],"|",$aryA[3],"|","\n";}
 		$sthA->finish();
-		##### END vicidial_dtmf_log end of day process removing records older than 7 days #####
+		##### END vicidial_dtmf_log end of day process removing records older than 24 hours #####
+
+
+		##### BEGIN vicidial_internal_log end of day process removing records older than 7 days #####
+		$stmtA = "DELETE from vicidial_internal_log where up_time < \"$SDSQLdate\";";
+		if($DBX){print STDERR "\n|$stmtA|\n";}
+		$affected_rows = $dbhA->do($stmtA);
+		if($DB){print STDERR "\n|$affected_rows vicidial_internal_log records older than 7 days purged|\n";}
+		if ($teodDB) {$event_string = "vicidial_internal_log records older than 7 days purged: |$stmtA|$affected_rows|";   &teod_logger;}
+
+		$stmtA = "optimize table vicidial_internal_log;";
+		if($DBX){print STDERR "\n|$stmtA|\n";}
+		$sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
+		$sthA->execute or die "executing: $stmtA ", $dbhA->errstr;
+		$sthArows=$sthA->rows;
+		@aryA = $sthA->fetchrow_array;
+		if ($DB) {print "|",$aryA[0],"|",$aryA[1],"|",$aryA[2],"|",$aryA[3],"|","\n";}
+		$sthA->finish();
+		##### END vicidial_internal_log end of day process removing records older than 7 days #####
 
 		
 		##### BEGIN vicidial_lead_messages end of day process removing records older than 1 day #####
@@ -5894,6 +5919,11 @@ if ($active_asterisk_server =~ /Y/)
 			if ($DBX) {print "Restart asterisk debug 1: |$Iaffected_rows|$stmtA|\n";}
 
 			`screen -XS asterisk eval 'stuff "/usr/sbin/asterisk -vvvvgcT\015"'`;
+
+			$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), action='start', stage='restarted', process='asterisk auto-restart', server_ip='$server_ip';";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
 			}
 		elsif ($uptime_seconds>300)
  			{

agc_2-X/trunk/bin/AST_VDadapt.pl

@@ -6,7 +6,7 @@
 # adjusts the auto_dial_level for vicidial adaptive-predictive campaigns. 
 # gather call stats for campaigns and in-groups
 #
-# Copyright (C) 2025  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGELOG
 # 60823-1302 - First build from AST_VDhopper.pl
@@ -64,9 +64,11 @@
 # 250131-1624 - Modifications to fix cached hour counts for realtime report
 # 250218-1613 - Modified master loop to use microseconds instead of a counter to execute drop/dial level/shared dialing functions
 # 251205-0949 - Added adaptive_percentmax_percentage option and ADAPT_PERCENTMAX dial_method, fix for dropped cached counts
+# 260515-1938 - Added internal logging
 #
 
-$build='251205-0949';
+$build='260515-1938';
+$script_name = 'AST_VDadapt.pl';
 # constants
 $DB=0;  # Debug flag, set to 0 for no debug messages, On an active system this will generate lots of lines of output per minute
 $US='__';
@@ -320,6 +322,8 @@ $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VA
 $dbhB = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
  or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER '.$build;   &internal_logger;
+
 if ($DBX) {print "CONNECTED TO DATABASE:  $VARDB_server|$VARDB_database\n";}
 
 ##### gather relevent system settings
@@ -434,6 +438,7 @@ if ($run_check > 0)
 	}
 
 $master_loop=0;
+$iLog_ct=0;
 
 ### Start master loop ###
 while ($master_loop < $CLIloops) 
@@ -474,7 +479,7 @@ while ($master_loop < $CLIloops)
 	if ($min < 10) {$min = "0$min";}
 	if ($sec < 10) {$sec = "0$sec";}
 
-	if ($DBXXX) {print "TIME DEBUG: $master_loop   $LOCAL_GMT_OFF_STD|$LOCAL_GMT_OFF|$isdst|   GMT: $hour:$min\n";}
+	if ($DBXXX) {print "TIME DEBUG: $master_loop|$iLog_ct   $LOCAL_GMT_OFF_STD|$LOCAL_GMT_OFF|$isdst|   GMT: $hour:$min\n";}
 
 	if (!$prev_iteration_ms) {$prev_iteration_ms=time();}
 
@@ -2487,6 +2492,15 @@ while ($master_loop < $CLIloops)
 
 	$stat_count++;
 	$master_loop++;
+	# update internal process log
+	$iLog_ct++;
+	if ($iLog_ct =~ /00$/) 
+		{
+		$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$VARserver_ip' order by db_time desc limit 1;";
+		if($DB){print STDERR "|$stmtA|";}
+		my $affected_rows = $dbhA->do($stmtA);
+		if($DB){print STDERR "$affected_rows|\n";}
+		}
 	}
 
 $dbhA->disconnect();
@@ -7401,6 +7415,14 @@ sub call_quota_logging
 ##### END Call Quota Lead Ranking logging #####
 
 
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$VARserver_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}
+
 ##### BEGIN math divisor sub #####
 sub MathZDC
 	{

agc_2-X/trunk/bin/AST_VDauto_dial.pl

@@ -25,7 +25,7 @@
 # It is good practice to keep this program running by placing the associated 
 # KEEPALIVE script running every minute to ensure this program is always running
 #
-# Copyright (C) 2025  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGELOG:
 # 50125-1201 - Changed dial timeout to 120 seconds from 180 seconds
@@ -162,10 +162,12 @@
 # 240731-0814 - Fix for 2nd DB connection timing out while running
 # 240917-1719 - Change multiple after-call processing to only run for active_voicemail_server
 # 251205-1455 - Added SHARED_ADAPT_PERCENTMAX option
+# 260515-1937 - Added internal logging
 #
 
-$build='251205-1455';
+$build='260515-1937';
 $script='AST_VDauto_dial';
+$script_name = 'AST_VDauto_dial.pl';
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
 	{
@@ -298,6 +300,8 @@ or die "Couldn't connect to database: " . DBI->errstr;
 $dbhC = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER '.$build;   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vd_server_logs,vicidial_recording_limit,asterisk_version,routing_prefix FROM servers where server_ip = '$server_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -418,6 +422,8 @@ while($one_day_interval > 0)
 	{
 	$endless_loop=5760000;		# 30 days minutes at XXX seconds per loop
 	$stat_count=1;
+	$iLog_ct=0;
+	$iLog_calls=0;
 
 	while($endless_loop > 0)
 		{
@@ -2405,6 +2411,7 @@ while($one_day_interval > 0)
 										#	usleep(1*50*1000);
 											usleep(1*$per_call_delay*1000);
 											$calls_placed++;
+											$iLog_calls++;
 											}
 										}
 									$call_CMPIPct++;
@@ -5288,6 +5295,16 @@ while($one_day_interval > 0)
 			$stmtA = "INSERT INTO vicidial_shared_log SET log_time='$now_date',total_agents='$shared_dial_camp_override',debug_output='BUILD: $build\n$debug_shared_output$temp_dead_debug',campaign_id='-SHARE-',server_ip='$VARserver_ip';";
 			$affected_rows = $dbhA->do($stmtA);
 			}
+
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /00$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct   Events: $iLog_calls' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
 		}
 
 
@@ -5887,6 +5904,14 @@ sub aad_output
 	}
 
 
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}
+
 # subroutine to parse the asterisk version
 # and return a hash with the various part
 sub parse_asterisk_version

agc_2-X/trunk/bin/AST_VDauto_dial_FILL.pl

@@ -12,7 +12,7 @@
 #
 # Should only be run on one server in a multi-server Asterisk/VICIDIAL cluster
 #
-# Copyright (C) 2023  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGELOG:
 # 61115-1246 - First build, framework setup, non-functional
@@ -55,6 +55,7 @@
 # 230830-1056 - Changed outbound_calls_per_second max to allow up to 1000 CPS
 # 231116-0916 - Added hopper_hold_inserts option
 # 231129-0905 - Added vicidial_phone_number_call_daily_counts updates/inserts
+# 260515-1941 - Added internal logging
 #
 
 ### begin parsing run-time options ###
@@ -124,6 +125,7 @@ $US='__';
 $MT[0]='';
 $RECcount=''; ### leave blank for no REC count
 $RECprefix='7'; ### leave blank for no REC prefix
+$script_name = 'AST_VDauto_dial_FILL.pl';
 
 
 # default path to astguiclient configuration file:
@@ -180,6 +182,8 @@ use DBI;
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vd_server_logs FROM servers where server_ip = '$server_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -241,6 +245,7 @@ if ($non_latin > 0)
 	$sthA->finish();
 	}
 
+$iLog_ct=0;
 $one_day_interval = 12;		# 1 month loops for one year 
 while($one_day_interval > 0)
 	{
@@ -1612,6 +1617,16 @@ while($one_day_interval > 0)
 			&get_time_now;
 			}
 
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /00$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
+
 		$stat_count++;
 		}
 
@@ -1785,3 +1800,12 @@ sub parse_asterisk_version
 
         return ( %ast_ver_hash );
 }
+
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/AST_VDremote_agents.pl

@@ -59,9 +59,11 @@
 # 240420-2246 - Added ConfBridge code
 # 240516-2149 - Allow for ALT start_call_url, added --version flag
 # 260403-2253 - Added vicidial_max_inbound_cache logging
+# 260515-1935 - Added internal logging
 #
 
-$build = '260403-2253';
+$build = '260515-1933';
+$script_name = 'AST_VDremote_agents.pl';
 
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
@@ -190,6 +192,7 @@ use DBI;
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
  or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER '.$build;   &internal_logger;
 
 #############################################
 ##### Gather system_settings #####
@@ -252,6 +255,7 @@ if ($run_check > 0)
 
 #$one_day_interval = 12;	# 1 month loops for one year 
 $one_day_interval = 1;		# 1 day
+$iLog_ct=0;
 while($one_day_interval > 0)
 	{
 	#$endless_loop=5760000;		# 30 days minutes at XXX seconds per loop
@@ -1253,7 +1257,7 @@ while($one_day_interval > 0)
 		usleep(1*$loop_delay*1000);
 
 		$endless_loop--;
-		if($DB){print STDERR "\nloop counter: |$endless_loop|$one_day_interval|     |$loop_delay|\n";}
+		if($DB){print STDERR "\nloop counter: |$endless_loop|$one_day_interval|$iLog_ct|     |$loop_delay|\n";}
 
 		### putting a blank file called "VDAD.kill" in the directory will automatically safely kill this program
 		if (-e "$PATHhome/VDAD.kill")
@@ -1265,6 +1269,16 @@ while($one_day_interval > 0)
 			}
 
 		$bad_grabber_counter=0;
+
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /00$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
 		}
 
 
@@ -1402,6 +1416,15 @@ sub event_logger
 	}
 
 
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}
+
+
 ##### BEGIN Call Quota Lead Ranking logging #####
 sub call_quota_logging
 	{

agc_2-X/trunk/bin/AST_cleanup_agent_log.pl

@@ -10,7 +10,7 @@
 #
 # This program only needs to be run by one server
 #
-# Copyright (C) 2022  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGES
 # 60711-0945 - Changed to DBI by Marin Blu
@@ -48,11 +48,14 @@
 # 210523-2321 - Added optional check for qa_data duplicates, -qm-qa-duplicate-check
 # 220206-0714 - Added new -roll-back-preview-skips function to roll back the last_local_call_time and called_count for leads that were previewed and not dialed
 # 250625-2329 - Added --wait-pause-code-swap option
+# 260515-2216 - Added internal logging
 #
 
 # constants
 $US='__';
 $MT[0]='';
+$script_name = 'AST_cleanup_agent_log.pl';
+$start_sec = time();
 
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
@@ -472,6 +475,8 @@ use DBI;
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 #############################################
 ##### START QUEUEMETRICS LOGGING LOOKUP #####
 $stmtA = "SELECT enable_queuemetrics_logging,queuemetrics_server_ip,queuemetrics_dbname,queuemetrics_login,queuemetrics_pass,queuemetrics_log_id,queuemetrics_eq_prepend,queuemetrics_loginout,queuemetrics_dispo_pause,queuemetrics_pause_type FROM system_settings;";
@@ -3088,6 +3093,14 @@ if ($enable_queuemetrics_logging > 0)
 
 
 
+# update internal process log
+$now_sec = time();
+$run_sec = ($now_sec - $start_sec);
+$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='finished', stage='Run seconds: $run_sec' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+if($DB){print STDERR "|$stmtA|";}
+my $affected_rows = $dbhA->do($stmtA);
+if($DB){print STDERR "$affected_rows|\n";}
+
 
 
 
@@ -3114,3 +3127,12 @@ sub event_logger
 	close(Lout);
 	$event_string='';
 	}
+
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/AST_conf_update_3way.pl

@@ -11,7 +11,7 @@
 #      script's crontab entry that does some of these functions:
 #      AST_conf_update.pl --no-vc-3way-check
 #
-# Copyright (C) 2023  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # 100811-2119 - First build, based upon AST_conf_update.pl script
 # 100928-1506 - Changed from hard-coded 60 minute limit to servers.vicidial_recording_limit
@@ -23,6 +23,7 @@
 # 200413-1356 - Fix for \n\n at the end of PING commands causing errors in AMI
 # 231117-2255 - Added AMI version 5 compatibility
 # 231118-1110 - Added override option of up to Xtimeout9 for 3WAY_... leave-3way sessions
+# 260515-1943 - Added internal logging
 #
 
 # constants
@@ -32,6 +33,7 @@ $US='__';
 $MT[0]='';
 $loops = '100000';
 $CLIdelay = '3';
+$script_name = 'AST_conf_update_3way.pl';
 
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
@@ -145,6 +147,8 @@ use Net::Telnet ();
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
  or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vicidial_recording_limit,asterisk_version,vd_server_logs FROM servers where server_ip = '$server_ip';";
 if ($DB) {print "|$stmtA|\n";}
@@ -210,6 +214,7 @@ $event_string = "AST_conf_update_3way.pl script starting: $year-$mon-$mday $hour
  &event_logger;
 
 $loop_counter=0;
+$iLog_ct=0;
 
 while ($loops > $loop_counter)
 	{
@@ -490,6 +495,16 @@ while ($loops > $loop_counter)
 		sleep(10);
 		$loop_counter=9999999;
 		}
+
+	# update internal process log
+	$iLog_ct++;
+	if ($iLog_ct =~ /00$/) 
+		{
+		$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+		if($DB){print STDERR "|$stmtA|";}
+		my $affected_rows = $dbhA->do($stmtA);
+		if($DB){print STDERR "$affected_rows|\n";}
+		}
 	}
 
 
@@ -557,3 +572,11 @@ sub event_logger
 		}
 	$event_string='';
 	}
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/AST_conf_update_screen.pl

@@ -11,12 +11,14 @@
 # 220414-1210 - Initial build
 # 250305-2104 - Fix for missing vicidial_confbridges limit purge code
 # 250326-2102 - Fixes for ConfBridge issues
+# 260515-1939 - Added internal logging
 #
 
 # constants
 $DB=0;  # Debug flag, set to 0 for no debug messages, lots of output
 $DBX=0;
 $run_check=1; # concurrency check
+$script_name = 'AST_conf_update_screen.pl';
 
 # loop time in seconds
 $loop_time = 0.4;
@@ -124,6 +126,8 @@ if (try_load($module))
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vd_server_logs,asterisk_version,conf_engine,conf_update_interval,vicidial_recording_limit FROM servers where server_ip = '$server_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -248,6 +252,7 @@ else
 	### BEGIN manager event handling for asterisk version >= 13
 	$endless_loop = 1;
 	$loop_count = 0;
+	$iLog_ct=0;
 
 	# get the number of microseconds the loop is supposed to take
 	$loop_time_usec = $loop_time * 1000000;
@@ -796,6 +801,15 @@ else
 			usleep($sleep_usec);
 			}
 
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /00$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
 		}
 	}
 
@@ -856,6 +870,14 @@ sub event_logger
 	}
 
 
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}
+
 
 # subroutine to parse the asterisk version
 # and return a hash with the various part

agc_2-X/trunk/bin/AST_flush_DBqueue.pl

@@ -9,7 +9,7 @@
 # !!!!!!!! IMPORTANT !!!!!!!!!!!!!!!!!!
 # THIS SCRIPT SHOULD ONLY BE RUN ON ONE SERVER ON YOUR CLUSTER
 #
-# Copyright (C) 2025  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGES
 # 60717-1214 - changed to DBI by Marin Blu
@@ -38,6 +38,7 @@
 # 240709-1300 - Added Validate XFER vicidial_auto_calls: "--check-xfers" flag
 # 250914-1537 - Added archiving of recording_live table
 # 251003-0837 - Added --preserve-dtmf flag (DTMF logs would then be kept for 1-2 days)
+# 260515-2212 - Added internal logging
 #
 
 $session_flush=0;
@@ -48,6 +49,8 @@ $stuck_listsSQL='';
 $preserve_dtmf=0;
 $check_xfers=0;
 $vicidial_recording_limit=60;
+$script_name = 'AST_flush_DBqueue.pl';
+$start_sec = time();
 
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
@@ -303,6 +306,8 @@ use DBI;
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
  or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT vd_server_logs,vicidial_recording_limit FROM servers where server_ip = '$VARserver_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -1127,7 +1132,24 @@ if ($rl_count > 0)
 	}
 ### END archive recording_live that are FINISHED ###
 
+# update internal process log
+$now_sec = time();
+$run_sec = ($now_sec - $start_sec);
+$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='finished', stage='Run seconds: $run_sec' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+if($DB){print STDERR "|$stmtA|";}
+my $affected_rows = $dbhA->do($stmtA);
+if($DB){print STDERR "$affected_rows|\n";}
+
 
 $dbhA->disconnect();
 
 exit;
+
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/AST_manager_listen_AMI2.pl

@@ -27,6 +27,7 @@
 # 210407-2009 - Added Event handlers for new manager events
 # 251007-2110 - Added code for recording_dtmf_detection and recording_dtmf_muting
 # 260327-1516 - Added support for PJSIP (requires patched Asterisk 18+)
+# 260515-1933 - Added internal logging
 #
 
 # constants
@@ -36,6 +37,7 @@ $full_listen_log=0; # set to 1 to log all output to log file
 $run_check=1; # concurrency check
 $last_keepalive_epoch = time();
 $keepalive_skips=0;
+$script_name = 'AST_manager_listen_AMI2.pl';
 
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
@@ -128,6 +130,8 @@ if (try_load($module))
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vd_server_logs,asterisk_version FROM servers where server_ip = '$server_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -264,6 +268,8 @@ else
 	{
 	### BEGIN manager event handling for asterisk version >= 13
 	$endless_loop = 1;
+	$iLog_ct=0;
+	$iLog_calls=0;
 	while($endless_loop > 0)
 		{
 		$breakout = 1;
@@ -453,6 +459,7 @@ else
 			# handle the event
 			$retcode = handle_event( %event_hash );
 			$last_event_epoch = time();
+			$iLog_calls++;
 			}
 		elsif ( ( exists($event_hash{"Response"}) ) && ( exists($event_hash{"Ping"}) ) )
 			{
@@ -470,6 +477,15 @@ else
 			$event_string="Asterisk server shutting down, PROCESS KILLED... EXITING|ONE DAY INTERVAL:$one_day_interval|";
 			&event_logger;
 			}
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /00$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct   Events: $iLog_calls' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
 		}
 
 	usleep(1*100*1000);
@@ -1107,6 +1123,14 @@ sub dtmf_logger
 		}
 	}
 
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}
+
 # subroutine to parse the asterisk version
 # and return a hash with the various part
 sub parse_asterisk_version

agc_2-X/trunk/bin/AST_manager_send.pl

@@ -16,7 +16,7 @@
 # scalability over just using a single process. Also, this means that a single
 # action execution lock cannot bring the entire system down.
 # 
-# Copyright (C) 2017  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2025  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGES
 # 50823-1514 - Added commandline debug options with debug printouts
@@ -31,6 +31,7 @@
 # 91129-2146 - removed SELECT STAR and formatting fixes
 # 141113-1356 - Added more logging, check for number of running instances, processing of QUEUE but not SENT commands
 # 170920-1419 - Fix for issue with recordings beginning with CALLID variable
+# 260515-1940 - Added internal logging
 #
 
 $|++;
@@ -40,6 +41,7 @@ use Getopt::Long;
 use Time::HiRes ('gettimeofday','usleep','sleep');  # necessary to have perl sleep command of less than one second
 
 my $run_check=1; # concurrency check
+my $script_name = 'AST_manager_send.pl';
 
 # constants and globals
 my $servConf;
@@ -102,6 +104,8 @@ $conf{VARDB_port} = '3306' unless ($conf{VARDB_port});
 my $dbhA = DBI->connect("DBI:mysql:" . $conf{VARDB_database} . ":" . $conf{VARDB_server} . ":" . $conf{VARDB_port},
 	$conf{VARDB_user}, $conf{VARDB_pass}) or die "Couldn't connect to database: " . DBI->errstr;
 
+my $action='start';   my $stage='LOGGED INTO MYSQL SERVER';   my $server_ip = $conf{VARserver_ip}; &internal_logger;
+
 ### Grab Server values from the database
 $servConf = getServerConfig($dbhA, $conf{VARserver_ip});
 $SYSLOG = 1 if ($servConf->{vd_server_logs} =~ /Y/);
@@ -130,6 +134,8 @@ while ($one_day_interval > 0)
 	my $endless_loop = 1728000;		# 2 days at .10 seconds per loop
 	my $affected_rows;
 	my $NEW_actions;
+	my $iLog_ct=0;
+	my $iLog_calls=0;
 	while ($endless_loop > 0) 
 		{
 		my $stmtA = "SELECT count(*) from vicidial_manager where server_ip = '" . $conf{VARserver_ip} . "' and status = 'NEW';";
@@ -277,6 +283,7 @@ while ($one_day_interval > 0)
 					eventLogger($conf{'PATHlogs'}, 'process', $event_string);
 					}
 				$processed_actions++;
+				$iLog_calls++;
 				}
 			$sthA->finish();
 			}
@@ -285,11 +292,22 @@ while ($one_day_interval > 0)
 			{
 			### sleep for 1 hundredth of a second if just send an ACTION
 			usleep(1*10*1000);
+			$iLog_ct++;
 			}
 		else
 			{
 			### sleep for 10 hundredths of a second if no actions sent
 			usleep(1*100*1000);
+			$iLog_ct++;
+			}
+
+		# update internal process log
+		if ($iLog_ct =~ /000$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct   Events: $iLog_calls' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
 			}
 
 		$endless_loop--;
@@ -438,3 +456,11 @@ sub logDate
 	}
 
 ### End of subs
+
+sub internal_logger
+	{
+	my $stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/AST_manager_sip_AMI2.pl

@@ -11,6 +11,7 @@
 # CHANGES
 # 190528-1304 - Copied AST_manager_listen_AMI2.pl to make the initial version
 # 260327-1509 - Added support for PJSIP (requires patched Asterisk 18+)
+# 260515-1934 - Added internal logging
 #
 
 # constants
@@ -21,6 +22,7 @@ $run_check=1; # concurrency check
 $last_keepalive_epoch = time();
 $keepalive_skips=0;
 $ami_user='sipcron';
+$script_name = 'AST_manager_sip_AMI2.pl';
 
 ### begin parsing run-time options ###
 if (length($ARGV[0])>1)
@@ -113,6 +115,8 @@ if (try_load($module))
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vd_server_logs,asterisk_version FROM servers where server_ip = '$server_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -225,6 +229,8 @@ $keep_alive_sec = 30;
 $keep_alive_skips = 0;
 $keep_alive_response = 1;
 $keep_alive_no_responses = 0;
+$iLog_ct=0;
+$iLog_calls=0;
 
 %ast_ver_str = parse_asterisk_version($asterisk_version);
 if (( $ast_ver_str{major} = 1 ) && ($ast_ver_str{minor} < 13))
@@ -427,6 +433,7 @@ else
 			# handle the event
 			$retcode = handle_event( %event_hash );
 			$last_event_epoch = time();
+			$iLog_calls++;
 			}
 		elsif ( ( exists($event_hash{"Response"}) ) && ( exists($event_hash{"Ping"}) ) )
 			{
@@ -444,6 +451,16 @@ else
 			$event_string="Asterisk server shutting down, PROCESS KILLED... EXITING|ONE DAY INTERVAL:$one_day_interval|";
 			&event_logger;
 			}
+
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /00$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct   Events: $iLog_calls' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
 		}
 
 	usleep(1*100*1000);
@@ -713,6 +730,15 @@ sub manager_output_logger
 	}
 
 
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}
+
+
 # subroutine to parse the asterisk version
 # and return a hash with the various part
 sub parse_asterisk_version

agc_2-X/trunk/bin/AST_update_AMI2.pl

@@ -5,7 +5,7 @@
 # This script uses the Asterisk Manager interface to update the live_channels
 # tables and verify the parked_channels table in the asterisk MySQL database
 #
-# Copyright (C) 2025  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGES
 # 170915-2110 - Initial version for Asterisk 13, based upon AST_update.pl
@@ -21,12 +21,14 @@
 # 220310-1136 - Fix for issue dealing with bad carrier 'P-Asserted-Identity' input
 # 251008-2114 - Added code for recording_dtmf_detection and recording_dtmf_muting
 # 251203-2218 - Added server_live_partitions inserts/updates
+# 260515-1939 - Added internal logging
 #
 
 # constants
 $DB=0;  # Debug flag, set to 0 for no debug messages, lots of output
 $DBX=0;
 $run_check=1; # concurrency check
+$script_name = 'AST_update_AMI2.pl';
 
 # loop time in seconds
 $loop_time = 0.4;
@@ -159,6 +161,8 @@ if (try_load($module))
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER '.$build;   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT telnet_host,telnet_port,ASTmgrUSERNAME,ASTmgrSECRET,ASTmgrUSERNAMEupdate,ASTmgrUSERNAMElisten,ASTmgrUSERNAMEsend,max_vicidial_trunks,answer_transfer_agent,local_gmt,ext_context,vd_server_logs,asterisk_version FROM servers where server_ip = '$server_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -341,6 +345,8 @@ else
 	### BEGIN manager event handling for asterisk version >= 13
 	$endless_loop = 1;
 	$loop_count = 0;
+	$iLog_ct=0;
+	$iLog_calls=0;
 
 	# get the number of microseconds the loop is supposed to take
 	$loop_time_usec = $loop_time * 1000000;
@@ -718,6 +724,15 @@ else
 
 				}
 			}
+		# update internal process log
+		$iLog_ct++;
+		if ($iLog_ct =~ /000$|500$/) 
+			{
+			$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+			if($DB){print STDERR "|$stmtA|";}
+			my $affected_rows = $dbhA->do($stmtA);
+			if($DB){print STDERR "$affected_rows|\n";}
+			}
 		}
 	}
 
@@ -1870,3 +1885,11 @@ sub get_disk_rw
 
 	return ( $reads, $writes );
 	}
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/FastAGI_log.pl

@@ -106,6 +106,7 @@
 # 250928-1556 - Added hosted_settings dialplan variable
 # 260327-0824 - Fixes for PJSIP compatibility
 # 260424-1644 - Fix for rare Auto-Alt Dial issue
+# 260515-1936 - Added internal logging
 #
 
 # defaults for PreFork
@@ -119,6 +120,7 @@ $VARfastagi_log_checkforwait =	'60';
 $DB=0;
 $DBX=0;
 $ADB=0;
+$script_name = 'FastAGI_log.pl';
 
 # default path to astguiclient configuration file:
 $PATHconf =		'/etc/astguiclient.conf';
@@ -180,6 +182,12 @@ use DBI;
 $dbhB = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
 	or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';
+$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$VARserver_ip', action='$action', stage='$stage';";
+if($DB){print STDERR "|$stmtA|";}
+my $affected_rows = $dbhB->do($stmtA);
+if($DB){print STDERR "$affected_rows|\n";}
+
 ### Grab Server values from the database
 $stmtB = "SELECT vd_server_logs,asterisk_version FROM servers where server_ip = '$VARserver_ip';";
 $sthB = $dbhB->prepare($stmtB) or die "preparing: ",$dbhB->errstr;

agc_2-X/trunk/bin/VD_email_inbound.pl

@@ -2,7 +2,7 @@
 # default path to astguiclient configuration file:
 $PATHconf =		'/etc/astguiclient.conf';
 
-# Copyright (C) 2024  Joe Johnson, Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Joe Johnson, Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # VD_email_inbound.pl
 # This script is responsible for assigning transferred and new emails to 
@@ -27,8 +27,11 @@ $PATHconf =		'/etc/astguiclient.conf';
 # 190216-0810 - Fix for user-group, email-group and campaign allowed/permissions matching issues
 # 191017-2043 - Added filtered routing options
 # 240219-1526 - Added daily_limit in-group parameter
+# 260515-1941 - Added internal logging
 #
 
+$script_name = 'VD_email_inbound.pl';
+
 if ($ARGV[0]=~/--help/) 
 	{
 	print "VD_email_inbound.pl\n";
@@ -107,6 +110,8 @@ $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VA
 $dbhB = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
     or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER';   $server_ip=$VARserver_ip;   &internal_logger;
+
 ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
 $year = ($year + 1900);
 $mon++;
@@ -155,7 +160,7 @@ $BDtsSQLdate = "$Byear$Bmon$Bmday$Bhour$Bmin$Bsec";
 #$vig_rslt=$dbhA->prepare($vig_stmt);
 #$vig_rslt->execute();
 
-
+$iLog_ct=0;
 for ($i=0; $i<=1000000; $i++) 
 	{
 	$vci=0;
@@ -233,6 +238,16 @@ for ($i=0; $i<=1000000; $i++)
 
 		if ($email_ct>0) {AssignAgents();} else {if ($ARGV[0]=~/debug/i) {print "**** NO EMAILS TO QUEUE ****\n";}}
 		}
+
+	# update internal process log
+	$iLog_ct++;
+	if ($iLog_ct =~ /000$/) 
+		{
+		$stmtA = "UPDATE vicidial_internal_log SET up_time=NOW(), action='running', stage='Loops: $iLog_ct' WHERE process='$script_name' and server_ip='$server_ip' order by db_time desc limit 1;";
+		if($DB){print STDERR "|$stmtA|";}
+		my $affected_rows = $dbhA->do($stmtA);
+		if($DB){print STDERR "$affected_rows|\n";}
+		}
 	usleep(100000);
 	}
 
@@ -407,3 +422,11 @@ sub mysql_error_logging
 		}
 	$one_mysql_log=0;
 	}
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$server_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/bin/start_asterisk_boot.pl

@@ -6,7 +6,7 @@
 # operate as intended
 #
 #
-# Copyright (C) 2013  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
+# Copyright (C) 2026  Matt Florell <vicidial@gmail.com>    LICENSE: AGPLv2
 #
 # CHANGES
 #
@@ -16,11 +16,13 @@
 # 90506-1443 - mikec - Added the T option to the asterisk command. This enables timestamping.
 # 91210-1500 - mattf - Added datetimestamp to astshell screen
 # 131022-1659 - mattf - Added CLI help display
+# 260515-1951 - Added internal logging
 #
 
-$build = '131022-1659';
+$build = '260515-1951';
 # default path to astguiclient configuration file:
 $PATHconf =		'/etc/astguiclient.conf';
+$script_name = 'start_asterisk_boot.pl';
 
 open(conf, "$PATHconf") || die "can't open $PATHconf: $!\n";
 @conf = <conf>;
@@ -105,6 +107,8 @@ use DBI;
 $dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
  or die "Couldn't connect to database: " . DBI->errstr;
 
+$action='start';   $stage='LOGGED INTO MYSQL SERVER '.$build;   &internal_logger;
+
 ### Grab Server values from the database
 $stmtA = "SELECT vd_server_logs FROM servers where server_ip = '$VARserver_ip';";
 $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr;
@@ -172,3 +176,12 @@ $affected_rows = $dbhA->do($stmtA);
 
 
 exit;
+
+
+sub internal_logger
+	{
+	$stmtA = "INSERT INTO vicidial_internal_log SET db_time=NOW(), up_time=NOW(), process='$script_name', server_ip='$VARserver_ip', action='$action', stage='$stage';";
+	if($DB){print STDERR "|$stmtA|";}
+	my $affected_rows = $dbhA->do($stmtA);
+	if($DB){print STDERR "$affected_rows|\n";}
+	}

agc_2-X/trunk/extras/MySQL_AST_CREATE_tables.sql

@@ -5479,6 +5479,16 @@ index (event_date),
 index (status)
 ) ENGINE=MyISAM;
 
+CREATE TABLE vicidial_internal_log (
+db_time DATETIME NOT NULL,
+up_time DATETIME NOT NULL,
+server_ip VARCHAR(15) NOT NULL,
+process VARCHAR(100) default '',
+action VARCHAR(100) default '',
+stage VARCHAR(255) default '',
+KEY intlog_dbtime_key (up_time)
+) ENGINE=MyISAM CHARSET=utf8 COLLATE=utf8_unicode_ci;
+
 
 ALTER TABLE vicidial_email_list MODIFY message text character set utf8;
 
@@ -5899,4 +5909,4 @@ INSERT INTO `wallboard_reports` VALUES ('AGENTS_AND_QUEUES','Agents and Queues',
 
 UPDATE system_settings set vdc_agent_api_active='1';
 
-UPDATE system_settings SET db_schema_version='1741',db_schema_update_date=NOW(),reload_timestamp=NOW();
+UPDATE system_settings SET db_schema_version='1742',db_schema_update_date=NOW(),reload_timestamp=NOW();

agc_2-X/trunk/extras/upgrade_2.14.sql

@@ -3092,3 +3092,15 @@ UPDATE system_settings SET db_schema_version='1740',db_schema_update_date=NOW()
 ALTER TABLE vicidial_users ADD modify_settings_containers ENUM('0','1','2','3','4','5','6') default '0';
 
 UPDATE system_settings SET db_schema_version='1741',db_schema_update_date=NOW() where db_schema_version < 1741;
+
+CREATE TABLE vicidial_internal_log (
+db_time DATETIME NOT NULL,
+up_time DATETIME NOT NULL,
+server_ip VARCHAR(15) NOT NULL,
+process VARCHAR(100) default '',
+action VARCHAR(100) default '',
+stage VARCHAR(255) default '',
+KEY intlog_dbtime_key (up_time)
+) ENGINE=MyISAM CHARSET=utf8 COLLATE=utf8_unicode_ci;
+
+UPDATE system_settings SET db_schema_version='1742',db_schema_update_date=NOW() where db_schema_version < 1742;

agc_2-X/trunk/www/vicidial/admin.php

@@ -3836,7 +3836,6 @@ if ($non_latin < 1)
 	$PHP_AUTH_USER = preg_replace('/[^-_0-9a-zA-Z]/','',$PHP_AUTH_USER);
 	$protocol = preg_replace('/[^-_0-9a-zA-Z]/','',$protocol);
 	$server_id = preg_replace('/[^-_0-9a-zA-Z]/','',$server_id);
-	$stage = preg_replace('/[^-_0-9a-zA-Z]/','',$stage);
 	$state_rule = preg_replace('/[^-_0-9a-zA-Z]/','',$state_rule);
 	$holiday_rule = preg_replace('/[^-_0-9a-zA-Z]/','',$holiday_rule);
 	$trunk_restriction = preg_replace('/[^-_0-9a-zA-Z]/','',$trunk_restriction);
@@ -4201,6 +4200,7 @@ if ($non_latin < 1)
 	$agent_hangup_value = preg_replace('/[^-\/\|\._0-9a-zA-Z]/','',$agent_hangup_value);
 	$second_alert_filename = preg_replace('/[^-\/\|\._0-9a-zA-Z]/','',$second_alert_filename);
 	$third_alert_filename = preg_replace('/[^-\/\|\._0-9a-zA-Z]/','',$third_alert_filename);
+	$stage = preg_replace('/[^-\._0-9a-zA-Z]/','',$stage);
 
 	### ALPHA-NUMERIC and underscore and dash and slash and dot and comma
 	$menu_prompt = preg_replace('/[^-\/\|\,\._0-9a-zA-Z]/','',$menu_prompt);
@@ -4586,7 +4586,6 @@ else
 	$PHP_AUTH_USER = preg_replace('/[^-_0-9\p{L}]/u','',$PHP_AUTH_USER);
 	$protocol = preg_replace('/[^-_0-9\p{L}]/u','',$protocol);
 	$server_id = preg_replace('/[^-_0-9\p{L}]/u','',$server_id);
-	$stage = preg_replace('/[^-_0-9\p{L}]/u','',$stage);
 	$state_rule = preg_replace('/[^-_0-9\p{L}]/u','',$state_rule);
 	$holiday_rule = preg_replace('/[^-_0-9\p{L}]/u','',$holiday_rule);
 	$trunk_restriction = preg_replace('/[^-_0-9\p{L}]/u','',$trunk_restriction);
@@ -4948,6 +4947,7 @@ else
 	$agent_hangup_value = preg_replace('/[^-\/\|\._0-9\p{L}]/u','',$agent_hangup_value);
 	$second_alert_filename = preg_replace('/[^-\/\|\._0-9\p{L}]/u','',$second_alert_filename);
 	$third_alert_filename = preg_replace('/[^-\/\|\._0-9\p{L}]/u','',$third_alert_filename);
+	$stage = preg_replace('/[^-\._0-9\p{L}]/u','',$stage);
 
 	### ALPHA-NUMERIC and underscore and dash and slash and dot and comma
 	$menu_prompt = preg_replace('/[^-\/\|\,\._0-9\p{L}]/u','',$menu_prompt);
@@ -6337,12 +6337,13 @@ if ($SSscript_remove_js > 0)
 # 260408-0827 - Added max_inbound_auto_reenable system setting and function to re-enable agent in-group selections after max_inbound_calls has been raised
 # 260410-1953 - Fix for max_inbound_auto_reenable feature in 4B admin screen
 # 260416-0909 - Added modify_settings_containers user setting
+# 260516-2350 - Added Internal Process Logs display page
 #
 
 # make sure you have added a user to the vicidial_users MySQL table with at least user_level 9 to access this page the first time
 
-$admin_version = '2.14-957a';
+$admin_version = '2.14-958a';
-$build = '260416-0909';
+$build = '260516-2350';
 
 $STARTtime = date("U");
 $SQLdate = date("Y-m-d H:i:s");
@@ -7421,6 +7422,7 @@ if ($ADD==999986)		{$hh='reports';		echo _QXZ("POSTAL CODES");}
 if ($ADD==999985)		{$hh='reports';		echo _QXZ("POSTAL CODES CITIES");}
 if ($ADD==999984)		{$hh='reports';		echo _QXZ("CRASHED DATABASE TABLES");}
 if ($ADD==999983)		{$hh='reports';		echo _QXZ("SERVER DRIVE PARTITIONS");}
+if ($ADD==999982)		{$hh='reports';		echo _QXZ("INTERNAL PROCESS LOGS");}
 
 echo "</title>\n";
 
@@ -52080,7 +52082,7 @@ if ($ADD==999994)
 		echo "<UL>\n";
 	#	echo "<LI><a href=\"welcome_languages.php\"><FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK SIZE=2>"._QXZ("Welcome Languages Page")."</a></FONT>\n";
 		echo "<LI><a href=\"help.php\"><FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK SIZE=2>"._QXZ("Old Help Page")."</a></FONT>\n";
-		echo "<LI><a href=\"$PHP_SELF?ADD=999991\"><FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK SIZE=2>"._QXZ("Servers Versions")."</a></FONT>\n";
+		echo "<LI><a href=\"$PHP_SELF?ADD=999991\"><FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK SIZE=2>"._QXZ("Servers Versions")."</a></FONT> | <a href=\"$PHP_SELF?ADD=999982\"><FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK SIZE=2>"._QXZ("Internal Process Logs")."</a></FONT>\n";
 		if ( (preg_match("/VERM Reports/",$LOGallowed_reports)) or (preg_match("/ALL REPORTS/",$LOGallowed_reports)) )
 			{echo "<LI><a href=\"../VERM/VERM_admin.php\"><FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK size=2>"._QXZ("VERM - Enhanced Reporting Module")."</a></FONT>\n";}
 		echo "<BR><BR>\n";
@@ -53465,6 +53467,274 @@ if ($ADD==999983)
 ##### END SERVER DRIVE PARTITIONS display page #####
 
 
+######################
+# ADD=999982 - INTERNAL PROCESS LOG display page
+######################
+if ($ADD==999982)
+	{
+	$servers=array();
+	$process_ct=array();
+	$last_time=array();
+	$last_process=array();
+	$last_action=array();
+	$last_stage=array();
+	$last_length=array();
+	$server_id=array();
+	$system_uptime=array();
+	$serverSQL='';
+	$serverURL='';
+	$backlink='';
+	$SERVERbacklink='';
+	if (strlen($query_date) < 4) {$query_date = 'today';}
+	$one_day_ago=date("Y-m-d", time()-(1*86400));
+	$two_days_ago=date("Y-m-d", time()-(2*86400));
+	$three_days_ago=date("Y-m-d", time()-(3*86400));
+	$four_days_ago=date("Y-m-d", time()-(4*86400));
+	$five_days_ago=date("Y-m-d", time()-(5*86400));
+	$six_days_ago=date("Y-m-d", time()-(6*86400));
+	if (strlen($server_ip) > 6)
+		{
+		$serverSQL = "and server_ip='$server_ip'";
+		$serverURL = "&server_ip=$server_ip";
+		$backlink = ""._QXZ(" FOR THIS SERVER").": $server_ip - <a href=\"$PHP_SELF?ADD=999982\">back</a>";
+		$SERVERbacklink = " - <a href=\"$PHP_SELF?ADD=999982$serverURL&query_date=$query_date\">back</a>";
+		}
+
+	$query_dateSQL = "up_time >= TIMESTAMP(CURDATE())";
+	if ($query_date == 'today') {$header_day = 'TODAY';}
+	if ($query_date == 'yesterday') {$header_day = 'YESTERDAY';   $query_dateSQL = "up_time >= \"$one_day_ago 00:00:00\" and up_time <= \"$one_day_ago 23:59:59\"";}
+	if ($query_date == $two_days_ago) {$header_day = $two_days_ago;   $query_dateSQL = "up_time >= \"$two_days_ago 00:00:00\" and up_time <= \"$two_days_ago 23:59:59\"";}
+	if ($query_date == $three_days_ago) {$header_day = $three_days_ago;   $query_dateSQL = "up_time >= \"$three_days_ago 00:00:00\" and up_time <= \"$three_days_ago 23:59:59\"";}
+	if ($query_date == $four_days_ago) {$header_day = $four_days_ago;   $query_dateSQL = "up_time >= \"$four_days_ago 00:00:00\" and up_time <= \"$four_days_ago 23:59:59\"";}
+	if ($query_date == $five_days_ago) {$header_day = $five_days_ago;   $query_dateSQL = "up_time >= \"$five_days_ago 00:00:00\" and up_time <= \"$five_days_ago 23:59:59\"";}
+	if ($query_date == $six_days_ago) {$header_day = $six_days_ago;   $query_dateSQL = "up_time >= \"$six_days_ago 00:00:00\" and up_time <= \"$six_days_ago 23:59:59\"";}
+
+	$internal_rows='';
+	$stmt="SELECT count(*),server_ip from vicidial_internal_log WHERE $query_dateSQL $serverSQL group by server_ip order by server_ip;";
+	$rslt=mysql_to_mysqli($stmt, $link);
+	$intlogs_to_print = mysqli_num_rows($rslt);
+	if ($DB > 0) {echo "DEBUG: $intlogs_to_print|$stmt|\n";}
+	$o=0;
+	while ($intlogs_to_print > $o) 
+		{
+		$rowx=mysqli_fetch_row($rslt);
+		$process_ct[$o] =	$rowx[0];
+		$servers[$o] =		$rowx[1];
+		$o++;
+		}
+	$row_color=0;
+	$o=0;   $o_ct=1;
+	while ($intlogs_to_print > $o) 
+		{
+		$stmt="SELECT up_time,process,action,stage,(UNIX_TIMESTAMP(up_time)-UNIX_TIMESTAMP(db_time)) from vicidial_internal_log WHERE $query_dateSQL and server_ip='$servers[$o]' order by up_time desc limit 1;";
+		$rslt=mysql_to_mysqli($stmt, $link);
+		$details_to_print = mysqli_num_rows($rslt);
+		if ($details_to_print > 0) 
+			{
+			$rowy=mysqli_fetch_row($rslt);
+			$last_time[$o] =	$rowy[0];
+			$last_process[$o] = $rowy[1];
+			$last_action[$o] =	$rowy[2];
+			$last_stage[$o] =	$rowy[3];
+			$last_length[$o] =	$rowy[4];
+			if ($last_length[$o] > 0) {$last_length[$o] = gmdate("H:i:s", $last_length[$o]);}
+			}
+		$stmt="SELECT server_id,system_uptime from servers WHERE server_ip='$servers[$o]' limit 1;";
+		$rslt=mysql_to_mysqli($stmt, $link);
+		$details_to_print = mysqli_num_rows($rslt);
+		if ($details_to_print > 0) 
+			{
+			$rowy=mysqli_fetch_row($rslt);
+			$server_id[$o] =		$rowy[0];
+			$system_uptime[$o] =	$rowy[1];
+			}
+		if (preg_match('/1$|3$|5$|7$|9$/i', $row_color))
+			{$bgcolor='bgcolor="#'. $SSstd_row2_background .'"';} 
+		else
+			{$bgcolor='bgcolor="#'. $SSstd_row1_background .'"';}
+		$intlog_rows .= "<tr $bgcolor>\n";
+		$intlog_rows .= "<td align=center><font size=1>$o_ct</font></td>\n";
+		$intlog_rows .= "<td align=left><font size=2> &nbsp; <a href=\"$PHP_SELF?ADD=999982&server_ip=$servers[$o]&query_date=$query_date\"><font color=black>$servers[$o]</font></a> - $server_id[$o]</font></td>\n";
+		$intlog_rows .= "<td align=right><font size=2> &nbsp; $system_uptime[$o]</font></td>\n";
+		$intlog_rows .= "<td align=left><font size=2> &nbsp; $process_ct[$o]</font></td>\n";
+		$intlog_rows .= "<td align=left><font size=2> &nbsp; $last_process[$o]</font></td>\n";
+		$intlog_rows .= "<td align=center><font size=2>$last_time[$o]</font></td>\n";
+		$intlog_rows .= "<td align=right><font size=2>$last_length[$o] &nbsp; </font></td>\n";
+		$intlog_rows .= "<td align=center><font size=2>$last_action[$o]</font></td>\n";
+		$intlog_rows .= "<td align=left><font size=2> &nbsp; $last_stage[$o]</font></td>\n";
+		$intlog_rows .= "</tr>\n";
+		$o++;   $o_ct++;
+		}
+
+	if ($o < 1)
+		{
+		echo "<CENTER><BR><BR><FONT FACE=\"ARIAL,HELVETICA\" COLOR=RED SIZE=2>"._QXZ("There are no internal log entries on your system for this day")."</FONT><BR><BR></CENTER>\n";
+		}
+
+	echo "<TABLE><TR><TD>\n";
+	echo "<FONT FACE=\"ARIAL,HELVETICA\" COLOR=BLACK SIZE=2>";
+
+	echo "<br>"._QXZ("INTERNAL PROCESS LOG ENTRIES FOR ")."$header_day $backlink\n";
+	echo "<center><TABLE width=1050 cellspacing=3>\n";
+	echo "<tr bgcolor=#$SSstd_row3_background>\n";
+	echo "<td align=center><B> # </B></td>\n";
+	echo "<td align=left><font size=2><B> &nbsp; "._QXZ("Server IP")."</B></td>\n";
+	echo "<td align=center><font size=2><B> &nbsp; "._QXZ("uptime")."</B></td>\n";
+	echo "<td align=left><font size=2><B> &nbsp; "._QXZ("L ct")."</B></td>\n";
+	echo "<td align=left><font size=2><B> &nbsp; "._QXZ("Last Process")."</B></td>\n";
+	echo "<td align=center><font size=2><B>"._QXZ("Last Date-Time")."</B></td>\n";
+	echo "<td align=right><font size=2><B>"._QXZ("Last Length")."</B> &nbsp; </td>\n";
+	echo "<td align=center><font size=2><B>"._QXZ("Last Action")."</B></td>\n";
+	echo "<td align=left><font size=2><B> &nbsp; "._QXZ("Last Notes")."</B></td>\n";
+	echo "</tr>\n";
+	echo "$intlog_rows";
+	echo "</TABLE>\n";
+
+	if (strlen($server_ip) > 6)
+		{
+		$sum_process_ct=array();
+		$sum_process=array();
+		$sum_last_time=array();
+		$sum_last_action=array();
+		$sum_last_stage=array();
+		$sum_last_length=array();
+		$proclog_rows='';
+		$stmt="SELECT count(*),process from vicidial_internal_log WHERE $query_dateSQL $serverSQL group by process order by process;";
+		$rslt=mysql_to_mysqli($stmt, $link);
+		$proclogs_to_print = mysqli_num_rows($rslt);
+		if ($DB > 0) {echo "DEBUG: $proclogs_to_print|$stmt|\n";}
+		$o=0;
+		while ($proclogs_to_print > $o) 
+			{
+			$rowx=mysqli_fetch_row($rslt);
+			$sum_process_ct[$o] =	$rowx[0];
+			$sum_process[$o] =		$rowx[1];
+			$o++;
+			}
+		$row_color=0;
+		$o=0;   $o_ct=1;
+		while ($proclogs_to_print > $o) 
+			{
+			$stmt="SELECT up_time,action,stage,(UNIX_TIMESTAMP(up_time)-UNIX_TIMESTAMP(db_time)) from vicidial_internal_log WHERE $query_dateSQL and process='$sum_process[$o]' $serverSQL order by up_time desc limit 1;";
+			$rslt=mysql_to_mysqli($stmt, $link);
+			$details_to_print = mysqli_num_rows($rslt);
+			if ($details_to_print > 0) 
+				{
+				$rowy=mysqli_fetch_row($rslt);
+				$sum_last_time[$o] =	$rowy[0];
+				$sum_last_action[$o] =	$rowy[1];
+				$sum_last_stage[$o] =	$rowy[2];
+				$sum_last_length[$o] =	$rowy[3];
+				if ($sum_last_length[$o] > 0) {$sum_last_length[$o] = gmdate("H:i:s", $sum_last_length[$o]);}
+				}
+			if (preg_match('/1$|3$|5$|7$|9$/i', $row_color))
+				{$bgcolor='bgcolor="#'. $SSstd_row2_background .'"';} 
+			else
+				{$bgcolor='bgcolor="#'. $SSstd_row1_background .'"';}
+			$proclog_rows .= "<tr $bgcolor>\n";
+			$proclog_rows .= "<td align=center><font size=1>$o_ct</font></td>\n";
+			$proclog_rows .= "<td align=left><font size=2> &nbsp; <a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$sum_process[$o]&query_date=$query_date\"><font color=black>$sum_process[$o]</font></a></font></td>\n";
+			$proclog_rows .= "<td align=left><font size=2> &nbsp; $sum_process_ct[$o]</font></td>\n";
+			$proclog_rows .= "<td align=center><font size=2>$sum_last_time[$o]</font></td>\n";
+			$proclog_rows .= "<td align=right><font size=2>$sum_last_length[$o] &nbsp; </font></td>\n";
+			$proclog_rows .= "<td align=left><font size=2> &nbsp; $sum_last_action[$o]</font></td>\n";
+			$proclog_rows .= "<td align=left><font size=2> &nbsp; $sum_last_stage[$o]</font></td>\n";
+			$proclog_rows .= "</tr>\n";
+			$o++;   $o_ct++;
+			}
+
+		echo "<br>"._QXZ("SUMMARY PROCESS LOG ENTRIES FOR THIS SERVER").": $server_ip $SERVERbacklink\n";
+		echo "<center><TABLE width=1050 cellspacing=3>\n";
+		echo "<tr bgcolor=#$SSstd_row3_background>\n";
+		echo "<td align=center><B> # </B></td>\n";
+		echo "<td align=left><font size=2><B> &nbsp; "._QXZ("Process")."</B></td>\n";
+		echo "<td align=left><font size=2><B> &nbsp; "._QXZ("Log ct")."</B></td>\n";
+		echo "<td align=center><font size=2><B>"._QXZ("Last Date-Time")."</B></td>\n";
+		echo "<td align=right><font size=2><B>"._QXZ("Last Length")."</B> &nbsp; </td>\n";
+		echo "<td align=left><font size=2> &nbsp; <B>"._QXZ("Last Action")."</B></td>\n";
+		echo "<td align=left><font size=2> &nbsp; <B>"._QXZ("Last Notes")."</B></td>\n";
+		echo "</tr>\n";
+		echo "$proclog_rows";
+		echo "</TABLE>\n";
+
+		if (strlen($stage) > 0)
+			{
+			$dt_time=array();
+			$dt_action=array();
+			$dt_stage=array();
+			$dt_length=array();
+			$dt_begin_time=array();
+			$dtlog_rows='';
+			$stmt="SELECT up_time,action,stage,(UNIX_TIMESTAMP(up_time)-UNIX_TIMESTAMP(db_time)),db_time from vicidial_internal_log WHERE $query_dateSQL and process='$stage' $serverSQL order by up_time desc limit 10000;";
+			$rslt=mysql_to_mysqli($stmt, $link);
+			$dtlogs_to_print = mysqli_num_rows($rslt);
+			if ($DB > 0) {echo "DEBUG: $dtlogs_to_print|$stmt|\n";}
+			$o=0;
+			while ($dtlogs_to_print > $o) 
+				{
+				$rowz=mysqli_fetch_row($rslt);
+				$dt_time[$o] =		$rowz[0];
+				$dt_action[$o] =	$rowz[1];
+				$dt_stage[$o] =		$rowz[2];
+				$dt_length[$o] =	$rowz[3];
+				if ($dt_length[$o] > 0) {$dt_length[$o] = gmdate("H:i:s", $dt_length[$o]);}
+				$dt_begin_time[$o] =	$rowz[4];
+				$o++;
+				}
+			$row_color=0;
+			$o=0;   $o_ct=1;
+			while ($dtlogs_to_print > $o) 
+				{
+				if (preg_match('/1$|3$|5$|7$|9$/i', $row_color))
+					{$bgcolor='bgcolor="#'. $SSstd_row2_background .'"';} 
+				else
+					{$bgcolor='bgcolor="#'. $SSstd_row1_background .'"';}
+				$dtlog_rows .= "<tr $bgcolor>\n";
+				$dtlog_rows .= "<td align=center><font size=1>$o_ct</font></td>\n";
+				$dtlog_rows .= "<td align=left><font size=2> &nbsp; $dt_begin_time[$o] to $dt_time[$o]</font></td>\n";
+				$dtlog_rows .= "<td align=right><font size=2>$dt_length[$o] &nbsp; </font></td>\n";
+				$dtlog_rows .= "<td align=center><font size=2>$dt_action[$o]</font></td>\n";
+				$dtlog_rows .= "<td align=left><font size=2> &nbsp; $dt_stage[$o]</font></td>\n";
+				$dtlog_rows .= "</tr>\n";
+				$o++;   $o_ct++;
+				}
+
+			echo "<br>"._QXZ("DETAIL PROCESS LOG ENTRIES FOR THIS SERVER PROCESS").": $server_ip - $stage\n";
+			echo "<center><TABLE width=1050 cellspacing=3>\n";
+			echo "<tr bgcolor=#$SSstd_row3_background>\n";
+			echo "<td align=center><B> # </B></td>\n";
+			echo "<td align=left><font size=2><B> &nbsp; "._QXZ("Date Time")."</B></td>\n";
+			echo "<td align=right><font size=2><B>"._QXZ("Length")."</B> &nbsp; </td>\n";
+			echo "<td align=center><font size=2><B>"._QXZ("Action")."</B></td>\n";
+			echo "<td align=center><font size=2><B>"._QXZ("Notes")."</B></td>\n";
+			echo "</tr>\n";
+			echo "$dtlog_rows";
+			echo "</TABLE>\n";
+			}
+		}
+
+	# alternate-day display options
+	if ($query_date == 'today') {$today_link = 'TODAY | ';  $header_day = 'TODAY';}
+	else {$today_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=today\"><font size=2 color=black>TODAY</a> | ";}
+	if ($query_date == 'yesterday') {$yesterday_link = 'YESTERDAY | ';  $header_day = 'YESTERDAY';}
+	else {$yesterday_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=yesterday\"><font size=2 color=black>YESTERDAY</a> | ";}
+	if ($query_date == $two_days_ago) {$two_days_link = "$two_days_ago | ";  $header_day = $two_days_ago;}
+	else {$two_days_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=$two_days_ago\"><font size=2 color=black>$two_days_ago</a> | ";}
+	if ($query_date == $three_days_ago) {$three_days_link = "$three_days_ago | ";  $header_day = $three_days_ago;}
+	else {$three_days_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=$three_days_ago\"><font size=2 color=black>$three_days_ago</a> | ";}
+	if ($query_date == $four_days_ago) {$four_days_link = "$four_days_ago | ";  $header_day = $four_days_ago;}
+	else {$four_days_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=$four_days_ago\"><font size=2 color=black>$four_days_ago</a> | ";}
+	if ($query_date == $five_days_ago) {$five_days_link = "$five_days_ago | ";  $header_day = $five_days_ago;}
+	else {$five_days_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=$five_days_ago\"><font size=2 color=black>$five_days_ago</a> | ";}
+	if ($query_date == $six_days_ago) {$six_days_link = "$six_days_ago";  $header_day = $six_days_ago;}
+	else {$six_days_link = "<a href=\"$PHP_SELF?ADD=999982$serverURL&stage=$stage&query_date=$six_days_ago\"><font size=2 color=black>$six_days_ago</a>";}
+
+	echo "<BR>$today_link$yesterday_link$two_days_link$three_days_link$four_days_link$five_days_link$six_days_link\n";
+	echo "</center>\n";
+	}
+##### END INTERNAL PROCESS LOG display page #####
+
+
 echo "</TD></TR></TABLE></center>\n";
 echo "</TD></TR></TABLE></center>\n";