From 36ed729122452ef61c0728686fa5349891965d43 Mon Sep 17 00:00:00 2001 From: mattf Date: Wed, 28 Sep 2011 11:37:13 +0000 Subject: [PATCH] fixed sheet2tab.pl for XL issues changed 851 playback with waitforsilence dialplan entries small changes to VD_amd.agi script for CPD git-svn-id: svn://192.168.202.10@1729 3d104415-ff17-0410-8863-d5cf3c621b8a --- agc_2-X/trunk/agi/VD_amd.agi | 34 +- .../docs/conf_examples/extensions.conf.sample | 13 +- .../conf_examples/extensions.conf.sample-1.4 | 13 +- agc_2-X/trunk/www/vicidial/sheet2tab.pl | 304 ++++++++++++++++-- 4 files changed, 312 insertions(+), 52 deletions(-) diff --git a/agc_2-X/trunk/agi/VD_amd.agi b/agc_2-X/trunk/agi/VD_amd.agi index ff00ea4e..2697f276 100644 --- a/agc_2-X/trunk/agi/VD_amd.agi +++ b/agc_2-X/trunk/agi/VD_amd.agi @@ -428,7 +428,7 @@ else {$sthA->finish();} } - ### if not a CPD message, log to the extended log and check for amd recording setting + ### if not a CPD message, log to the extended log if ($AMDSTATUS !~ /CPD/) { $VLEcount=0; @@ -451,21 +451,27 @@ else if ($AGILOG) {$agi_string = "|$stmtA|"; &agi_output;} if ($AGILOG) {$agi_string = "-- VDAD vicidial_log_extended insert: |$affected_rows|\n|$stmtA|"; &agi_output;} } + } - if ($survey_recording =~ /Y_WITH_AMD/) + if ($survey_recording =~ /Y_WITH_AMD/) + { + $RLcount=0; + if ($AMDSTATUS =~ /CPD/) { - $stmtA = "SELECT vendor_lead_code FROM vicidial_list where lead_id='$VD_lead_id';"; + $stmtA = "SELECT count(*) FROM recording_log where vicidial_id='$uniqueid' and lead_id='$VD_lead_id' and user='VDAD';"; $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; $sthArows=$sthA->rows; - if ($AGILOG) {$agi_string = "|$stmtA|"; &agi_output;} if ($sthArows > 0) { @aryA = $sthA->fetchrow_array; - $vendor_lead_code = $aryA[0]; + $RLcount = $aryA[0]; } $sthA->finish(); - + if ($AGILOG) {$agi_string = "$RLcount|$stmtA|"; &agi_output;} + } + if ($RLcount < 1) + { # get date/time ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); $year = ($year + 1900); @@ -482,6 +488,21 @@ else $recdate = "$year$mon$mday-$hour$min$sec"; $tinydate = "$Tyear$mon$mday$hour$min$sec"; + if ($campaign_rec_filename =~ /VENDORLEADCODE/) + { + $stmtA = "SELECT vendor_lead_code FROM vicidial_list where lead_id='$VD_lead_id';"; + $sthA = $dbhA->prepare($stmtA) or die "preparing: ",$dbhA->errstr; + $sthA->execute or die "executing: $stmtA ", $dbhA->errstr; + $sthArows=$sthA->rows; + if ($AGILOG) {$agi_string = "|$stmtA|"; &agi_output;} + if ($sthArows > 0) + { + @aryA = $sthA->fetchrow_array; + $vendor_lead_code = $aryA[0]; + } + $sthA->finish(); + } + $campaign_rec_filename =~ s/CAMPAIGN/$VD_campaign_id/gi; $campaign_rec_filename =~ s/CUSTPHONE/$VD_phone_number/gi; $campaign_rec_filename =~ s/FULLDATE/$recdate/gi; @@ -502,7 +523,6 @@ else } } - ### check for list_id override if ( ($DB_amd_send_to_vmx =~ /Y/) || ($AMDSTATUS =~ /CPD/) ) { diff --git a/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample b/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample index b947ed91..e15434c4 100644 --- a/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample +++ b/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample @@ -54,12 +54,13 @@ exten => _851XXXXX,1,Answer exten => _851XXXXX,2,Playback(${EXTEN}) exten => _851XXXXX,3,Hangup -; this is used for playing a message to an answering machine forwarded from AMD in VICIDIAL -exten => _7851XXXXX,1,WaitForSilence(2000,2) ; AMD got machine. leave message after recording -exten => _7851XXXXX,2,Playback(${EXTEN:1}) -exten => _7851XXXXX,3,AGI(VD_amd_post.agi,${EXTEN:1}) -exten => _7851XXXXX,4,Hangup - +; this is used for playing a phone-recorded 851XXXXX audio file with waitforsilence +exten => _6851XXXXX,1,Answer() +exten => _6851XXXXX,n,WaitForSilence(2000,1,90) +exten => _6851XXXXX,n,Playback(sip-silence) +exten => _6851XXXXX,n,Playback(${EXTEN:1}) +exten => _6851XXXXX,n,Hangup +exten => _7851XXXXX,1,Goto(default,6${EXTEN:1},1) ; FastAGI for VICIDIAL/astGUIclient call logging exten => h,1,DeadAGI(agi://127.0.0.1:4577/call_log--HVcauses--PRI-----NODEBUG-----${HANGUPCAUSE}-----${DIALSTATUS}-----${DIALEDTIME}-----${ANSWEREDTIME}) diff --git a/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample-1.4 b/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample-1.4 index aafe4489..32838b8f 100644 --- a/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample-1.4 +++ b/agc_2-X/trunk/docs/conf_examples/extensions.conf.sample-1.4 @@ -58,12 +58,13 @@ exten => _851XXXXX,1,Answer exten => _851XXXXX,2,Playback(${EXTEN}) exten => _851XXXXX,3,Hangup -; this is used for playing a message to an answering machine forwarded from AMD in VICIDIAL -exten => _7851XXXXX,1,WaitForSilence(2000,2) ; AMD got machine. leave message after recording -exten => _7851XXXXX,2,Playback(${EXTEN:1}) -exten => _7851XXXXX,3,AGI(VD_amd_post.agi,${EXTEN:1}) -exten => _7851XXXXX,4,Hangup - +; this is used for playing a phone-recorded 851XXXXX audio file with waitforsilence +exten => _6851XXXXX,1,Answer() +exten => _6851XXXXX,n,WaitForSilence(2000,1,90) +exten => _6851XXXXX,n,Playback(sip-silence) +exten => _6851XXXXX,n,Playback(${EXTEN:1}) +exten => _6851XXXXX,n,Hangup +exten => _7851XXXXX,1,Goto(default,6${EXTEN:1},1) ; FastAGI for VICIDIAL/astGUIclient call logging exten => h,1,DeadAGI(agi://127.0.0.1:4577/call_log--HVcauses--PRI-----NODEBUG-----${HANGUPCAUSE}-----${DIALSTATUS}-----${DIALEDTIME}-----${ANSWEREDTIME}) diff --git a/agc_2-X/trunk/www/vicidial/sheet2tab.pl b/agc_2-X/trunk/www/vicidial/sheet2tab.pl index 9be16dba..6f282b35 100644 --- a/agc_2-X/trunk/www/vicidial/sheet2tab.pl +++ b/agc_2-X/trunk/www/vicidial/sheet2tab.pl @@ -2,7 +2,7 @@ # # sheet2tab.pl - Convert spreadsheet to tab-delimited text file version 2.4 # -# Copyright (C) 2010 Matt Florell & Michael Cargile LICENSE: AGPLv2 +# Copyright (C) 2011 Matt Florell & Michael Cargile LICENSE: AGPLv2 # # Lead file conversion and scrubbing script. This is the first stage in the lead loading process. # @@ -26,13 +26,17 @@ # CHANGES # 100706-0833 - Initial build # 100706-1244 - Reformat and add comments +# 110927-1750 - Fixed issue with improperly CSV files locking up servers # # disable when not debugging +#use 5.010; #use strict; #use warnings; use Spreadsheet::Read; +use File::Basename; + sub scrub_lead_field { @@ -50,6 +54,12 @@ sub scrub_lead_field my $infile; my $outfile; +my $error_log = "sheet2tab_error.txt"; + +my $csv_chuck_size = 500; + +my $crap_loop_time = 5; + if ( $#ARGV == 1 ) { $infile = $ARGV[0]; @@ -72,52 +82,280 @@ my $count = 0; my $colPos = 0; my $rowPos = 0; -# parse the csv file -my $parser = ReadData ( "$infile" ); +my $time = time(); -my $maxCol = $parser->[1]{maxcol}; -my $maxRow = $parser->[1]{maxrow}; +my $tempfile = "sheet2tab_temp_file_$time.csv"; -if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; -if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; +# file types we are looking for +my @exts = qw(.csv); -# loop through the rows -for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) - { - # loop through the cols - for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) - { - my $cell = cr2cell( $colPos, $rowPos ); +# dont screw up the actual infile variable +my $exten_file = $infile; +chomp $exten_file; - if ($debug) { print STDERR "cell = '$cell'\n"; }; +# break the file up into its parts +my ($dir, $name, $ext) = fileparse($exten_file, @exts); - my $field; +given ($ext) { + # if we are a csv file + when ('.csv') { + open( IN, $infile ) or die "can't open $infile: $!\n"; + open( TMPFILE , ">$tempfile" ) or die $!; - # make sure the field has a value - if ( $parser->[1]{$cell} ) + my $cur_loop_time = time(); + my $old_loop_time = time(); + my $loop_time = 0; + my $loop_sleep = 0; + + my $loop_count = 0; + + # loop through the file and process it in chunks + while( ) { - $field = $parser->[1]{$cell}; - } - else + $loop_count++; + + print TMPFILE $_; + if ($debug) { print STDERR "csv line = '$_'\n"; }; + + # break + if ( $loop_count % $csv_chuck_size == 0 ) + { + + # close the temp file + close( TMPFILE ); + + # process the temp file + my $parser = ReadData ( "$tempfile" ); + + my $maxCol = $parser->[1]{maxcol}; + my $maxRow = $parser->[1]{maxrow}; + + # Something is not right if they have 0 or over 100 columns in their lead file. + if (( $maxCol >= 100 ) || ( $maxCol == 0 )) + { + print STDERR "ERROR: Improperly formatted lead file.\n"; + print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n"; + exit; + } + + if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; + if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; + + # loop through the rows + for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) + { + # loop through the cols + for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) + { + my $cell = cr2cell( $colPos, $rowPos ); + + if ($debug) { print STDERR "cell = '$cell'\n"; }; + + my $field; + + # make sure the field has a value + if ( $parser->[1]{$cell} ) + { + $field = $parser->[1]{$cell}; + } + else + { + $field = ""; + } + + if ($debug) { print STDERR "field = '$field'\n"; }; + + $field = scrub_lead_field( $field ); + + print OUTFILE $field; + + if ( $colPos < $maxCol ) + { + print OUTFILE $out_delim; + } + else + { + print OUTFILE "\n"; + } + } + } + + + # delete the TMPFILE + unlink( $tempfile ) or die $!; + + # repopn the TMPFILE + open( TMPFILE , ">$tempfile" ) or die $!; + + + # figure out how long it took to loop + $old_loop_time = $cur_loop_time; + $cur_loop_time = time(); + $loop_time = $cur_loop_time - $old_loop_time; + + #print STDERR "loop_count = '$loop_count' $loop_time $loop_sleep\n"; + + if ( $loop_time > $crap_loop_time ) + { + # this should not take this long to run through a loop. + # sleep for a bit to let the CPU recover. + $loop_sleep = $loop_sleep + $loop_time; + + # they have waited 60 seconds lets just kill this and get it over with + if ( $loop_sleep > 60 ) { + close( TMPFILE ); + close( IN ); + + # delete the TMPFILE + unlink( $tempfile ) or die $!; + + open( ERRFILE, ">>$error_log" ); + + print ERRFILE "$cur_loop_time: Sheet2tab.pl aborting. Penalized them long enough for their junk leads in $infile \n\n"; + + close( ERRFILE ); + + exit; + } + + sleep($loop_sleep); + + open( ERRFILE, ">>$error_log" ); + + print ERRFILE "$cur_loop_time: Sheet2tab.pl took $loop_time to process $csv_chuck_size leads from the $infile lead file. Making them sleep $loop_sleep so we can recover.\n\n"; + + close( ERRFILE ); + + # do not penalize the next loop through because we forced them to sleep + $cur_loop_time = time(); + } + } + } + + # close the temp and In files + close( TMPFILE ); + close( IN ); + + # see if we have any left overs + my $temp_file_size = -s $tempfile; + + # if not exit + if ( $temp_file_size == 0 ) { + # delete the TMPFILE + unlink( $tempfile ) or die $!; + exit; + } + + # other wise parse the last temp file + my $parser = ReadData ( "$tempfile" ); + + my $maxCol = $parser->[1]{maxcol}; + my $maxRow = $parser->[1]{maxrow}; + + if (( $maxCol >= 100 ) || ( $maxCol == 0 )) { - $field = ""; + print STDERR "ERROR: Improperly formatted lead file.\n"; + print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n"; + exit; } - if ($debug) { print STDERR "field = '$field'\n"; }; + if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; + if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; - $field = scrub_lead_field( $field ); - - print OUTFILE $field; - - if ( $colPos < $maxCol ) + # loop through the rows + for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) { - print OUTFILE $out_delim; - } - else + # loop through the cols + for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) + { + my $cell = cr2cell( $colPos, $rowPos ); + + if ($debug) { print STDERR "cell = '$cell'\n"; }; + + my $field; + + # make sure the field has a value + if ( $parser->[1]{$cell} ) + { + $field = $parser->[1]{$cell}; + } + else + { + $field = ""; + } + + if ($debug) { print STDERR "field = '$field'\n"; }; + + $field = scrub_lead_field( $field ); + + print OUTFILE $field; + + if ( $colPos < $maxCol ) + { + print OUTFILE $out_delim; + } + else + { + print OUTFILE "\n"; + } + } + } + + # delete the TMPFILE + unlink( $tempfile ) or die $!; + + } + + # not a CSV file + default { + # parse the file + my $parser = ReadData ( "$infile" ); + + my $maxCol = $parser->[1]{maxcol}; + my $maxRow = $parser->[1]{maxrow}; + + + if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; + if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; + + # loop through the rows + for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) { - print OUTFILE "\n"; + # loop through the cols + for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) + { + my $cell = cr2cell( $colPos, $rowPos ); + + if ($debug) { print STDERR "cell = '$cell'\n"; }; + + my $field; + + # make sure the field has a value + if ( $parser->[1]{$cell} ) + { + $field = $parser->[1]{$cell}; + } + else + { + $field = ""; + } + + if ($debug) { print STDERR "field = '$field'\n"; }; + + $field = scrub_lead_field( $field ); + + print OUTFILE $field; + + if ( $colPos < $maxCol ) + { + print OUTFILE $out_delim; + } + else + { + print OUTFILE "\n"; + } + } } } } - exit;