ViciDial/agc_2-X
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed issue with sheet2tab.pl

Browse Source 
Added several new features to the CLI lead loader based around creation of new lists and one new format

git-svn-id: svn://192.168.202.10@1731 3d104415-ff17-0410-8863-d5cf3c621b8a
This commit is contained in:
mattf
2011-10-03 19:56:55 +00:00
parent d7b6cf4ea4
commit ef591374a1
2 changed files with 289 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/VICIDIAL_IN_new_leads_file.pl
+84 -6
View File
@@ -52,9 +52,10 @@
# 110420-0944 - Fixed file prefix issue with multiple processes running
# 110424-0948 - Added time-zone-code-gmt option to use time zone code from the owner field
# 110705-1913 - Added options for USACAN prefix(no 0 or 1) and valid areacode filtering
# 110929-1423 - Added new format for abbreviated timezone in owner(pipe30tz) and list creation options
#
$version = '110705-1913';
$version = '110929-1423';
$secX = time();
$MT[0]='';
@@ -148,8 +149,8 @@ if (length($ARGV[0])>1)
if ($args =~ /--help|-h/i)
{
print "allowed run time options:\n";
print " [-q] = quiet\n";
print " [-t] = test\n";
print " [--quiet] = quiet\n";
print " [--test] = test\n";
print " [--version] = version\n";
print " [--forcegmt] = forces gmt value of column after comments column\n";
print " [--debug] = debug output\n";
@@ -162,6 +163,9 @@ if (length($ARGV[0])>1)
print " [--new-listid-prefix=X] = prefix for listID when creating new lists, must be only numbers, and 4 or less digits\n";
print " [--new-listname-prefix=X] = prefix for list name when creating new lists, will be followed by filename\n";
print " [--new-list-campaign=X] = campaign that the new list will be assigned to\n";
print " [--new-list-active=X] = Y or N, if list is to be set as active when created, default N\n";
print " [--new-list-reset-times=X] = reset times(4 digits each dash separated) default is blank\n";
print " [--new-list-tz-setting=X] = COUNTRY_AND_AREA_CODE|POSTAL_CODE|NANPA_PREFIX|OWNER_TIME_ZONE_CODE default COUNTRY_AND_AREA_CODE\n";
print " [--USACAN-prefix-check] = check for the 4th digit 2-9, USA and Canada validation\n";
print " [--USACAN-areacode-check] = check for the valid phone code 1 areacodes, USA and Canada validation\n";
print " [--duplicate-check] = checks for the same phone number in the same list id before inserting lead\n";
@@ -202,6 +206,8 @@ if (length($ARGV[0])>1)
print "dccsv10:\n";
print "VENDOR_ID,FIRST_NAME,LAST_NAME,PHONE_1,PHONE_2,PHONE_3,PHONE_4,PHONE_5,PHONE_6,PHONE_7\n";
print "\"100998\",\"ANGELA \",\"SMITH \",\"3145551212\",\"3145551213\",\"3145551214\",\"0\",\"3145551215\",\"3145551216\",\"0\",\n\n";
print "pipe30tz:\n";
print "TEST01||09292011|1|5125554727||Mike||Frank|||||||||||||||||C|2145559922|TESTSURVEY|TESTSURVEY|111\n";
print "dccsv43, dccsvref51, dccsv52 and dccsvref52:\n";
print "---format too confusing to list in the help screen---\n\n";
@@ -209,7 +215,7 @@ if (length($ARGV[0])>1)
}
else
{
if ($args =~ /-q/i)
if ($args =~ /--quiet/i)
{
$q=1;
$DB=0;
@@ -228,7 +234,7 @@ if (length($ARGV[0])>1)
}
else {$DBX=0;}
if ($args =~ /-t/i)
if ($args =~ /--test/i)
{
$T=1;
$TEST=1;
@@ -363,6 +369,33 @@ if (length($ARGV[0])>1)
}
else
{$list_campaign = '';}
if ($args =~ /--new-list-active=/i)
{
@data_in = split(/--new-list-active=/,$args);
$active_list = $data_in[1];
$active_list =~ s/ .*//gi;
if ($q < 1) {print "\n----- NEW LIST ACTIVE: $active_list -----\n\n";}
}
else
{$active_list = 'N';}
if ($args =~ /--new-list-reset-times=/i)
{
@data_in = split(/--new-list-reset-times=/,$args);
$reset_time = $data_in[1];
$reset_time =~ s/ .*//gi;
if ($q < 1) {print "\n----- NEW LIST RESET TIMES: $reset_time -----\n\n";}
}
else
{$reset_time = '';}
if ($args =~ /--new-list-tz-setting=/i)
{
@data_in = split(/--new-list-tz-setting=/,$args);
$time_zone_setting = $data_in[1];
$time_zone_setting =~ s/ .*//gi;
if ($q < 1) {print "\n----- NEW LIST TZ SETTING: $time_zone_setting -----\n\n";}
}
else
{$time_zone_setting = 'COUNTRY_AND_AREA_CODE';}
if ($args =~ /-ftp-pull/i)
{
@@ -600,8 +633,9 @@ foreach(@FILES)
$xloop++;
}
$forcelistfilename_listid = $new_list_id;
$stmtZ = "INSERT INTO vicidial_lists (list_id,list_name,list_description,campaign_id,active,list_changedate) values('$new_list_id','$list_name_prefix $FILES[$i]','Created: $insert_date','$list_campaign','N','$insert_date');";
$stmtZ = "INSERT INTO vicidial_lists (list_id,list_name,list_description,campaign_id,active,list_changedate,time_zone_setting,reset_time) values('$new_list_id','$list_name_prefix $FILES[$i]','Created: $insert_date','$list_campaign','$active_list','$insert_date','$time_zone_setting','$reset_time');";
if (!$T) {$affected_rows = $dbhA->do($stmtZ); } # or die "Couldn't execute query: |$stmtZ|\n";
if ($DB > 0) {print "LIST CREATED: $new_list_id|$affected_rows|$stmtZ\n";}
}
@@ -1498,7 +1532,51 @@ foreach(@FILES)
$format_set++;
}
# This is the format for the pipe30tz lead files
# TEST01||09292011|1|5125554727||Mike||Frank|||||||||||||||||C|2145559922|TESTSURVEY|TESTSURVEY|111
if ( ($format =~ /pipe30tz/) && ($format_set < 1) )
{
$source_id = $m[0]; chomp($source_id);
$vendor_lead_code = $m[2]; chomp($vendor_lead_code);
$phone_code = $m[3]; chomp($phone_code); $phone_code =~ s/\D//gi;
$phone_number = $m[4]; chomp($phone_number); $phone_number =~ s/\D//gi;
$USarea = substr($phone_number, 0, 3);
$title = $m[5]; chomp($title);
$first_name = $m[6]; chomp($first_name);
$middle_initial = $m[7]; chomp($middle_initial);
$last_name = $m[8]; chomp($last_name);
# $address1 = $m[9]; chomp($address1);
# $address2 = $m[10]; chomp($address2);
# $address3 = $m[11]; chomp($address3);
# $city = $m[12]; chomp($city);
# $state = $m[13]; chomp($state);
# $province = $m[14]; chomp($province);
# $postal_code = $m[15]; chomp($postal_code);
# $country = $m[16]; chomp($country);
# $gender = $m[17];
# $date_of_birth = $m[18];
# $alt_phone = $m[19]; chomp($alt_phone); $alt_phone =~ s/\D//gi;
# $email = $m[20];
# $security_phrase = $m[21];
# $comments = $m[22];
$owner = $m[25];
if ($owner =~ /E/) {$owner='EST';}
if ($owner =~ /C/) {$owner='CST';}
if ($owner =~ /M/) {$owner='MST';}
if ($owner =~ /P/) {$owner='PST';}
$security_phrase = $m[26];
$province = $m[27];
$address3 = $m[28];
$alt_phone = $m[29];
$list_id=$new_list_id;
$called_count=0;
$status='NEW';
$insert_date=$pulldate0;
$map_count=0;
$format_set++;
}
# This is the format for the standard lead files
www/vicidial/sheet2tab.pl
+205 -203
View File
@@ -30,7 +30,6 @@
#
# disable when not debugging
#use 5.010;
#use strict;
#use warnings;
@@ -96,264 +95,267 @@ chomp $exten_file;
# break the file up into its parts
my ($dir, $name, $ext) = fileparse($exten_file, @exts);
given ($ext) {
# if we are a csv file
when ('.csv') {
open( IN, $infile ) or die "can't open $infile: $!\n";
open( TMPFILE , ">$tempfile" ) or die $!;
# if we are a csv file
my $cur_loop_time = time();
my $old_loop_time = time();
my $loop_time = 0;
my $loop_sleep = 0;
if ($ext eq '.csv')
{
open( IN, $infile ) or die "can't open $infile: $!\n";
open( TMPFILE , ">$tempfile" ) or die $!;
my $loop_count = 0;
my $cur_loop_time = time();
my $old_loop_time = time();
my $loop_time = 0;
my $loop_sleep = 0;
# loop through the file and process it in chunks
while( <IN> )
my $loop_count = 0;
# loop through the file and process it in chunks
while( <IN> )
{
$loop_count++;
print TMPFILE $_;
if ($debug) { print STDERR "csv line = '$_'\n"; };
# break
if ( $loop_count % $csv_chuck_size == 0 )
{
$loop_count++;
print TMPFILE $_;
if ($debug) { print STDERR "csv line = '$_'\n"; };
# close the temp file
close( TMPFILE );
# break
if ( $loop_count % $csv_chuck_size == 0 )
# process the temp file
my $parser = ReadData ( "$tempfile" );
my $maxCol = $parser->[1]{maxcol};
my $maxRow = $parser->[1]{maxrow};
# Something is not right if they have 0 or over 100 columns in their lead file.
if (( $maxCol >= 100 ) || ( $maxCol == 0 ))
{
print STDERR "ERROR: Improperly formatted lead file.\n";
print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n";
exit;
}
# close the temp file
close( TMPFILE );
if ($debug) { print STDERR "maxCol = '$maxCol'\n"; };
if ($debug) { print STDERR "maxRow = '$maxRow'\n"; };
# process the temp file
my $parser = ReadData ( "$tempfile" );
my $maxCol = $parser->[1]{maxcol};
my $maxRow = $parser->[1]{maxrow};
# Something is not right if they have 0 or over 100 columns in their lead file.
if (( $maxCol >= 100 ) || ( $maxCol == 0 ))
# loop through the rows
for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ )
{
# loop through the cols
for ( $colPos = 1; $colPos <= $maxCol; $colPos++ )
{
print STDERR "ERROR: Improperly formatted lead file.\n";
print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n";
exit;
}
my $cell = cr2cell( $colPos, $rowPos );
if ($debug) { print STDERR "maxCol = '$maxCol'\n"; };
if ($debug) { print STDERR "maxRow = '$maxRow'\n"; };
if ($debug) { print STDERR "cell = '$cell'\n"; };
# loop through the rows
for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ )
{
# loop through the cols
for ( $colPos = 1; $colPos <= $maxCol; $colPos++ )
my $field;
# make sure the field has a value
if ( $parser->[1]{$cell} )
{
my $cell = cr2cell( $colPos, $rowPos );
$field = $parser->[1]{$cell};
}
else
{
$field = "";
}
if ($debug) { print STDERR "cell = '$cell'\n"; };
if ($debug) { print STDERR "field = '$field'\n"; };
my $field;
$field = scrub_lead_field( $field );
# make sure the field has a value
if ( $parser->[1]{$cell} )
{
$field = $parser->[1]{$cell};
}
else
{
$field = "";
}
print OUTFILE $field;
if ($debug) { print STDERR "field = '$field'\n"; };
$field = scrub_lead_field( $field );
print OUTFILE $field;
if ( $colPos < $maxCol )
{
print OUTFILE $out_delim;
}
else
{
print OUTFILE "\n";
}
if ( $colPos < $maxCol )
{
print OUTFILE $out_delim;
}
else
{
print OUTFILE "\n";
}
}
}
# delete the TMPFILE
unlink( $tempfile ) or die $!;
# delete the TMPFILE
unlink( $tempfile ) or die $!;
# repopn the TMPFILE
open( TMPFILE , ">$tempfile" ) or die $!;
# repopn the TMPFILE
open( TMPFILE , ">$tempfile" ) or die $!;
# figure out how long it took to loop
$old_loop_time = $cur_loop_time;
$cur_loop_time = time();
$loop_time = $cur_loop_time - $old_loop_time;
# figure out how long it took to loop
$old_loop_time = $cur_loop_time;
$cur_loop_time = time();
#print STDERR "loop_count = '$loop_count' $loop_time $loop_sleep\n";
$loop_time = $cur_loop_time - $old_loop_time;
if ( $loop_time > $crap_loop_time )
#print STDERR "loop_count = '$loop_count' $loop_time $loop_sleep\n";
if ( $loop_time > $crap_loop_time )
{
# this should not take this long to run through a loop.
# sleep for a bit to let the CPU recover.
$loop_sleep = $loop_sleep + $loop_time;
# they have waited 60 seconds lets just kill this and get it over with
if ( $loop_sleep > 60 )
{
# this should not take this long to run through a loop.
# sleep for a bit to let the CPU recover.
$loop_sleep = $loop_sleep + $loop_time;
close( TMPFILE );
close( IN );
# they have waited 60 seconds lets just kill this and get it over with
if ( $loop_sleep > 60 ) {
close( TMPFILE );
close( IN );
# delete the TMPFILE
unlink( $tempfile ) or die $!;
open( ERRFILE, ">>$error_log" );
print ERRFILE "$cur_loop_time: Sheet2tab.pl aborting. Penalized them long enough for their junk leads in $infile \n\n";
close( ERRFILE );
exit;
}
sleep($loop_sleep);
# delete the TMPFILE
unlink( $tempfile ) or die $!;
open( ERRFILE, ">>$error_log" );
print ERRFILE "$cur_loop_time: Sheet2tab.pl took $loop_time to process $csv_chuck_size leads from the $infile lead file. Making them sleep $loop_sleep so we can recover.\n\n";
print ERRFILE "$cur_loop_time: Sheet2tab.pl aborting. Penalized them long enough for their junk leads in $infile \n\n";
close( ERRFILE );
# do not penalize the next loop through because we forced them to sleep
$cur_loop_time = time();
exit;
}
sleep($loop_sleep);
open( ERRFILE, ">>$error_log" );
print ERRFILE "$cur_loop_time: Sheet2tab.pl took $loop_time to process $csv_chuck_size leads from the $infile lead file. Making them sleep $loop_sleep so we can recover.\n\n";
close( ERRFILE );
# do not penalize the next loop through because we forced them to sleep
$cur_loop_time = time();
}
}
# close the temp and In files
close( TMPFILE );
close( IN );
# see if we have any left overs
my $temp_file_size = -s $tempfile;
# if not exit
if ( $temp_file_size == 0 ) {
# delete the TMPFILE
unlink( $tempfile ) or die $!;
exit;
}
# close the temp and In files
close( TMPFILE );
close( IN );
# other wise parse the last temp file
my $parser = ReadData ( "$tempfile" );
my $maxCol = $parser->[1]{maxcol};
my $maxRow = $parser->[1]{maxrow};
if (( $maxCol >= 100 ) || ( $maxCol == 0 ))
{
print STDERR "ERROR: Improperly formatted lead file.\n";
print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n";
exit;
}
if ($debug) { print STDERR "maxCol = '$maxCol'\n"; };
if ($debug) { print STDERR "maxRow = '$maxRow'\n"; };
# loop through the rows
for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ )
{
# loop through the cols
for ( $colPos = 1; $colPos <= $maxCol; $colPos++ )
{
my $cell = cr2cell( $colPos, $rowPos );
if ($debug) { print STDERR "cell = '$cell'\n"; };
my $field;
# make sure the field has a value
if ( $parser->[1]{$cell} )
{
$field = $parser->[1]{$cell};
}
else
{
$field = "";
}
if ($debug) { print STDERR "field = '$field'\n"; };
$field = scrub_lead_field( $field );
print OUTFILE $field;
if ( $colPos < $maxCol )
{
print OUTFILE $out_delim;
}
else
{
print OUTFILE "\n";
}
}
}
# see if we have any left overs
my $temp_file_size = -s $tempfile;
# if not exit
if ( $temp_file_size == 0 )
{
# delete the TMPFILE
unlink( $tempfile ) or die $!;
exit;
}
# not a CSV file
default {
# parse the file
my $parser = ReadData ( "$infile" );
my $maxCol = $parser->[1]{maxcol};
my $maxRow = $parser->[1]{maxrow};
# other wise parse the last temp file
my $parser = ReadData ( "$tempfile" );
my $maxCol = $parser->[1]{maxcol};
my $maxRow = $parser->[1]{maxrow};
if ($debug) { print STDERR "maxCol = '$maxCol'\n"; };
if ($debug) { print STDERR "maxRow = '$maxRow'\n"; };
if (( $maxCol >= 100 ) || ( $maxCol == 0 ))
{
print STDERR "ERROR: Improperly formatted lead file.\n";
print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n";
exit;
}
# loop through the rows
for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ )
if ($debug) { print STDERR "maxCol = '$maxCol'\n"; };
if ($debug) { print STDERR "maxRow = '$maxRow'\n"; };
# loop through the rows
for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ )
{
# loop through the cols
for ( $colPos = 1; $colPos <= $maxCol; $colPos++ )
{
# loop through the cols
for ( $colPos = 1; $colPos <= $maxCol; $colPos++ )
my $cell = cr2cell( $colPos, $rowPos );
if ($debug) { print STDERR "cell = '$cell'\n"; };
my $field;
# make sure the field has a value
if ( $parser->[1]{$cell} )
{
my $cell = cr2cell( $colPos, $rowPos );
$field = $parser->[1]{$cell};
}
else
{
$field = "";
}
if ($debug) { print STDERR "field = '$field'\n"; };
$field = scrub_lead_field( $field );
print OUTFILE $field;
if ( $colPos < $maxCol )
{
print OUTFILE $out_delim;
}
else
{
print OUTFILE "\n";
}
}
}
# delete the TMPFILE
unlink( $tempfile ) or die $!;
}
else
# not a CSV file
{
# parse the file
my $parser = ReadData ( "$infile" );
if ($debug) { print STDERR "cell = '$cell'\n"; };
my $maxCol = $parser->[1]{maxcol};
my $maxRow = $parser->[1]{maxrow};
my $field;
# make sure the field has a value
if ( $parser->[1]{$cell} )
{
$field = $parser->[1]{$cell};
}
else
{
$field = "";
}
if ($debug) { print STDERR "maxCol = '$maxCol'\n"; };
if ($debug) { print STDERR "maxRow = '$maxRow'\n"; };
if ($debug) { print STDERR "field = '$field'\n"; };
# loop through the rows
for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ )
{
# loop through the cols
for ( $colPos = 1; $colPos <= $maxCol; $colPos++ )
{
my $cell = cr2cell( $colPos, $rowPos );
if ($debug) { print STDERR "cell = '$cell'\n"; };
$field = scrub_lead_field( $field );
my $field;
print OUTFILE $field;
# make sure the field has a value
if ( $parser->[1]{$cell} )
{
$field = $parser->[1]{$cell};
}
else
{
$field = "";
}
if ( $colPos < $maxCol )
{
print OUTFILE $out_delim;
}
else
{
print OUTFILE "\n";
}
if ($debug) { print STDERR "field = '$field'\n"; };
$field = scrub_lead_field( $field );
print OUTFILE $field;
if ( $colPos < $maxCol )
{
print OUTFILE $out_delim;
}
else
{
print OUTFILE "\n";
}
}
}