From ef591374a11c3ffa15901385a2888f2ec330bec0 Mon Sep 17 00:00:00 2001 From: mattf Date: Mon, 3 Oct 2011 19:56:55 +0000 Subject: [PATCH] Fixed issue with sheet2tab.pl Added several new features to the CLI lead loader based around creation of new lists and one new format git-svn-id: svn://192.168.202.10@1731 3d104415-ff17-0410-8863-d5cf3c621b8a --- bin/VICIDIAL_IN_new_leads_file.pl | 90 ++++++- www/vicidial/sheet2tab.pl | 408 +++++++++++++++--------------- 2 files changed, 289 insertions(+), 209 deletions(-) diff --git a/bin/VICIDIAL_IN_new_leads_file.pl b/bin/VICIDIAL_IN_new_leads_file.pl index 8dd0adaf..bd415fc9 100644 --- a/bin/VICIDIAL_IN_new_leads_file.pl +++ b/bin/VICIDIAL_IN_new_leads_file.pl @@ -52,9 +52,10 @@ # 110420-0944 - Fixed file prefix issue with multiple processes running # 110424-0948 - Added time-zone-code-gmt option to use time zone code from the owner field # 110705-1913 - Added options for USACAN prefix(no 0 or 1) and valid areacode filtering +# 110929-1423 - Added new format for abbreviated timezone in owner(pipe30tz) and list creation options # -$version = '110705-1913'; +$version = '110929-1423'; $secX = time(); $MT[0]=''; @@ -148,8 +149,8 @@ if (length($ARGV[0])>1) if ($args =~ /--help|-h/i) { print "allowed run time options:\n"; - print " [-q] = quiet\n"; - print " [-t] = test\n"; + print " [--quiet] = quiet\n"; + print " [--test] = test\n"; print " [--version] = version\n"; print " [--forcegmt] = forces gmt value of column after comments column\n"; print " [--debug] = debug output\n"; @@ -162,6 +163,9 @@ if (length($ARGV[0])>1) print " [--new-listid-prefix=X] = prefix for listID when creating new lists, must be only numbers, and 4 or less digits\n"; print " [--new-listname-prefix=X] = prefix for list name when creating new lists, will be followed by filename\n"; print " [--new-list-campaign=X] = campaign that the new list will be assigned to\n"; + print " [--new-list-active=X] = Y or N, if list is to be set as active when created, default N\n"; + print " [--new-list-reset-times=X] = reset times(4 digits each dash separated) default is blank\n"; + print " [--new-list-tz-setting=X] = COUNTRY_AND_AREA_CODE|POSTAL_CODE|NANPA_PREFIX|OWNER_TIME_ZONE_CODE default COUNTRY_AND_AREA_CODE\n"; print " [--USACAN-prefix-check] = check for the 4th digit 2-9, USA and Canada validation\n"; print " [--USACAN-areacode-check] = check for the valid phone code 1 areacodes, USA and Canada validation\n"; print " [--duplicate-check] = checks for the same phone number in the same list id before inserting lead\n"; @@ -202,6 +206,8 @@ if (length($ARGV[0])>1) print "dccsv10:\n"; print "VENDOR_ID,FIRST_NAME,LAST_NAME,PHONE_1,PHONE_2,PHONE_3,PHONE_4,PHONE_5,PHONE_6,PHONE_7\n"; print "\"100998\",\"ANGELA \",\"SMITH \",\"3145551212\",\"3145551213\",\"3145551214\",\"0\",\"3145551215\",\"3145551216\",\"0\",\n\n"; + print "pipe30tz:\n"; + print "TEST01||09292011|1|5125554727||Mike||Frank|||||||||||||||||C|2145559922|TESTSURVEY|TESTSURVEY|111\n"; print "dccsv43, dccsvref51, dccsv52 and dccsvref52:\n"; print "---format too confusing to list in the help screen---\n\n"; @@ -209,7 +215,7 @@ if (length($ARGV[0])>1) } else { - if ($args =~ /-q/i) + if ($args =~ /--quiet/i) { $q=1; $DB=0; @@ -228,7 +234,7 @@ if (length($ARGV[0])>1) } else {$DBX=0;} - if ($args =~ /-t/i) + if ($args =~ /--test/i) { $T=1; $TEST=1; @@ -363,6 +369,33 @@ if (length($ARGV[0])>1) } else {$list_campaign = '';} + if ($args =~ /--new-list-active=/i) + { + @data_in = split(/--new-list-active=/,$args); + $active_list = $data_in[1]; + $active_list =~ s/ .*//gi; + if ($q < 1) {print "\n----- NEW LIST ACTIVE: $active_list -----\n\n";} + } + else + {$active_list = 'N';} + if ($args =~ /--new-list-reset-times=/i) + { + @data_in = split(/--new-list-reset-times=/,$args); + $reset_time = $data_in[1]; + $reset_time =~ s/ .*//gi; + if ($q < 1) {print "\n----- NEW LIST RESET TIMES: $reset_time -----\n\n";} + } + else + {$reset_time = '';} + if ($args =~ /--new-list-tz-setting=/i) + { + @data_in = split(/--new-list-tz-setting=/,$args); + $time_zone_setting = $data_in[1]; + $time_zone_setting =~ s/ .*//gi; + if ($q < 1) {print "\n----- NEW LIST TZ SETTING: $time_zone_setting -----\n\n";} + } + else + {$time_zone_setting = 'COUNTRY_AND_AREA_CODE';} if ($args =~ /-ftp-pull/i) { @@ -600,8 +633,9 @@ foreach(@FILES) $xloop++; } + $forcelistfilename_listid = $new_list_id; - $stmtZ = "INSERT INTO vicidial_lists (list_id,list_name,list_description,campaign_id,active,list_changedate) values('$new_list_id','$list_name_prefix $FILES[$i]','Created: $insert_date','$list_campaign','N','$insert_date');"; + $stmtZ = "INSERT INTO vicidial_lists (list_id,list_name,list_description,campaign_id,active,list_changedate,time_zone_setting,reset_time) values('$new_list_id','$list_name_prefix $FILES[$i]','Created: $insert_date','$list_campaign','$active_list','$insert_date','$time_zone_setting','$reset_time');"; if (!$T) {$affected_rows = $dbhA->do($stmtZ); } # or die "Couldn't execute query: |$stmtZ|\n"; if ($DB > 0) {print "LIST CREATED: $new_list_id|$affected_rows|$stmtZ\n";} } @@ -1498,7 +1532,51 @@ foreach(@FILES) $format_set++; } + # This is the format for the pipe30tz lead files + # TEST01||09292011|1|5125554727||Mike||Frank|||||||||||||||||C|2145559922|TESTSURVEY|TESTSURVEY|111 + if ( ($format =~ /pipe30tz/) && ($format_set < 1) ) + { + $source_id = $m[0]; chomp($source_id); + $vendor_lead_code = $m[2]; chomp($vendor_lead_code); + $phone_code = $m[3]; chomp($phone_code); $phone_code =~ s/\D//gi; + $phone_number = $m[4]; chomp($phone_number); $phone_number =~ s/\D//gi; + $USarea = substr($phone_number, 0, 3); + $title = $m[5]; chomp($title); + $first_name = $m[6]; chomp($first_name); + $middle_initial = $m[7]; chomp($middle_initial); + $last_name = $m[8]; chomp($last_name); + # $address1 = $m[9]; chomp($address1); + # $address2 = $m[10]; chomp($address2); + # $address3 = $m[11]; chomp($address3); + # $city = $m[12]; chomp($city); + # $state = $m[13]; chomp($state); + # $province = $m[14]; chomp($province); + # $postal_code = $m[15]; chomp($postal_code); + # $country = $m[16]; chomp($country); + # $gender = $m[17]; + # $date_of_birth = $m[18]; + # $alt_phone = $m[19]; chomp($alt_phone); $alt_phone =~ s/\D//gi; + # $email = $m[20]; + # $security_phrase = $m[21]; + # $comments = $m[22]; + $owner = $m[25]; + if ($owner =~ /E/) {$owner='EST';} + if ($owner =~ /C/) {$owner='CST';} + if ($owner =~ /M/) {$owner='MST';} + if ($owner =~ /P/) {$owner='PST';} + $security_phrase = $m[26]; + $province = $m[27]; + $address3 = $m[28]; + $alt_phone = $m[29]; + $list_id=$new_list_id; + $called_count=0; + $status='NEW'; + $insert_date=$pulldate0; + $map_count=0; + + $format_set++; + } # This is the format for the standard lead files diff --git a/www/vicidial/sheet2tab.pl b/www/vicidial/sheet2tab.pl index 6f282b35..7abf691d 100644 --- a/www/vicidial/sheet2tab.pl +++ b/www/vicidial/sheet2tab.pl @@ -30,7 +30,6 @@ # # disable when not debugging -#use 5.010; #use strict; #use warnings; @@ -96,264 +95,267 @@ chomp $exten_file; # break the file up into its parts my ($dir, $name, $ext) = fileparse($exten_file, @exts); -given ($ext) { - # if we are a csv file - when ('.csv') { - open( IN, $infile ) or die "can't open $infile: $!\n"; - open( TMPFILE , ">$tempfile" ) or die $!; +# if we are a csv file - my $cur_loop_time = time(); - my $old_loop_time = time(); - my $loop_time = 0; - my $loop_sleep = 0; +if ($ext eq '.csv') + { + open( IN, $infile ) or die "can't open $infile: $!\n"; + open( TMPFILE , ">$tempfile" ) or die $!; - my $loop_count = 0; + my $cur_loop_time = time(); + my $old_loop_time = time(); + my $loop_time = 0; + my $loop_sleep = 0; - # loop through the file and process it in chunks - while( ) + my $loop_count = 0; + + # loop through the file and process it in chunks + while( ) + { + $loop_count++; + + print TMPFILE $_; + if ($debug) { print STDERR "csv line = '$_'\n"; }; + + # break + if ( $loop_count % $csv_chuck_size == 0 ) { - $loop_count++; - print TMPFILE $_; - if ($debug) { print STDERR "csv line = '$_'\n"; }; + # close the temp file + close( TMPFILE ); - # break - if ( $loop_count % $csv_chuck_size == 0 ) + # process the temp file + my $parser = ReadData ( "$tempfile" ); + + my $maxCol = $parser->[1]{maxcol}; + my $maxRow = $parser->[1]{maxrow}; + + # Something is not right if they have 0 or over 100 columns in their lead file. + if (( $maxCol >= 100 ) || ( $maxCol == 0 )) { + print STDERR "ERROR: Improperly formatted lead file.\n"; + print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n"; + exit; + } - # close the temp file - close( TMPFILE ); + if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; + if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; - # process the temp file - my $parser = ReadData ( "$tempfile" ); - - my $maxCol = $parser->[1]{maxcol}; - my $maxRow = $parser->[1]{maxrow}; - - # Something is not right if they have 0 or over 100 columns in their lead file. - if (( $maxCol >= 100 ) || ( $maxCol == 0 )) + # loop through the rows + for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) + { + # loop through the cols + for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) { - print STDERR "ERROR: Improperly formatted lead file.\n"; - print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n"; - exit; - } + my $cell = cr2cell( $colPos, $rowPos ); - if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; - if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; + if ($debug) { print STDERR "cell = '$cell'\n"; }; - # loop through the rows - for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) - { - # loop through the cols - for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) + my $field; + + # make sure the field has a value + if ( $parser->[1]{$cell} ) { - my $cell = cr2cell( $colPos, $rowPos ); + $field = $parser->[1]{$cell}; + } + else + { + $field = ""; + } - if ($debug) { print STDERR "cell = '$cell'\n"; }; + if ($debug) { print STDERR "field = '$field'\n"; }; - my $field; + $field = scrub_lead_field( $field ); - # make sure the field has a value - if ( $parser->[1]{$cell} ) - { - $field = $parser->[1]{$cell}; - } - else - { - $field = ""; - } + print OUTFILE $field; - if ($debug) { print STDERR "field = '$field'\n"; }; - - $field = scrub_lead_field( $field ); - - print OUTFILE $field; - - if ( $colPos < $maxCol ) - { - print OUTFILE $out_delim; - } - else - { - print OUTFILE "\n"; - } + if ( $colPos < $maxCol ) + { + print OUTFILE $out_delim; + } + else + { + print OUTFILE "\n"; } } + } - # delete the TMPFILE - unlink( $tempfile ) or die $!; + # delete the TMPFILE + unlink( $tempfile ) or die $!; - # repopn the TMPFILE - open( TMPFILE , ">$tempfile" ) or die $!; + # repopn the TMPFILE + open( TMPFILE , ">$tempfile" ) or die $!; - # figure out how long it took to loop - $old_loop_time = $cur_loop_time; - $cur_loop_time = time(); - $loop_time = $cur_loop_time - $old_loop_time; + # figure out how long it took to loop + $old_loop_time = $cur_loop_time; + $cur_loop_time = time(); - #print STDERR "loop_count = '$loop_count' $loop_time $loop_sleep\n"; + $loop_time = $cur_loop_time - $old_loop_time; - if ( $loop_time > $crap_loop_time ) + #print STDERR "loop_count = '$loop_count' $loop_time $loop_sleep\n"; + + if ( $loop_time > $crap_loop_time ) + { + # this should not take this long to run through a loop. + # sleep for a bit to let the CPU recover. + $loop_sleep = $loop_sleep + $loop_time; + + # they have waited 60 seconds lets just kill this and get it over with + if ( $loop_sleep > 60 ) { - # this should not take this long to run through a loop. - # sleep for a bit to let the CPU recover. - $loop_sleep = $loop_sleep + $loop_time; + close( TMPFILE ); + close( IN ); - # they have waited 60 seconds lets just kill this and get it over with - if ( $loop_sleep > 60 ) { - close( TMPFILE ); - close( IN ); - - # delete the TMPFILE - unlink( $tempfile ) or die $!; - - open( ERRFILE, ">>$error_log" ); - - print ERRFILE "$cur_loop_time: Sheet2tab.pl aborting. Penalized them long enough for their junk leads in $infile \n\n"; - - close( ERRFILE ); - - exit; - } - - sleep($loop_sleep); + # delete the TMPFILE + unlink( $tempfile ) or die $!; open( ERRFILE, ">>$error_log" ); - print ERRFILE "$cur_loop_time: Sheet2tab.pl took $loop_time to process $csv_chuck_size leads from the $infile lead file. Making them sleep $loop_sleep so we can recover.\n\n"; + print ERRFILE "$cur_loop_time: Sheet2tab.pl aborting. Penalized them long enough for their junk leads in $infile \n\n"; close( ERRFILE ); - # do not penalize the next loop through because we forced them to sleep - $cur_loop_time = time(); + exit; } + + sleep($loop_sleep); + + open( ERRFILE, ">>$error_log" ); + + print ERRFILE "$cur_loop_time: Sheet2tab.pl took $loop_time to process $csv_chuck_size leads from the $infile lead file. Making them sleep $loop_sleep so we can recover.\n\n"; + + close( ERRFILE ); + + # do not penalize the next loop through because we forced them to sleep + $cur_loop_time = time(); } } - - # close the temp and In files - close( TMPFILE ); - close( IN ); - - # see if we have any left overs - my $temp_file_size = -s $tempfile; - - # if not exit - if ( $temp_file_size == 0 ) { - # delete the TMPFILE - unlink( $tempfile ) or die $!; - exit; } + + # close the temp and In files + close( TMPFILE ); + close( IN ); - # other wise parse the last temp file - my $parser = ReadData ( "$tempfile" ); - - my $maxCol = $parser->[1]{maxcol}; - my $maxRow = $parser->[1]{maxrow}; - - if (( $maxCol >= 100 ) || ( $maxCol == 0 )) - { - print STDERR "ERROR: Improperly formatted lead file.\n"; - print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n"; - exit; - } - - if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; - if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; - - # loop through the rows - for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) - { - # loop through the cols - for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) - { - my $cell = cr2cell( $colPos, $rowPos ); - - if ($debug) { print STDERR "cell = '$cell'\n"; }; - - my $field; - - # make sure the field has a value - if ( $parser->[1]{$cell} ) - { - $field = $parser->[1]{$cell}; - } - else - { - $field = ""; - } - - if ($debug) { print STDERR "field = '$field'\n"; }; - - $field = scrub_lead_field( $field ); - - print OUTFILE $field; - - if ( $colPos < $maxCol ) - { - print OUTFILE $out_delim; - } - else - { - print OUTFILE "\n"; - } - } - } + # see if we have any left overs + my $temp_file_size = -s $tempfile; + # if not exit + if ( $temp_file_size == 0 ) + { # delete the TMPFILE unlink( $tempfile ) or die $!; - + exit; } - # not a CSV file - default { - # parse the file - my $parser = ReadData ( "$infile" ); - - my $maxCol = $parser->[1]{maxcol}; - my $maxRow = $parser->[1]{maxrow}; + # other wise parse the last temp file + my $parser = ReadData ( "$tempfile" ); + my $maxCol = $parser->[1]{maxcol}; + my $maxRow = $parser->[1]{maxrow}; - if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; - if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; + if (( $maxCol >= 100 ) || ( $maxCol == 0 )) + { + print STDERR "ERROR: Improperly formatted lead file.\n"; + print OUTFILE "BAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\tBAD_LEAD_FILE\t\n"; + exit; + } - # loop through the rows - for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) + if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; + if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; + + # loop through the rows + for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) + { + # loop through the cols + for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) { - # loop through the cols - for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) + my $cell = cr2cell( $colPos, $rowPos ); + + if ($debug) { print STDERR "cell = '$cell'\n"; }; + + my $field; + + # make sure the field has a value + if ( $parser->[1]{$cell} ) { - my $cell = cr2cell( $colPos, $rowPos ); + $field = $parser->[1]{$cell}; + } + else + { + $field = ""; + } + + if ($debug) { print STDERR "field = '$field'\n"; }; + + $field = scrub_lead_field( $field ); + + print OUTFILE $field; + + if ( $colPos < $maxCol ) + { + print OUTFILE $out_delim; + } + else + { + print OUTFILE "\n"; + } + } + } + + # delete the TMPFILE + unlink( $tempfile ) or die $!; + + } +else + # not a CSV file + { + # parse the file + my $parser = ReadData ( "$infile" ); - if ($debug) { print STDERR "cell = '$cell'\n"; }; + my $maxCol = $parser->[1]{maxcol}; + my $maxRow = $parser->[1]{maxrow}; - my $field; - # make sure the field has a value - if ( $parser->[1]{$cell} ) - { - $field = $parser->[1]{$cell}; - } - else - { - $field = ""; - } + if ($debug) { print STDERR "maxCol = '$maxCol'\n"; }; + if ($debug) { print STDERR "maxRow = '$maxRow'\n"; }; - if ($debug) { print STDERR "field = '$field'\n"; }; + # loop through the rows + for ( $rowPos = 1; $rowPos <= $maxRow; $rowPos++ ) + { + # loop through the cols + for ( $colPos = 1; $colPos <= $maxCol; $colPos++ ) + { + my $cell = cr2cell( $colPos, $rowPos ); + + if ($debug) { print STDERR "cell = '$cell'\n"; }; - $field = scrub_lead_field( $field ); + my $field; - print OUTFILE $field; + # make sure the field has a value + if ( $parser->[1]{$cell} ) + { + $field = $parser->[1]{$cell}; + } + else + { + $field = ""; + } - if ( $colPos < $maxCol ) - { - print OUTFILE $out_delim; - } - else - { - print OUTFILE "\n"; - } + if ($debug) { print STDERR "field = '$field'\n"; }; + + $field = scrub_lead_field( $field ); + + print OUTFILE $field; + + if ( $colPos < $maxCol ) + { + print OUTFILE $out_delim; + } + else + { + print OUTFILE "\n"; } } }