ViciDial/agc_2-X
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
caf44ed315276ef2b8663b13f022af2ee44763e4
agc_2-X/bin/AST_inbound_email_parser.pl
T
mattf caf44ed315 better support for non-latin email characters. 
new cpan module required: install MIME::Base64

git-svn-id: svn://192.168.202.10@1929 3d104415-ff17-0410-8863-d5cf3c621b8a
2013-01-27 05:30:12 +00:00

860 lines
44 KiB
Perl
Raw Blame History

#!/usr/bin/perl
#use strict;
# use warnings;
use HTML::Entities;
use HTML::Strip;
use Switch;
use Time::Local;
use MIME::Decoder;
use Encode;
use MIME::Base64;
# Copyright (C) 2013 Matt Florell, Joe Johnson <vicidial@gmail.com> LICENSE: AGPLv2
#
# AST_inbound_email_parser.pl - This script is essential for periodically checking any active POP3 or IMAP
# email accounts set up through the Vicidial admin.
#
# You cannot check an email account more often than 5 minutes at a time. This is in place because several
# email providers will lock an email account if it is being checked too often; currently the most restrictive
# email provider encountered allows no more than three logins every 15 minutes, hence 5 minutes being the
# most frequent time allowed.
#
# Upon execution, this script will query the vicidial_email_accounts table in the asterisk database and
# gather the information for all email accounts set up through the dialer that are currently active. It
# will then check the time setting on each of those accounts to see if it is time to check that particular
# account. If so, the program then will attempt to access the account using the appropriate Perl module
# depending on the account protocol (POP3 or IMAP). If the program successfully connects to the email
# account, then it will download any unread email messages, and grab any important information in the
# email headers, along with any attachments and the message itself. Each bit of information will be
# stored in the vicidial_email_list table, and any attachments will be stored in the
# inbound_email_attachments table as BLOB data.
#
# Additionally, depending on the email account settings, prior to putting the email record into the
# vicidial_email_list table, the accounts can be set up to check if the "from" address is already part
# of a lead in the vicidial_list table. The accounts can be set up to not check at all (EMAIL), in
# which all email messages go in as new leads, first in the vicidial_list table and then in the
# vicidial_email_list table with the lead_id gleaned from the vicidial_list insert. They can also be
# set up to check if the email address is already on a lead in a particular list (EMAILLOOKUPRL), or
# if the email address is in a lead belonging to any list in a given campaign (EMAILLOOKUPRC), or if
# the email address is on any lead currently in the vicidial_list table (EMAILLOOKUP). In these
# cases, the script will grab the lead_id of the most recently loaded lead (i.e. highest lead_id value),
# and use that as the lead_id in the vicidial_email_list table without creating a new lead.
#
# After checking and downloading any unread messages, the script will then log off the email account,
# and when all active email accounts have been checked in this manner, the script will exit.
#
# This script should be set up in the cron to run once per minute, not continuously.
# * * * * * /usr/share/astguiclient/AST_inbound_email_parser.pl
#
# Use the --debug variable when executing the script to have it print out what it is attempting to do on a
# step-by-step basis. Use the --debugX variable to include outputting of mail information and SQL queries the script
# attempts to execute.
#
# changes:
# 121213-2200 - First Build
# 130127-0025 - Added better non-latin characters support
#
# default path to astguiclient configuration file:
$PATHconf = '/etc/astguiclient.conf';
$|=1;
if ($ARGV[0]=~/--help/) {
print "AST_inbound_email_parser.pl - This script is essential for periodically checking any active POP3 or IMAP \n";
print "email accounts set up through the Vicidial admin.\n";
print "\n";
print "You cannot check an email account more often than 5 minutes at a time. This is in place because several \n";
print "email providers will lock an email account if it is being checked too often; currently the most restrictive\n";
print "email provider encountered allows no more than three logins every 15 minutes, hence 5 minutes being the\n";
print "most frequent time allowed.\n";
print "\n";
print "Upon execution, this script will query the vicidial_email_accounts table in the asterisk database and \n";
print "gather the information for all email accounts set up through the dialer that are currently active. It\n";
print "will then check the time setting on each of those accounts to see if it is time to check that particular \n";
print "account. If so, the program then will attempt to access the account using the appropriate Perl module\n";
print "depending on the account protocol (POP3 or IMAP). If the program successfully connects to the email \n";
print "account, then it will download any unread email messages, and grab any important information in the \n";
print "email headers, along with any attachments and the message itself. Each bit of information will be \n";
print "stored in the vicidial_email_list table, and any attachments will be stored in the \n";
print "inbound_email_attachments table as BLOB data.\n";
print "\n";
print "Additionally, depending on the email account settings, prior to putting the email record into the \n";
print "vicidial_email_list table, the accounts can be set up to check if the 'from' address is already part \n";
print "of a lead in the vicidial_list table. The accounts can be set up to not check at all (EMAIL), in \n";
print "which all email messages go in as new leads, first in the vicidial_list table and then in the \n";
print "vicidial_email_list table with the lead_id gleaned from the vicidial_list insert. They can also be\n";
print "set up to check if the email address is already on a lead in a particular list (EMAILLOOKUPRL), or\n";
print "if the email address is in a lead belonging to any list in a given campaign (EMAILLOOKUPRC), or if \n";
print "the email address is on any lead currently in the vicidial_list table (EMAILLOOKUP). In these \n";
print "cases, the script will grab the lead_id of the most recently loaded lead (i.e. highest lead_id value), \n";
print "and use that as the lead_id in the vicidial_email_list table without creating a new lead.\n";
print "\n";
print "After checking and downloading any unread messages, the script will then log off the email account, \n";
print "and when all active email accounts have been checked in this manner, the script will exit.\n";
print "\n";
print "This script should be set up in the cron to run once per minute, not continuously.\n";
print "* * * * * /usr/share/astguiclient/AST_inbound_email_parser.pl\n";
print "\n";
print "Use the --debug variable when executing the script to have it print out what it is attempting to do on a\n";
print "step-by-step basis. Use the --debugX variable to include outputting of mail information and SQL queries the script \n";
print "attempts to execute.\n";
exit;
}
open(conf, "$PATHconf") || die "can't open $PATHconf: $!\n";
@conf = <conf>;
close(conf);
$i=0;
foreach(@conf)
{
$line = $conf[$i];
$line =~ s/ |>|\n|\r|\t|\#.*|;.*//gi;
if ( ($line =~ /^PATHhome/) && ($CLIhome < 1) )
{$PATHhome = $line; $PATHhome =~ s/.*=//gi;}
if ( ($line =~ /^PATHlogs/) && ($CLIlogs < 1) )
{$PATHlogs = $line; $PATHlogs =~ s/.*=//gi;}
if ( ($line =~ /^PATHagi/) && ($CLIagi < 1) )
{$PATHagi = $line; $PATHagi =~ s/.*=//gi;}
if ( ($line =~ /^PATHweb/) && ($CLIweb < 1) )
{$PATHweb = $line; $PATHweb =~ s/.*=//gi;}
if ( ($line =~ /^PATHsounds/) && ($CLIsounds < 1) )
{$PATHsounds = $line; $PATHsounds =~ s/.*=//gi;}
if ( ($line =~ /^PATHmonitor/) && ($CLImonitor < 1) )
{$PATHmonitor = $line; $PATHmonitor =~ s/.*=//gi;}
if ( ($line =~ /^VARserver_ip/) && ($CLIserver_ip < 1) )
{$VARserver_ip = $line; $VARserver_ip =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_server/) && ($CLIDB_server < 1) )
{$VARDB_server = $line; $VARDB_server =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_database/) && ($CLIDB_database < 1) )
{$VARDB_database = $line; $VARDB_database =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_user/) && ($CLIDB_user < 1) )
{$VARDB_user = $line; $VARDB_user =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_pass/) && ($CLIDB_pass < 1) )
{$VARDB_pass = $line; $VARDB_pass =~ s/.*=//gi;}
if ( ($line =~ /^VARDB_port/) && ($CLIDB_port < 1) )
{$VARDB_port = $line; $VARDB_port =~ s/.*=//gi;}
$i++;
}
if (!$VARDB_port) {$VARDB_port='3306';}
use DBI;
$dbhA = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
or die "Couldn't connect to database: " . DBI->errstr;
$dbhA2 = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
or die "Couldn't connect to database: " . DBI->errstr;
$dbhA3 = DBI->connect("DBI:mysql:$VARDB_database:$VARDB_server:$VARDB_port", "$VARDB_user", "$VARDB_pass")
or die "Couldn't connect to database: " . DBI->errstr;
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
if($min==0) {$min=60;}
$minutes=($hour*60)+$min;
$minutes=0;
$stmt="select * from vicidial_email_accounts where active='Y'";
$rslt=$dbhA->prepare($stmt);
$rslt->execute();
while (@row=$rslt->fetchrow_array) {
$VARemail_ID=$row[0];
$VARemail_protocol=$row[4];
$VARemail_server=$row[6];
$VARemail_user=$row[7];
$VARemail_pwd=$row[8];
$VARemail_frequency=$row[10];
$VARemail_groupid=$row[11];
$default_list_id=$row[12];
$call_handle_method=$row[13];
$agent_search_method=$row[14];
$campaign_id=$row[15];
$list_id=$row[16];
if ($minutes%$VARemail_frequency==0) {
if ($ARGV[0]=~/debug/i) {print "Attempting to connect to $VARemail_protocol server ($VARemail_server)\n\n";}
if ($VARemail_protocol eq "IMAP") {
# if (!$sleep_time || $sleep_time<10) {$sleep_time=10;} # Ten second minimum wait time for IMAP.
# Connect to IMAP server
use Mail::IMAPClient;
use Mail::Message;
my $client = Mail::IMAPClient->new(
Server => "$VARemail_server",
User => "$VARemail_user",
Password => "$VARemail_pwd",
Port => 993,
Ssl => 1,
)
or die "Cannot connect through IMAPClient: $!";
# Include options for folders in IMAP? POP3 doesn't support this.
# List folders on remote server (see if all is ok)
if ( $client->IsAuthenticated() ) {
#print "Folders:\n";
$new_messages=0;
my @folder_array=$client->folders("INBOX");
for (my $i=0; $i<scalar(@folder_array); $i++) {
#print "- ".$folder_array[$i]."\n" ;
my $msgcount = $client->message_count($folder_array[$i]);
#print "+---+ Messages: ".$msgcount."\n";
if ($msgcount>0) {
$file=open(STORAGE, "> ./raw_emails.txt");
$client->select($folder_array[$i]);
my @msgs = $client->messages or die "Could not messages: $@\n";
my @unseenMsgs = $client->unseen;
for(my $j=0; $j<scalar(@msgs); $j++) {
if (grep {$_ eq $msgs[$j]} @unseenMsgs) {
$new_messages++;
my $string = $client->message_string($msgs[$j]) or die "Could not message_string: $@\n";
$client->message_to_file($file,$msgs[$j]) or die "Could not message_to_file: $@\n";
my $hashref = $client->parse_headers($msgs[$j], 'ALL');
my %email_values=%{$hashref};
my $email_to=$email_values{"To"}->[0];
my $email_from=$email_values{"From"}->[0];
my $email_date=$email_values{"Date"}->[0];
my $subject=$email_values{"Subject"}->[0];
my $mime_type=$email_values{"MIME-Version"}->[0];
my $content_type=$email_values{"Content-Type"}->[0];
my $content_transfer_encoding=$email_values{"Content-Transfer-Encoding"}->[0];
my $x_mailer=$email_values{"X-Mailer"}->[0];
my $auth_results=$email_values{"Authentication-Results"}->[0];
my $spf=$email_values{"Received-SPF"}->[0];
$message = $client->body_string($msgs[$j]);
print "\n############\n$content_transfer_encoding - $content_type - $message\n###########\n";
$text_written=0; ## Keeps track of whether or not text of email was grabbed
$attach_ct=0; ## Keeps number
@ins_values=();
@output_ins_values=();
if ($email_date=~/[0-9]{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[0-9]{4}\s+[0-9]{1,2}\:[0-9]{1,2}(\:[0-9]{1,2})?/) {
$email_date=$&;
$email_date=~s/\s+/ /gi;
@date_array=split(/\s/, $email_date);
$day=substr("0".$date_array[0], -2);
$month=$date_array[1];
$year=$date_array[2];
@time_array=split(/\:/, $date_array[3]);
$hour=substr("00".$time_array[0], -2);
$min=substr("00".$time_array[1], -2);
$sec=substr("00".$time_array[2], -2);
$email_date="$day $month $year $hour:$min:$sec";
if ($ARGV[0]=~/debug/) {print "Time stamp on email is $email_date\n";}
} elsif ($ARGV[0]=~/debug/) {print "WARNING: Time stamp on email is $email_date\n";}
############# MESSAGE ACTIONS
if ($content_type=~/^text\/plain/i) {
## Do nothing - it's plain text and needs no further work on it.
if ($ARGV[0]=~/debug/i) {print "Email message is text/plain. Nothing needs to be done.\n\n";}
} elsif ($content_type=~/^text\/html/i) {
## Message is HTML, so it needs to be stripped.
if ($ARGV[0]=~/debug/i) {print "Email message is text/html. Needs to have tags stripped via HTML::Strip.\n\n";}
StripHTML();
} elsif ($content_type=~/^multipart\/(alternative|mixed)/i) {
## Message is multipart/alternative, so it needs to be read and partitioned. The multipart/alternative subtype indicates that each part is an "alternative"
## version of the same (or similar) content, each in a different format denoted by its "Content-Type" header. The formats are ordered by how faithful they are to
## the original, with the least faithful first and the most faithful last. Systems can then choose the "best" representation they are capable of processing; in
## general, this will be the last part that the system can understand, although other factors may affect this.
if ($ARGV[0]=~/debug/i) {print "Email message is multipart. Need to select the best format type and parse it.\n";}
## First, get the boundary from the content-type and use it to break the email into it's multiple parts
if ($content_type=~/boundary\=\"?[^\"]+\"?/i) {
$boundary=$&;
$boundary=~s/(^boundary=\"?|\"?$)//gi;
$boundary="--".$boundary;
@alternatives_array=split(/$boundary/, $message);
for ($k=1; $k<(scalar(@alternatives_array)-1); $k++) { # Ignore the first and last entry in the array; it's just a blank entry and the two dashes after the last boundary.
## Grab the message by using Mail::Message to strip header information
$alt_email_text=$alternatives_array[$k];
$alt_email_text=~s/(^(\s|\r|\n)+|(\s|\r|\n)+$)//; # This needs to be trimmed so the Mail::Message->read command will strip out the headers - leading carriage returns ruin it
# Reset certain variables;
$sub_content_disposition="";
$sub_content_type="";
$attachment_fulltype="";
$attachment_filename="";
$attachment_type="";
if ($alt_email_text=~/Content\-Type\:\s+(.*?)\n/i) {$sub_content_type=$&;}
if ($alt_email_text=~/Content\-Disposition\:\s+(.*?)\n/i) {$sub_content_disposition=$&;}
if ($alt_email_text=~/Content\-Transfer\-Encoding\:\s+(.*?)\n/i) {
$encoding_type=$&;
$encoding_type=~s/(^Content-Transfer-Encoding\:\s+|\;$)//gi;
$encoding_type=~s/[\r\n]//g;
}
$sub_content_type=~s/[\r\n]//g;
$sub_content_disposition=~s/[\r\n]//g;
my $msg = Mail::Message->read($alt_email_text);
$body_contents=$msg->body;
$attachment_filesize=$msg->size();
$head=$msg->head;
if ($ARGV[0]=~/debugX/i) {print "Part content disposition is $sub_content_disposition\n Part content type is $sub_content_type.\n";}
if ($ARGV[0]=~/debugX/i) {print "Part content size is $attachment_filesize\n";}
## Check if the content-type is plain or html
if ($alt_email_text=~/Content\-Type\:\s+text\/html/i && $text_written==0) {
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/html. Stripping headers and stripping tags from the body/message...\n";}
if ($ARGV[0]=~/debugX/i) {print "$k)\n***Pre-HTML strip:\n$message\n***\n";}
StripHTML();
if ($ARGV[0]=~/debugX/i) {print "$k)\n***Post-HTML strip:\n$message\n***\n";}
$text_written=1;
} elsif ($alt_email_text=~/Content\-Type\:\s+text\/plain/i && $text_written==0) {
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/plain. Stripping headers to get text...\n";}
$text_written=1;
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
}
## Check for attachments
if ($sub_content_disposition=~/attachment/) {
$attachment_fulltype="";
$attachment_filename="";
## Check to see if attachment is an accepted filetype
$sub_content_type=~/Content-Type\:\s+[^\;]+\;/i; $attachment_type=$&;
$attachment_type=~s/(^Content-Type\:\s+|\;$)//gi;
$attachment_type=~s/[\r\n]//g;
switch ($attachment_type) {
case "application/pdf" {$attachment_fulltype="PDF";}
case "image/jpeg" {$attachment_fulltype="JPG";}
case "image/jpg" {$attachment_fulltype="JPG";}
case "image/gif" {$attachment_fulltype="GIF";}
case "image/x-png" {$attachment_fulltype="PNG";}
case "image/png" {$attachment_fulltype="PNG";}
case "image/bmp" {$attachment_fulltype="BMP";}
case "image/x-ms-bmp" {$attachment_fulltype="BMP";}
case "application/msword" {$attachment_fulltype="DOC";}
case "application/rtf" {$attachment_fulltype="RTF";}
case "application/vnd.ms-powerpoint" {$attachment_fulltype="PPT";}
case "application/vnd.ms-excel" {$attachment_fulltype="XLS";}
case "application/x-msexcel" {$attachment_fulltype="XLS";}
case "application/ms-excel" {$attachment_fulltype="XLS";}
case "application/zip" {$attachment_fulltype="ZIP";}
case "text/csv" {$attachment_fulltype="CSV";}
case "text/plain" {$attachment_fulltype="TXT";}
case "application/vnd.oasis.opendocument.text" {$attachment_fulltype="ODT";}
case "application/vnd.oasis.opendocument.spreadsheet" {$attachment_fulltype="ODS";}
}
if ($sub_content_disposition=~/filename\=\"?(.*?)\"?$/i) {$attachment_filename=$&;}
if (length($attachment_filename)==0) {
if ($ARGV[0]=~/debug/i) {print "Couldn't find file name with content-disposition. Searching full header for 'filename' value...\n";}
if ($alt_email_text=~/filename\=\"?(.*?)\"?$/i) {$attachment_filename=$&;}
if (length($attachment_filename)==0) {
if ($ARGV[0]=~/debug/i) {print "Couldn't find file name anywhere in header. Searching Content-Type for 'name' value...\n";}
if ($sub_content_type=~/name\=\"?(.*?)\"?$/i) {$attachment_filename=$&;}
}
}
$attachment_filename=~s/(file)?name\=\"?|\"?$//gi;
# If the attachment check is good, then proceed with storing the attachment. Otherwise continue.
if ($attachment_fulltype && $attachment_fulltype ne "") {
if ($attachment_filename && $attachment_filename ne "") {
if ($ARGV[0]=~/debug/i) {print "Found valid attachment, type $attachment_type \n Attachment filename is $attachment_filename ($attachment_fulltype)\nFile can be stored in database....\n";}
$ins_values[$attach_ct]="'$attachment_filename','$attachment_type','$encoding_type','$attachment_filesize','$attachment_fulltype','$body_contents'";
$output_ins_values[$attach_ct]="'$attachment_filename','$attachment_type','$encoding_type','$attachment_filesize','$attachment_fulltype','<FILE CONTENTS>'";
$attach_ct++;
} else {
if ($ARGV[0]=~/debug/i) {print "!!!!WARNING - Found valid attachment, type $attachment_type \n Attachment does not have file name. Skipping...\n";}
}
} else {
if ($ARGV[0]=~/debug/i) {print "!!!!WARNING - Found attachment $attachment_filename, but is NOT a valid file type \n Attachment type is $attachment_type. Skipping...\n";}
}
}
# print "\n";
}
} else {
if ($ARGV[0]=~/debug/i) {print "!!!!WARNING - Mail is multi-part, but no boundary value was found. Email will be ignored.\n";}
}
}
#print "\n";
# Do a clean-and-covert on the message, to decode base64 messages and also to UTF8 decode quoted printable text, in order to pick up special characters
if ($content_transfer_encoding eq "base64") {
$message=decode_base64($message);
} elsif ($content_transfer_encoding eq "quoted-printable") {
UTFTextDecoder();
}
$message=~s/(\"|\||\'|\;)/\\$&/g;
$email_to=~s/(\"|\||\'|\;)/\\$&/g;
$email_from=~s/(\"|\||\'|\;)/\\$&/g;
$subject=~s/(\"|\||\'|\;)/\\$&/g;
$mime_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_transfer_encoding=~s/(\"|\||\'|\;)/\\$&/g;
$x_mailer=~s/(\"|\||\'|\;)/\\$&/g;
$sender_ip=~s/(\"|\||\'|\;)/\\$&/g;
$message=~s/\s{2,}/ /gi;
my $status="NEW";
### Parses the actual email address from the "Email From:" value in order to run it against vicidial_list
### Not sure how accurate this is
$email_from_name=$email_from;
$email_from_name=~s/\<?([^\s\@])+\@(([^\s\@\.])+\.)+[a-zA-Z]{2,}\>?//gi;
$email_from_name=~s/^\s*(.*?)\s*$/$1/;
$email_from=~/\<?([^\s\@])+\@(([^\s\@\.])+\.)+[a-zA-Z]{2,}\>?/i;
$email_from=$&;
$email_from=~s/(^\<)|(\>$)//gi;
$auth_results=~/([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){3}/i;
my $sender_ip=$&;
$spf=~/([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){3}/i;
my $sender_ip2=$&;
if (length($sender_ip)==0) {$sender_ip=$sender_ip2;}
$limit=1;
$call_handle_clause="";
if ($call_handle_method eq "EMAILLOOKUP") {
$call_handle_clause="where email='$email_from' order by lead_id desc";
} elsif ($call_handle_method eq "EMAILLOOKUPRL") {
$call_handle_clause="where email='$email_from' and list_id='$list_id' order by lead_id desc";
} elsif ($call_handle_method eq "EMAILLOOKUPRC") {
my $list_id_stmt="select list_id from vicidial_lists where campaign_id='$campaign_id'";
my $list_id_rslt=$dbhA->prepare($list_id_stmt);
$list_id_rslt->execute();
$list_id_str="";
while (@list_id_row=$list_id_rslt->fetchrow_array) {
$list_id_str.="$list_id_row[0],";
}
$list_id_str=substr($list_id_str, 0, -1);
$call_handle_clause="where email='$email_from' and list_id in ($list_id_str) order by lead_id desc";
} else {
$limit=0;
}
### CHECK if lead exists in vicidial_list table via a search by email based on the email account settings
my $vicidial_lead_check_stmt="select lead_id, list_id from vicidial_list $call_handle_clause limit $limit";
if ($ARGV[0]=~/debugX/i) {print $vicidial_lead_check_stmt;}
my $vicidial_lead_check_rslt=$dbhA->prepare($vicidial_lead_check_stmt);
$vicidial_lead_check_rslt->execute();
if ($vicidial_lead_check_rslt->rows>0) {
my @lead_id_row=$vicidial_lead_check_rslt->fetchrow_array;
$lead_id=$lead_id_row[0];
} else {
my $vicidial_list_stmt="insert into vicidial_list(list_id, email, comments, status) values('$default_list_id', '$email_from', '".substr($message,0,255)."', '$status')";
if ($ARGV[0]=~/debugX/i) {print $vicidial_list_stmt;}
my $vicidial_list_rslt=$dbhA->prepare($vicidial_list_stmt);
if ($vicidial_list_rslt->execute()) {
$lead_id=$dbhA->last_insert_id(undef, undef, 'vicidial_list', 'lead_id');
} else {
die "Vicidial list insert failed. Check SQL in:\n $vicidial_list_stmt\n";
}
}
## Insert a new record into vicidial_email_list. This is ALWAYS done for new email messages.
$ins_stmt="insert into vicidial_email_list(lead_id, protocol, email_date, email_to, email_from, email_from_name, subject, mime_type, content_type, content_transfer_encoding, x_mailer, sender_ip, message, email_account_id, group_id, status, direction) values('$lead_id', 'IMAP', STR_TO_DATE('$email_date', '%d %b %Y %T'), '$email_to', '$email_from', '$email_from_name', '$subject', '$mime_type', '$content_type', '$content_transfer_encoding', '$x_mailer', '$sender_ip', trim('$message'), '$VARemail_ID', '$VARemail_groupid', '$status', 'INBOUND')";
if ($ARGV[0]=~/debugX/i) {print $ins_stmt."\n";}
my $ins_rslt=$dbhA->prepare($ins_stmt);
if ($ins_rslt->execute()) {
if ($attach_ct>0) {
$email_id=$dbhA->last_insert_id(undef, undef, 'vicidial_email_list', 'email_row_id');
$multistmt="";
for ($k=0; $k<$attach_ct; $k++) {
$ins_values[$k]="('$email_id',$ins_values[$k])";
$multistmt.="$ins_values[$k],";
$output_ins_values[$k]="('$email_id',$output_ins_values[$k])";
$output_multistmt.="$output_ins_values[$k],";
}
$attachment_ins_stmt="insert into inbound_email_attachments(email_row_id, filename, file_type, file_encoding, file_size, file_extension, file_contents) VALUES ".substr($multistmt,0,-1);
$attachment_ins_rslt=$dbhA->prepare($attachment_ins_stmt);
$attachment_ins_rslt->execute();
$output_ins_stmt="insert into inbound_email_attachments(email_row_id, filename, file_type, file_encoding, file_size, file_extension, file_contents) VALUES ".substr($output_multistmt,0,-1);
if ($ARGV[0]=~/debugX/i) {print $output_ins_stmt."\n";}
}
if ($ARGV[0]=~/debug/i) {print "\n Found $attach_ct attachments in email\n";}
} else {
die "Email insert failed. Check SQL in:\n $ins_stmt\n";
}
}
}
close(STORAGE);
}
}
if ($ARGV[0]=~/debug/i) {
if ($ARGV[0]=~/debugX/i) {
print "Iteration #$q - found ".($new_messages+0)." new messages\n";
} else {
# print ".";
}
}
# sleep($sleep_time);
}
$client->logout();
} elsif ($VARemail_protocol eq "POP3") {
# if (!$sleep_time || $sleep_time<300) {$sleep_time=300;} # Some servers don't allow multiple connections within a certain time frame.
# if ($ARGV[0]=~/debug/) {$mail_check_iterations=5;} else {$mail_check_iterations=1000000;}
use Mail::POP3Client;
$pop = new Mail::POP3Client( USER => "$VARemail_user",
PASSWORD => "$VARemail_pwd",
HOST => "$VARemail_server",
PORT => 995,
USESSL => true,
)
or die "Cannot connect through POP3Client: $!";
if ($pop->Count()<0) {die "Error connecting to server. Please try again later.\n";}
for( $i = 1; $i <= $pop->Count(); $i++ ) {
foreach( $pop->Head( $i ) ) {
# print $_."\n";
if ($_=~/^(To|From|Date|Subject|MIME-Version|Content-Type|Content-Transfer-Encoding|X-Mailer|Authentication-Results|Message|Body):\s+/i) {
$value=$_;
$ptn=$&;
$value=~s/$ptn//i;
$ptn=~s/^\s*(.*?)\s*$/$1/;
$ptn=~s/:$//;
$ptn=~s/\-/_/gi;
$varname=lc($ptn);
$$varname=$value;
}
if ($_=~/boundary\=\"?[^\"]+\"?/i) {
$bkup_boundary=$&;
}
}
#Rename variables so they aren't as vague.
$email_to=$to;
$email_from=$from;
$message=$pop->Body( $i );
$text_written=0; ## Keeps track of whether or not text of email was grabbed
$attach_ct=0; ## Keeps number
@ins_values=();
@output_ins_values=();
if ($content_type=~/^text\/plain/i) {
## Do nothing - it's plain text and needs no further work on it.
if ($ARGV[0]=~/debug/i) {print "Email message is text/plain. Nothing needs to be done.\n\n";}
} elsif ($content_type=~/^text\/html/i) {
## Message is HTML, so it needs to be stripped.
if ($ARGV[0]=~/debug/i) {print "Email message is text/html. Needs to have tags stripped via HTML::Strip.\n\n";}
StripHTML();
} elsif ($content_type=~/^multipart\/(alternative|mixed)/i) {
## Message is multipart/alternative, so it needs to be read and partitioned. The multipart/alternative subtype indicates that each part is an "alternative"
## version of the same (or similar) content, each in a different format denoted by its "Content-Type" header. The formats are ordered by how faithful they are to
## the original, with the least faithful first and the most faithful last. Systems can then choose the "best" representation they are capable of processing; in
## general, this will be the last part that the system can understand, although other factors may affect this.
if ($ARGV[0]=~/debug/i) {print "Email message is multipart. Need to select the best format type and parse it.\n";}
## First, get the boundary from the content-type and use it to break the email into it's multiple parts
if ($content_type=~/boundary\=\"?[^\"]+\"?/i) {
$boundary=$&;
} else {
$boundary=$bkup_boundary;
}
if (length($boundary)>0) {
$boundary=~s/(^boundary=\"?|\"?$)//gi;
$boundary="--".$boundary;
@alternatives_array=split(/$boundary/, $message);
for ($k=1; $k<(scalar(@alternatives_array)-1); $k++) { # Ignore the first and last entry in the array; it's just a blank entry and the two dashes after the last boundary.
## Grab the message by using Mail::Message to strip header information
$alt_email_text=$alternatives_array[$k];
$alt_email_text=~s/(^(\s|\r|\n)+|(\s|\r|\n)+$)//; # This needs to be trimmed so the Mail::Message->read command will strip out the headers - leading carriage returns ruin it
# Reset certain variables;
$sub_content_disposition="";
$sub_content_type="";
$attachment_fulltype="";
$attachment_filename="";
$attachment_type="";
if ($alt_email_text=~/Content\-Type\:\s+(.*?)\n/i) {$sub_content_type=$&;}
if ($alt_email_text=~/Content\-Disposition\:\s+(.*?)\n/i) {$sub_content_disposition=$&;}
if ($alt_email_text=~/Content\-Transfer\-Encoding\:\s+(.*?)\n/i) {
$encoding_type=$&;
$encoding_type=~s/(^Content-Transfer-Encoding\:\s+|\;$)//gi;
$encoding_type=~s/[\r\n]//g;
}
$sub_content_type=~s/[\r\n]//g;
$sub_content_disposition=~s/[\r\n]//g;
my $msg = Mail::Message->read($alt_email_text);
$body_contents=$msg->body;
$attachment_filesize=$msg->size();
$head=$msg->head;
if ($ARGV[0]=~/debugX/i) {print "Part content disposition is $sub_content_disposition\n Part content type is $sub_content_type.\n";}
if ($ARGV[0]=~/debugX/i) {print "Part content size is $attachment_filesize\n";}
## Check if the content-type is plain or html
if ($alt_email_text=~/Content\-Type\:\s+text\/html/i && $text_written==0) {
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/html. Stripping headers and stripping tags from the body/message...\n";}
if ($ARGV[0]=~/debugX/i) {print "$k)\n***Pre-HTML strip:\n$message\n***\n";}
StripHTML();
if ($ARGV[0]=~/debugX/i) {print "$k)\n***Post-HTML strip:\n$message\n***\n";}
$text_written=1;
} elsif ($alt_email_text=~/Content\-Type\:\s+text\/plain/i && $text_written==0) {
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/plain. Stripping headers to get text...\n";}
$text_written=1;
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
}
## Check for attachments
if ($sub_content_disposition=~/attachment/) {
$attachment_fulltype="";
$attachment_filename="";
## Check to see if attachment is an accepted filetype
$sub_content_type=~/Content-Type\:\s+[^\;]+\;/i; $attachment_type=$&;
$attachment_type=~s/(^Content-Type\:\s+|\;$)//gi;
$attachment_type=~s/[\r\n]//g;
switch ($attachment_type) {
case "application/pdf" {$attachment_fulltype="PDF";}
case "image/jpeg" {$attachment_fulltype="JPG";}
case "image/jpg" {$attachment_fulltype="JPG";}
case "image/gif" {$attachment_fulltype="GIF";}
case "image/x-png" {$attachment_fulltype="PNG";}
case "image/png" {$attachment_fulltype="PNG";}
case "image/bmp" {$attachment_fulltype="BMP";}
case "image/x-ms-bmp" {$attachment_fulltype="BMP";}
case "application/msword" {$attachment_fulltype="DOC";}
case "application/rtf" {$attachment_fulltype="RTF";}
case "application/vnd.ms-powerpoint" {$attachment_fulltype="PPT";}
case "application/vnd.ms-excel" {$attachment_fulltype="XLS";}
case "application/x-msexcel" {$attachment_fulltype="XLS";}
case "application/ms-excel" {$attachment_fulltype="XLS";}
case "application/zip" {$attachment_fulltype="ZIP";}
case "text/csv" {$attachment_fulltype="CSV";}
case "text/plain" {$attachment_fulltype="TXT";}
case "application/vnd.oasis.opendocument.text" {$attachment_fulltype="ODT";}
case "application/vnd.oasis.opendocument.spreadsheet" {$attachment_fulltype="ODS";}
}
if ($sub_content_disposition=~/filename\=\"?(.*?)\"?$/i) {$attachment_filename=$&;}
if (length($attachment_filename)==0) {
if ($ARGV[0]=~/debug/i) {print "Couldn't find file name with content-disposition. Searching full header for 'filename' value...\n";}
if ($alt_email_text=~/filename\=\"?(.*?)\"?$/i) {$attachment_filename=$&;}
if (length($attachment_filename)==0) {
if ($ARGV[0]=~/debug/i) {print "Couldn't find file name anywhere in header. Searching Content-Type for 'name' value...\n";}
if ($sub_content_type=~/name\=\"?(.*?)\"?$/i) {$attachment_filename=$&;}
}
}
$attachment_filename=~s/(file)?name\=\"?|\"?$//gi;
# If the attachment check is good, then proceed with storing the attachment. Otherwise continue.
if ($attachment_fulltype && $attachment_fulltype ne "") {
if ($attachment_filename && $attachment_filename ne "") {
$body_contents=~s/(\"|\||\'|\;)/\\$&/g; # This is a problem for POP3 (of course) in the attachment contents.
if ($ARGV[0]=~/debug/i) {print "Found valid attachment, type $attachment_type \n Attachment filename is $attachment_filename ($attachment_fulltype)\nFile can be stored in database....\n";}
$ins_values[$attach_ct]="'$attachment_filename','$attachment_type','$encoding_type','$attachment_filesize','$attachment_fulltype','$body_contents'";
$output_ins_values[$attach_ct]="'$attachment_filename','$attachment_type','$encoding_type','$attachment_filesize','$attachment_fulltype','<FILE CONTENTS>'";
$attach_ct++;
} else {
if ($ARGV[0]=~/debug/i) {print "!!!!WARNING - Found valid attachment, type $attachment_type \n Attachment does not have file name. Skipping...\n";}
}
} else {
if ($ARGV[0]=~/debug/i) {print "!!!!WARNING - Found attachment $attachment_filename, but is NOT a valid file type \n Attachment type is $attachment_type. Skipping...\n";}
}
}
# print "\n";
}
} else {
if ($ARGV[0]=~/debug/i) {print "!!!!WARNING - Mail is multi-part, but no boundary value was found. Email will be ignored.\n";}
}
}
# print "\n";
# Do a clean-and-covert on the message, to decode base64 messages and also to UTF8 decode quoted printable text, in order to pick up special characters
if ($content_transfer_encoding eq "base64") {
$message=decode_base64($message);
} elsif ($content_transfer_encoding eq "quoted-printable") {
UTFTextDecoder();
}
if ($date=~/[0-9]{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[0-9]{4}\s+[0-9]{1,2}\:[0-9]{1,2}(\:[0-9]{1,2})?/) {
$date=$&;
$date=~s/\s+/ /gi;
@date_array=split(/\s/, $date);
$day=substr("0".$date_array[0], -2);
$month=$date_array[1];
$year=$date_array[2];
@time_array=split(/\:/, $date_array[3]);
$hour=substr("00".$time_array[0], -2);
$min=substr("00".$time_array[1], -2);
$sec=substr("00".$time_array[2], -2);
$date="$day $month $year $hour:$min:$sec";
if ($ARGV[0]=~/debug/) {print "Time stamp on email is $date\n";}
} elsif ($ARGV[0]=~/debug/) {print "WARNING: Time stamp on email is $date\n";}
$message=~s/(\"|\||\'|\;)/\\$&/g;
$email_to=~s/(\"|\||\'|\;)/\\$&/g;
$email_from=~s/(\"|\||\'|\;)/\\$&/g;
$subject=~s/(\"|\||\'|\;)/\\$&/g;
$mime_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_transfer_encoding=~s/(\"|\||\'|\;)/\\$&/g;
$x_mailer=~s/(\"|\||\'|\;)/\\$&/g;
$sender_ip=~s/(\"|\||\'|\;)/\\$&/g;
$message=~s/\s{2,}/ /gi;
### Parses the actual email address from the "Email From:" value in order to run it against vicidial_list
### Not sure how accurate this is
$email_from_name=$email_from;
$email_from_name=~s/\<?([^\s\@])+\@(([^\s\@\.])+\.)+[a-zA-Z]{2,}\>?//gi;
$email_from_name=~s/^\s*(.*?)\s*$/$1/;
$email_from=~/\<?([^\s\@])+\@(([^\s\@\.])+\.)+[a-zA-Z]{2,}\>?/i;
$email_from=$&;
$email_from=~s/(^\<)|(\>$)//gi;
$status="NEW";
$authentication_results=~/([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){3}/i;
$sender_ip=$&;
$received_spf=~/([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){3}/i;
$sender_ip2=$&;
if (length($sender_ip)==0) {$sender_ip=$sender_ip2;}
$limit=1;
$call_handle_clause="";
if ($call_handle_method eq "EMAILLOOKUP") {
$call_handle_clause="where email='$email_from' order by lead_id desc";
} elsif ($call_handle_method eq "EMAILLOOKUPRL") {
$call_handle_clause="where email='$email_from' and list_id='$list_id' order by lead_id desc";
} elsif ($call_handle_method eq "EMAILLOOKUPRC") {
my $list_id_stmt="select list_id from vicidial_lists where campaign_id='$campaign_id'";
my $list_id_rslt=$dbhA->prepare($list_id_stmt);
$list_id_rslt->execute();
$list_id_str="";
while (@list_id_row=$list_id_rslt->fetchrow_array) {
$list_id_str.="$list_id_row[0],";
}
$list_id_str=substr($list_id_str, 0, -1);
$call_handle_clause="where email='$email_from' and list_id in ($list_id_str) order by lead_id desc";
} else {
$limit=0;
}
### CHECK if lead exists in vicidial_list table via a search by email based on the email account settings
my $vicidial_lead_check_stmt="select lead_id from vicidial_list $call_handle_clause limit $limit";
my $vicidial_lead_check_rslt=$dbhA->prepare($vicidial_lead_check_stmt);
$vicidial_lead_check_rslt->execute();
if ($vicidial_lead_check_rslt->rows>0) {
my @lead_id_row=$vicidial_lead_check_rslt->fetchrow_array;
$lead_id=$lead_id_row[0];
} else {
my $vicidial_list_stmt="insert into vicidial_list(list_id, email, comments, status) values('$default_list_id', '$email_from', '".substr($message,0,255)."', '$status')";
if ($ARGV[0]=~/debugX/i) {print $vicidial_list_stmt."\n";}
my $vicidial_list_rslt=$dbhA->prepare($vicidial_list_stmt);
if ($vicidial_list_rslt->execute()) {
$lead_id=$dbhA->last_insert_id(undef, undef, 'vicidial_list', 'lead_id');
} else {
die "Vicidial list insert failed. Check SQL in:\n $vicidial_list_stmt\n";
}
}
## Insert a new record into vicidial_email_list. This is ALWAYS done for new email messages.
my $ins_stmt="insert into vicidial_email_list(lead_id, protocol, email_date, email_to, email_from, email_from_name, subject, mime_type, content_type, content_transfer_encoding, x_mailer, sender_ip, message, email_account_id, group_id, status, direction) values('$lead_id', 'POP3', STR_TO_DATE('$date', '%d %b %Y %T'), '$email_to', '$email_from', '$email_from_name', '$subject', '$mime_type', '$content_type', '$content_transfer_encoding', '$x_mailer', '$sender_ip', trim('$message'), '$VARemail_ID', '$VARemail_groupid', '$status', 'INBOUND')";
if ($ARGV[0]=~/debugX/i) {print $ins_stmt."\n";}
my $ins_rslt=$dbhA->prepare($ins_stmt);
if ($ins_rslt->execute()) {
$pop->Delete($i);
if ($attach_ct>0) {
$email_id=$dbhA->last_insert_id(undef, undef, 'vicidial_email_list', 'email_row_id');
$multistmt="";
for ($k=0; $k<$attach_ct; $k++) {
$ins_values[$k]="('$email_id',$ins_values[$k])";
$multistmt.="$ins_values[$k],";
$output_ins_values[$k]="('$email_id',$output_ins_values[$k])";
$output_multistmt.="$output_ins_values[$k],";
}
$attachment_ins_stmt="insert into inbound_email_attachments(email_row_id, filename, file_type, file_encoding, file_size, file_extension, file_contents) VALUES ".substr($multistmt,0,-1);
$attachment_ins_rslt=$dbhA->prepare($attachment_ins_stmt);
$attachment_ins_rslt->execute();
$output_ins_stmt="insert into inbound_email_attachments(email_row_id, filename, file_type, file_encoding, file_size, file_extension, file_contents) VALUES ".substr($output_multistmt,0,-1);
if ($ARGV[0]=~/debugX/i) {print $output_ins_stmt."\n";}
}
if ($ARGV[0]=~/debug/i) {print "\n $attach_ct attachments in email\n";}
} else {
die "Email insert failed. Check SQL in:\n $ins_stmt\n";
}
}
#### END CYCLING THROUGH EMAILS
if ($ARGV[0]=~/debug/i) {
if ($ARGV[0]=~/debugX/i) {
print "Iteration #$q - found ".($pop->Count()+0)." new messages\n";
}
}
if ($ARGV[0]=~/debugX/i) {
print "Closing connection to finalize deletion of emails\n";
}
$pop->Close();
# sleep($sleep_time);
if ($ARGV[0]=~/debugX/i) {
print "Reconnecting to email account...\n";
}
$pop = new Mail::POP3Client( USER => "$VARemail_user",
PASSWORD => "$VARemail_pwd",
HOST => "$VARemail_server",
PORT => 995,
USESSL => true,
#DEBUG => true,
)
or die "Cannot connect through POP3Client: $!";
$pop->Close();
} else {
die "No valid protocol specified for this program: ($VARemail_protocol) is the current protocol - needs to be IMAP or POP3.\n";
}
}
}
sub StripHTML()
{
$message=~s/^\s*(.*?)\s*$/$1/;
$message=~s/[\r\n]/ /g;
if ($message=~/<html.*?<\/html>/i) {
$message=$&;
my $hs = HTML::Strip->new();
$message = $hs->parse( $message );
$hs->eof;
}
}
sub UTFTextDecoder()
{
# Clean carriage returns preceded by an equal sign; these can throw off some of the decoding
$message=~s/\=(\r|\n)+//gi;
# Decode occurrences of two sets of characters together
@m = ($message=~/(\=[a-fA-F0-9][a-fA-F0-9]\=[a-fA-F0-9][a-fA-F0-9])/g );
for ($utf_ct=0; $utf_ct<scalar(@m); $utf_ct++) {
$char=$m[$utf_ct];
$char=~s/\=//gi;
$char=~ s/([a-fA-F0-9][a-fA-F0-9])/chr(hex($1))/eg;
$message =~ s/$m[$utf_ct]/$char/eg;
}
# Decode one set of characters together
@m = ($message=~/(\=[a-fA-F0-9][a-fA-F0-9])/g );
for ($utf_ct=0; $utf_ct<scalar(@m); $utf_ct++) {
$char=$m[$utf_ct];
$char=~s/\=//gi;
$char=~ s/([a-fA-F0-9][a-fA-F0-9])/chr(hex($1))/eg;
$message =~ s/$m[$utf_ct]/$char/eg;
}
}
Reference in New Issue View Git Blame Copy Permalink