ViciDial/ViciDialSVN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better support for non-latin email characters.

Browse Source 
new cpan module required: install MIME::Base64

git-svn-id: svn://192.168.202.10@1929 3d104415-ff17-0410-8863-d5cf3c621b8a
This commit is contained in:
mattf
2013-01-27 05:30:12 +00:00
parent 889c92ade1
commit 808089e8e4
6 changed files with 90 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agc_2-X/trunk/UPGRADE
+1
View File
@@ -148,6 +148,7 @@ OTHER CHANGES:
install Mail::IMAPClient
install Mail::Message
install IO::Socket::SSL
install MIME::Base64
Also, the inbound email parser must be added to the crontab of ONLY ONE
server:
### inbound email parser
agc_2-X/trunk/bin/AST_inbound_email_parser.pl
+71 -13
View File
@@ -6,8 +6,10 @@ use HTML::Strip;
use Switch;
use Time::Local;
use MIME::Decoder;
use Encode;
use MIME::Base64;
# Copyright (C) 2012 Joe Johnson <freewermadmin@gmail.com> LICENSE: AGPLv2
# Copyright (C) 2013 Matt Florell, Joe Johnson <vicidial@gmail.com> LICENSE: AGPLv2
#
# AST_inbound_email_parser.pl - This script is essential for periodically checking any active POP3 or IMAP
# email accounts set up through the Vicidial admin.
@@ -50,6 +52,7 @@ use MIME::Decoder;
#
# changes:
# 121213-2200 - First Build
# 130127-0025 - Added better non-latin characters support
#
# default path to astguiclient configuration file:
@@ -184,7 +187,7 @@ while (@row=$rslt->fetchrow_array) {
Port => 993,
Ssl => 1,
)
or die "Cannot connect through IMAPClient: $@";
or die "Cannot connect through IMAPClient: $!";
# Include options for folders in IMAP? POP3 doesn't support this.
@@ -198,12 +201,17 @@ while (@row=$rslt->fetchrow_array) {
my $msgcount = $client->message_count($folder_array[$i]);
#print "+---+ Messages: ".$msgcount."\n";
if ($msgcount>0) {
$file=open(STORAGE, "> ./raw_emails.txt");
$client->select($folder_array[$i]);
my @msgs = $client->messages or die "Could not messages: $@\n";
my @unseenMsgs = $client->unseen;
for(my $j=0; $j<scalar(@msgs); $j++) {
if (grep {$_ eq $msgs[$j]} @unseenMsgs) {
$new_messages++;
my $string = $client->message_string($msgs[$j]) or die "Could not message_string: $@\n";
$client->message_to_file($file,$msgs[$j]) or die "Could not message_to_file: $@\n";
my $hashref = $client->parse_headers($msgs[$j], 'ALL');
my %email_values=%{$hashref};
my $email_to=$email_values{"To"}->[0];
@@ -212,10 +220,14 @@ while (@row=$rslt->fetchrow_array) {
my $subject=$email_values{"Subject"}->[0];
my $mime_type=$email_values{"MIME-Version"}->[0];
my $content_type=$email_values{"Content-Type"}->[0];
my $content_transfer_encoding=$email_values{"Content-Transfer-Encoding"}->[0];
my $x_mailer=$email_values{"X-Mailer"}->[0];
my $auth_results=$email_values{"Authentication-Results"}->[0];
my $spf=$email_values{"Received-SPF"}->[0];
$message = $client->body_string($msgs[$j]);
print "\n############\n$content_transfer_encoding - $content_type - $message\n###########\n";
$text_written=0; ## Keeps track of whether or not text of email was grabbed
$attach_ct=0; ## Keeps number
@ins_values=();
@@ -236,7 +248,6 @@ while (@row=$rslt->fetchrow_array) {
if ($ARGV[0]=~/debug/) {print "Time stamp on email is $email_date\n";}
} elsif ($ARGV[0]=~/debug/) {print "WARNING: Time stamp on email is $email_date\n";}
############# MESSAGE ACTIONS
if ($content_type=~/^text\/plain/i) {
## Do nothing - it's plain text and needs no further work on it.
@@ -293,6 +304,7 @@ while (@row=$rslt->fetchrow_array) {
## Check if the content-type is plain or html
if ($alt_email_text=~/Content\-Type\:\s+text\/html/i && $text_written==0) {
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/html. Stripping headers and stripping tags from the body/message...\n";}
if ($ARGV[0]=~/debugX/i) {print "$k)\n***Pre-HTML strip:\n$message\n***\n";}
StripHTML();
@@ -302,6 +314,7 @@ while (@row=$rslt->fetchrow_array) {
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/plain. Stripping headers to get text...\n";}
$text_written=1;
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
}
## Check for attachments
@@ -368,12 +381,20 @@ while (@row=$rslt->fetchrow_array) {
}
#print "\n";
# Do a clean-and-covert on the message, to decode base64 messages and also to UTF8 decode quoted printable text, in order to pick up special characters
if ($content_transfer_encoding eq "base64") {
$message=decode_base64($message);
} elsif ($content_transfer_encoding eq "quoted-printable") {
UTFTextDecoder();
}
$message=~s/(\"|\||\'|\;)/\\$&/g;
$email_to=~s/(\"|\||\'|\;)/\\$&/g;
$email_from=~s/(\"|\||\'|\;)/\\$&/g;
$subject=~s/(\"|\||\'|\;)/\\$&/g;
$mime_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_transfer_encoding=~s/(\"|\||\'|\;)/\\$&/g;
$x_mailer=~s/(\"|\||\'|\;)/\\$&/g;
$sender_ip=~s/(\"|\||\'|\;)/\\$&/g;
$message=~s/\s{2,}/ /gi;
@@ -435,7 +456,7 @@ while (@row=$rslt->fetchrow_array) {
}
## Insert a new record into vicidial_email_list. This is ALWAYS done for new email messages.
$ins_stmt="insert into vicidial_email_list(lead_id, protocol, email_date, email_to, email_from, email_from_name, subject, mime_type, content_type, x_mailer, sender_ip, message, email_account_id, group_id, status, direction) values('$lead_id', 'IMAP', STR_TO_DATE('$email_date', '%d %b %Y %T'), '$email_to', '$email_from', '$email_from_name', '$subject', '$mime_type', '$content_type', '$x_mailer', '$sender_ip', trim('$message'), '$VARemail_ID', '$VARemail_groupid', '$status', 'INBOUND')";
$ins_stmt="insert into vicidial_email_list(lead_id, protocol, email_date, email_to, email_from, email_from_name, subject, mime_type, content_type, content_transfer_encoding, x_mailer, sender_ip, message, email_account_id, group_id, status, direction) values('$lead_id', 'IMAP', STR_TO_DATE('$email_date', '%d %b %Y %T'), '$email_to', '$email_from', '$email_from_name', '$subject', '$mime_type', '$content_type', '$content_transfer_encoding', '$x_mailer', '$sender_ip', trim('$message'), '$VARemail_ID', '$VARemail_groupid', '$status', 'INBOUND')";
if ($ARGV[0]=~/debugX/i) {print $ins_stmt."\n";}
my $ins_rslt=$dbhA->prepare($ins_stmt);
@@ -464,6 +485,7 @@ while (@row=$rslt->fetchrow_array) {
}
}
close(STORAGE);
}
}
if ($ARGV[0]=~/debug/i) {
@@ -485,20 +507,20 @@ while (@row=$rslt->fetchrow_array) {
HOST => "$VARemail_server",
PORT => 995,
USESSL => true,
DEBUG => true,
)
or die "Cannot connect through POP3Client: $!";
if ($pop->Count()<0) {die "Error connecting to server. Please try again later.\n";}
for( $i = 1; $i <= $pop->Count(); $i++ ) {
foreach( $pop->Head( $i ) ) {
if ($_=~/^(To|From|Date|Subject|MIME-Version|Content-Type|X-Mailer|Authentication-Results|Received-SPF|Message|Body):\s+/i) {
# print $_."\n";
if ($_=~/^(To|From|Date|Subject|MIME-Version|Content-Type|Content-Transfer-Encoding|X-Mailer|Authentication-Results|Message|Body):\s+/i) {
$value=$_;
$ptn=$&;
$value=~s/$ptn//i;
$ptn=~s/^\s*(.*?)\s*$/$1/;
$ptn=~s/:$//;
$ptn=~s/\-/_/i;
$ptn=~s/\-/_/gi;
$varname=lc($ptn);
$$varname=$value;
}
@@ -506,6 +528,10 @@ while (@row=$rslt->fetchrow_array) {
$bkup_boundary=$&;
}
}
#Rename variables so they aren't as vague.
$email_to=$to;
$email_from=$from;
$message=$pop->Body( $i );
$text_written=0; ## Keeps track of whether or not text of email was grabbed
$attach_ct=0; ## Keeps number
@@ -571,6 +597,7 @@ while (@row=$rslt->fetchrow_array) {
## Check if the content-type is plain or html
if ($alt_email_text=~/Content\-Type\:\s+text\/html/i && $text_written==0) {
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/html. Stripping headers and stripping tags from the body/message...\n";}
if ($ARGV[0]=~/debugX/i) {print "$k)\n***Pre-HTML strip:\n$message\n***\n";}
StripHTML();
@@ -580,6 +607,7 @@ while (@row=$rslt->fetchrow_array) {
if ($ARGV[0]=~/debug/i) {print "First acceptable content-type match is text/plain. Stripping headers to get text...\n";}
$text_written=1;
$message=$body_contents;
$content_transfer_encoding=$encoding_type;
}
## Check for attachments
@@ -647,6 +675,13 @@ while (@row=$rslt->fetchrow_array) {
}
# print "\n";
# Do a clean-and-covert on the message, to decode base64 messages and also to UTF8 decode quoted printable text, in order to pick up special characters
if ($content_transfer_encoding eq "base64") {
$message=decode_base64($message);
} elsif ($content_transfer_encoding eq "quoted-printable") {
UTFTextDecoder();
}
if ($date=~/[0-9]{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[0-9]{4}\s+[0-9]{1,2}\:[0-9]{1,2}(\:[0-9]{1,2})?/) {
$date=$&;
$date=~s/\s+/ /gi;
@@ -668,11 +703,10 @@ while (@row=$rslt->fetchrow_array) {
$subject=~s/(\"|\||\'|\;)/\\$&/g;
$mime_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_type=~s/(\"|\||\'|\;)/\\$&/g;
$content_transfer_encoding=~s/(\"|\||\'|\;)/\\$&/g;
$x_mailer=~s/(\"|\||\'|\;)/\\$&/g;
$sender_ip=~s/(\"|\||\'|\;)/\\$&/g;
$message=~s/\s{2,}/ /gi;
my $status="NEW";
$message=~s/\s{2,}/ /gi;
### Parses the actual email address from the "Email From:" value in order to run it against vicidial_list
### Not sure how accurate this is
@@ -720,7 +754,7 @@ while (@row=$rslt->fetchrow_array) {
$lead_id=$lead_id_row[0];
} else {
my $vicidial_list_stmt="insert into vicidial_list(list_id, email, comments, status) values('$default_list_id', '$email_from', '".substr($message,0,255)."', '$status')";
if ($ARGV[0]=~/debugX/i) {print $vicidial_list_stmt;}
if ($ARGV[0]=~/debugX/i) {print $vicidial_list_stmt."\n";}
my $vicidial_list_rslt=$dbhA->prepare($vicidial_list_stmt);
if ($vicidial_list_rslt->execute()) {
$lead_id=$dbhA->last_insert_id(undef, undef, 'vicidial_list', 'lead_id');
@@ -730,9 +764,8 @@ while (@row=$rslt->fetchrow_array) {
}
## Insert a new record into vicidial_email_list. This is ALWAYS done for new email messages.
my $ins_stmt="insert into vicidial_email_list(lead_id, protocol, email_date, email_to, email_from, email_from_name, subject, mime_type, content_type, x_mailer, sender_ip, message, email_account_id, group_id, status, direction) values('$lead_id', 'IMAP', STR_TO_DATE('$email_date', '%d %b %Y %T'), '$email_to', '$email_from', '$email_from_name', '$subject', '$mime_type', '$content_type', '$x_mailer', '$sender_ip', trim('$message'), '$VARemail_ID', '$VARemail_groupid', '$status', 'INBOUND')";
if ($ARGV[0]=~/debugX/i) {print $ins_stmt;}
my $ins_stmt="insert into vicidial_email_list(lead_id, protocol, email_date, email_to, email_from, email_from_name, subject, mime_type, content_type, content_transfer_encoding, x_mailer, sender_ip, message, email_account_id, group_id, status, direction) values('$lead_id', 'POP3', STR_TO_DATE('$date', '%d %b %Y %T'), '$email_to', '$email_from', '$email_from_name', '$subject', '$mime_type', '$content_type', '$content_transfer_encoding', '$x_mailer', '$sender_ip', trim('$message'), '$VARemail_ID', '$VARemail_groupid', '$status', 'INBOUND')";
if ($ARGV[0]=~/debugX/i) {print $ins_stmt."\n";}
my $ins_rslt=$dbhA->prepare($ins_stmt);
if ($ins_rslt->execute()) {
$pop->Delete($i);
@@ -799,3 +832,28 @@ sub StripHTML()
$hs->eof;
}
}
sub UTFTextDecoder()
{
# Clean carriage returns preceded by an equal sign; these can throw off some of the decoding
$message=~s/\=(\r|\n)+//gi;
# Decode occurrences of two sets of characters together
@m = ($message=~/(\=[a-fA-F0-9][a-fA-F0-9]\=[a-fA-F0-9][a-fA-F0-9])/g );
for ($utf_ct=0; $utf_ct<scalar(@m); $utf_ct++) {
$char=$m[$utf_ct];
$char=~s/\=//gi;
$char=~ s/([a-fA-F0-9][a-fA-F0-9])/chr(hex($1))/eg;
$message =~ s/$m[$utf_ct]/$char/eg;
}
# Decode one set of characters together
@m = ($message=~/(\=[a-fA-F0-9][a-fA-F0-9])/g );
for ($utf_ct=0; $utf_ct<scalar(@m); $utf_ct++) {
$char=$m[$utf_ct];
$char=~s/\=//gi;
$char=~ s/([a-fA-F0-9][a-fA-F0-9])/chr(hex($1))/eg;
$message =~ s/$m[$utf_ct]/$char/eg;
}
}
agc_2-X/trunk/docs/Email_user_manual.txt
+1
View File
@@ -19,6 +19,7 @@ install Mail::POP3Client
install Mail::IMAPClient
install Mail::Message
install IO::Socket::SSL
install MIME::Base64
quit
agc_2-X/trunk/extras/MySQL_AST_CREATE_tables.sql
+5 -1
View File
@@ -2788,6 +2788,7 @@ email_from_name VARCHAR(255) DEFAULT NULL,
subject TEXT,
mime_type TEXT,
content_type TEXT,
content_transfer_encoding TEXT,
x_mailer TEXT,
sender_ip VARCHAR(25) DEFAULT NULL,
message TEXT,
@@ -2855,6 +2856,9 @@ KEY vicidial_email_log_lead_id_key (lead_id),
KEY vicidial_email_log_email_row_id_key (email_row_id)
);
ALTER TABLE vicidial_email_list MODIFY message text character set utf8;
ALTER TABLE vicidial_email_log MODIFY message text character set utf8;
ALTER TABLE vicidial_qc_codes ADD qc_result_type ENUM( 'PASS', 'FAIL', 'CANCEL', 'COMMIT' ) NOT NULL;
@@ -3090,4 +3094,4 @@ UPDATE vicidial_configuration set value='1766' where name='qc_database_version';
UPDATE system_settings set vdc_agent_api_active='1';
UPDATE system_settings SET db_schema_version='1338',db_schema_update_date=NOW();
UPDATE system_settings SET db_schema_version='1339',db_schema_update_date=NOW();
agc_2-X/trunk/extras/upgrade_2.6.sql
+7
View File
@@ -296,3 +296,10 @@ CREATE INDEX pvmid on phones (voicemail_id);
CREATE INDEX pdpn on phones (dialplan_number);
UPDATE system_settings SET db_schema_version='1338',db_schema_update_date=NOW() where db_schema_version < 1338;
ALTER TABLE vicidial_email_list ADD COLUMN content_transfer_encoding TEXT AFTER content_type;
ALTER TABLE vicidial_email_list MODIFY message text character set utf8;
ALTER TABLE vicidial_email_log MODIFY message text character set utf8;
UPDATE system_settings SET db_schema_version='1339',db_schema_update_date=NOW() where db_schema_version < 1339;
agc_2-X/trunk/www/agc/vdc_email_display.php
+5 -3
View File
@@ -1,7 +1,7 @@
<?php
# vdc_email_display.php - VICIDIAL administration page
#
# Copyright (C) 2012 Joe Johnson <freewermadmin@gmail.com> LICENSE: AGPLv2
# Copyright (C) 2013 Matt Florell, Joe Johnson <vicidial@gmail.com> LICENSE: AGPLv2
#
# This page displays any incoming emails in the Vicidial user interface. It
# also allows the user to download and view any attachments sent in the email,
@@ -11,6 +11,7 @@
#
# changes:
# 121214-2300 - First Build
# 130127-0027 - Better non-latin characters support
#
require("dbconnect.php");
@@ -172,7 +173,7 @@ if ($REPLY)
$semi_rand = md5(time());
$mime_boundary = "==Multipart_Boundary_x{$semi_rand}x";
$headers .= "\nMIME-Version: 1.0\n" . "Content-Type: multipart/mixed;\n" . " boundary=\"{$mime_boundary}\"";
$message = "This is a multi-part message in MIME format.\n\n" . "--{$mime_boundary}\n" . "Content-Type: text/plain; charset=\"iso-8859-1\"\n" . "Content-Transfer-Encoding: 7bit\n\n" . $message . "\n\n";
$message = "This is a multi-part message in MIME format.\n\n" . "--{$mime_boundary}\n" . "Content-Type: text/plain; charset=\"utf-8\"\n" . "Content-Transfer-Encoding: 7bit\n\n" . $message . "\n\n";
$message .= "--{$mime_boundary}\n";
for ($i=1; $i<=5; $i++)
@@ -343,9 +344,10 @@ if ($lead_id) {
else
{
full_msg+=msg_line+"\n";
msg_line="> ";
msg_line="> "+msg_array[i]+" ";
}
}
full_msg+=msg_line+"\n";
var email_field_value=document.getElementById("reply_message").value+"\n";
email_field_value+=full_msg;
document.getElementById("reply_message").value=email_field_value;