Add inbound mail handling

Add normalized event, signal, and webhooks for inbound mail.

Closes #43
Closes #86
This commit is contained in:
Mike Edmunds
2018-02-02 10:38:53 -08:00
committed by GitHub
parent c924c9ec03
commit b57eb94f64
35 changed files with 2968 additions and 130 deletions

View File

@@ -1,4 +1,3 @@
import re
import warnings
import six
@@ -128,6 +127,9 @@ class AnymailBaseWebhookView(AnymailBasicAuthMixin, View):
"""
Read-only name of the ESP for this webhook view.
(E.g., MailgunTrackingWebhookView will return "Mailgun")
Subclasses must override with class attr. E.g.:
esp_name = "Postmark"
esp_name = "SendGrid" # (use ESP's preferred capitalization)
"""
return re.sub(r'(Tracking|Inbox)WebhookView$', "", self.__class__.__name__)
raise NotImplementedError("%s.%s must declare esp_name class attr" %
(self.__class__.__module__, self.__class__.__name__))

View File

@@ -8,13 +8,15 @@ from django.utils.timezone import utc
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailWebhookValidationFailure
from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
from ..utils import get_anymail_setting, combine, querydict_getfirst
class MailgunBaseWebhookView(AnymailBaseWebhookView):
"""Base view class for Mailgun webhooks"""
esp_name = "Mailgun"
warn_if_no_basic_auth = False # because we validate against signature
api_key = None # (Declaring class attr allows override by kwargs in View.as_view.)
@@ -40,12 +42,6 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView):
if not constant_time_compare(signature, expected_signature):
raise AnymailWebhookValidationFailure("Mailgun webhook called with incorrect signature")
def parse_events(self, request):
return [self.esp_to_anymail_event(request.POST)]
def esp_to_anymail_event(self, esp_event):
raise NotImplementedError()
class MailgunTrackingWebhookView(MailgunBaseWebhookView):
"""Handler for Mailgun delivery and engagement tracking webhooks"""
@@ -75,6 +71,9 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
607: RejectReason.SPAM, # previous spam complaint
}
def parse_events(self, request):
return [self.esp_to_anymail_event(request.POST)]
def esp_to_anymail_event(self, esp_event):
# esp_event is a Django QueryDict (from request.POST),
# which has multi-valued fields, but is *not* case-insensitive.
@@ -194,3 +193,69 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
'opened': _common_event_fields,
'unsubscribed': _common_event_fields,
}
class MailgunInboundWebhookView(MailgunBaseWebhookView):
"""Handler for Mailgun inbound (route forward-to-url) webhook"""
signal = inbound
def parse_events(self, request):
return [self.esp_to_anymail_event(request)]
def esp_to_anymail_event(self, request):
# Inbound uses the entire Django request as esp_event, because we need POST and FILES.
# Note that request.POST is case-sensitive (unlike email.message.Message headers).
esp_event = request
if 'body-mime' in request.POST:
# Raw-MIME
message = AnymailInboundMessage.parse_raw_mime(request.POST['body-mime'])
else:
# Fully-parsed
message = self.message_from_mailgun_parsed(request)
message.envelope_sender = request.POST.get('sender', None)
message.envelope_recipient = request.POST.get('recipient', None)
message.stripped_text = request.POST.get('stripped-text', None)
message.stripped_html = request.POST.get('stripped-html', None)
message.spam_detected = message.get('X-Mailgun-Sflag', 'No').lower() == 'yes'
try:
message.spam_score = float(message['X-Mailgun-Sscore'])
except (TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=datetime.fromtimestamp(int(request.POST['timestamp']), tz=utc),
event_id=request.POST.get('token', None),
esp_event=esp_event,
message=message,
)
def message_from_mailgun_parsed(self, request):
"""Construct a Message from Mailgun's "fully-parsed" fields"""
# Mailgun transcodes all fields to UTF-8 for "fully parsed" messages
try:
attachment_count = int(request.POST['attachment-count'])
except (KeyError, TypeError):
attachments = None
else:
# Load attachments from posted files: Mailgun file field names are 1-based
att_ids = ['attachment-%d' % i for i in range(1, attachment_count+1)]
att_cids = { # filename: content-id (invert content-id-map)
att_id: cid for cid, att_id
in json.loads(request.POST.get('content-id-map', '{}')).items()
}
attachments = [
AnymailInboundMessage.construct_attachment_from_uploaded_file(
request.FILES[att_id], content_id=att_cids.get(att_id, None))
for att_id in att_ids
]
return AnymailInboundMessage.construct(
headers=json.loads(request.POST['message-headers']), # includes From, To, Cc, Subject, etc.
text=request.POST.get('body-plain', None),
html=request.POST.get('body-html', None),
attachments=attachments,
)

View File

@@ -4,12 +4,14 @@ from datetime import datetime
from django.utils.timezone import utc
from .base import AnymailBaseWebhookView
from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
class MailjetTrackingWebhookView(AnymailBaseWebhookView):
"""Handler for Mailjet delivery and engagement tracking webhooks"""
esp_name = "Mailjet"
signal = tracking
def parse_events(self, request):
@@ -95,3 +97,84 @@ class MailjetTrackingWebhookView(AnymailBaseWebhookView):
user_agent=esp_event.get('agent', None),
esp_event=esp_event,
)
class MailjetInboundWebhookView(AnymailBaseWebhookView):
"""Handler for Mailjet inbound (parse API) webhook"""
esp_name = "Mailjet"
signal = inbound
def parse_events(self, request):
esp_event = json.loads(request.body.decode('utf-8'))
return [self.esp_to_anymail_event(esp_event)]
def esp_to_anymail_event(self, esp_event):
# You could _almost_ reconstruct the raw mime message from Mailjet's Headers and Parts fields,
# but it's not clear which multipart boundary to use on each individual Part. Although each Part's
# Content-Type header still has the multipart boundary, not knowing the parent part means typical
# nested multipart structures can't be reliably recovered from the data Mailjet provides.
# We'll just use our standarized multipart inbound constructor.
headers = self._flatten_mailjet_headers(esp_event.get("Headers", {}))
attachments = [
self._construct_mailjet_attachment(part, esp_event)
for part in esp_event.get("Parts", [])
if "Attachment" in part.get("ContentRef", "") # Attachment<N> or InlineAttachment<N>
]
message = AnymailInboundMessage.construct(
headers=headers,
text=esp_event.get("Text-part", None),
html=esp_event.get("Html-part", None),
attachments=attachments,
)
message.envelope_sender = esp_event.get("Sender", None)
message.envelope_recipient = esp_event.get("Recipient", None)
message.spam_detected = None # Mailjet doesn't provide a boolean; you'll have to interpret spam_score
try:
message.spam_score = float(esp_event['SpamAssassinScore'])
except (KeyError, TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # Mailjet doesn't provide inbound event timestamp (esp_event['Date'] is time sent)
event_id=None, # Mailjet doesn't provide an idempotent inbound event id
esp_event=esp_event,
message=message,
)
@staticmethod
def _flatten_mailjet_headers(headers):
"""Convert Mailjet's dict-of-strings-and/or-lists header format to our list-of-name-value-pairs
{'name1': 'value', 'name2': ['value1', 'value2']}
--> [('name1', 'value'), ('name2', 'value1'), ('name2', 'value2')]
"""
result = []
for name, values in headers.items():
if isinstance(values, list): # Mailjet groups repeated headers together as a list of values
for value in values:
result.append((name, value))
else:
result.append((name, values)) # single-valued (non-list) header
return result
def _construct_mailjet_attachment(self, part, esp_event):
# Mailjet includes unparsed attachment headers in each part; it's easiest to temporarily
# attach them to a MIMEPart for parsing. (We could just turn this into the attachment,
# but we want to use the payload handling from AnymailInboundMessage.construct_attachment later.)
part_headers = AnymailInboundMessage() # temporary container for parsed attachment headers
for name, value in self._flatten_mailjet_headers(part.get("Headers", {})):
part_headers.add_header(name, value)
content_base64 = esp_event[part["ContentRef"]] # Mailjet *always* base64-encodes attachments
return AnymailInboundMessage.construct_attachment(
content_type=part_headers.get_content_type(),
content=content_base64, base64=True,
filename=part_headers.get_filename(None),
content_id=part_headers.get("Content-ID", "") or None,
)

View File

@@ -8,8 +8,9 @@ from django.utils.crypto import constant_time_compare
from django.utils.timezone import utc
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailWebhookValidationFailure, AnymailConfigurationError
from ..signals import tracking, AnymailTrackingEvent, EventType
from ..exceptions import AnymailWebhookValidationFailure
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType
from ..utils import get_anymail_setting, getfirst, get_request_uri
@@ -59,22 +60,34 @@ class MandrillSignatureMixin(object):
"Mandrill webhook called with incorrect signature (for url %r)" % url)
class MandrillBaseWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView):
"""Base view class for Mandrill webhooks"""
class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView):
"""Unified view class for Mandrill tracking and inbound webhooks"""
esp_name = "Mandrill"
warn_if_no_basic_auth = False # because we validate against signature
signal = None # set in esp_to_anymail_event
def parse_events(self, request):
esp_events = json.loads(request.POST['mandrill_events'])
return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
def esp_to_anymail_event(self, esp_event):
raise NotImplementedError()
"""Route events to the inbound or tracking handler"""
esp_type = getfirst(esp_event, ['event', 'type'], 'unknown')
if esp_type == 'inbound':
assert self.signal is not tracking # Mandrill should never mix event types in the same batch
self.signal = inbound
return self.mandrill_inbound_to_anymail_event(esp_event)
else:
assert self.signal is not inbound # Mandrill should never mix event types in the same batch
self.signal = tracking
return self.mandrill_tracking_to_anymail_event(esp_event)
class MandrillTrackingWebhookView(MandrillBaseWebhookView):
signal = tracking
#
# Tracking events
#
event_types = {
# Message events:
@@ -94,13 +107,9 @@ class MandrillTrackingWebhookView(MandrillBaseWebhookView):
'inbound': EventType.INBOUND,
}
def esp_to_anymail_event(self, esp_event):
def mandrill_tracking_to_anymail_event(self, esp_event):
esp_type = getfirst(esp_event, ['event', 'type'], None)
event_type = self.event_types.get(esp_type, EventType.UNKNOWN)
if event_type == EventType.INBOUND:
raise AnymailConfigurationError(
"You seem to have set Mandrill's *inbound* webhook URL "
"to Anymail's Mandrill *tracking* webhook URL.")
try:
timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc)
@@ -149,3 +158,33 @@ class MandrillTrackingWebhookView(MandrillBaseWebhookView):
timestamp=timestamp,
user_agent=esp_event.get('user_agent', None),
)
#
# Inbound events
#
def mandrill_inbound_to_anymail_event(self, esp_event):
# It's easier (and more accurate) to just work from the original raw mime message
message = AnymailInboundMessage.parse_raw_mime(esp_event['msg']['raw_msg'])
message.envelope_sender = None # (Mandrill's 'sender' field only applies to outbound messages)
message.envelope_recipient = esp_event['msg'].get('email', None)
message.spam_detected = None # no simple boolean field; would need to parse the spam_report
message.spam_score = esp_event['msg'].get('spam_report', {}).get('score', None)
try:
timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc)
except (KeyError, ValueError):
timestamp = None
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=timestamp,
event_id=None, # Mandrill doesn't provide an idempotent inbound message event id
esp_event=esp_event,
message=message,
)
# Backwards-compatibility: earlier Anymail versions had only MandrillTrackingWebhookView:
MandrillTrackingWebhookView = MandrillCombinedWebhookView

View File

@@ -4,13 +4,16 @@ from django.utils.dateparse import parse_datetime
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailConfigurationError
from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
from ..utils import getfirst
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
from ..utils import getfirst, EmailAddress
class PostmarkBaseWebhookView(AnymailBaseWebhookView):
"""Base view class for Postmark webhooks"""
esp_name = "Postmark"
def parse_events(self, request):
esp_event = json.loads(request.body.decode('utf-8'))
return [self.esp_to_anymail_event(esp_event)]
@@ -107,3 +110,72 @@ class PostmarkTrackingWebhookView(PostmarkBaseWebhookView):
user_agent=esp_event.get('UserAgent', None),
click_url=esp_event.get('OriginalLink', None),
)
class PostmarkInboundWebhookView(PostmarkBaseWebhookView):
"""Handler for Postmark inbound webhook"""
signal = inbound
def esp_to_anymail_event(self, esp_event):
attachments = [
AnymailInboundMessage.construct_attachment(
content_type=attachment["ContentType"],
content=attachment["Content"], base64=True,
filename=attachment.get("Name", "") or None,
content_id=attachment.get("ContentID", "") or None,
)
for attachment in esp_event.get("Attachments", [])
]
message = AnymailInboundMessage.construct(
from_email=self._address(esp_event.get("FromFull")),
to=', '.join([self._address(to) for to in esp_event.get("ToFull", [])]),
cc=', '.join([self._address(cc) for cc in esp_event.get("CcFull", [])]),
# bcc? Postmark specs this for inbound events, but it's unclear how it could occur
subject=esp_event.get("Subject", ""),
headers=[(header["Name"], header["Value"]) for header in esp_event.get("Headers", [])],
text=esp_event.get("TextBody", ""),
html=esp_event.get("HtmlBody", ""),
attachments=attachments,
)
# Postmark strips these headers and provides them as separate event fields:
if "Date" in esp_event and "Date" not in message:
message["Date"] = esp_event["Date"]
if "ReplyTo" in esp_event and "Reply-To" not in message:
message["Reply-To"] = esp_event["ReplyTo"]
# Postmark doesn't have a separate envelope-sender field, but it can be extracted
# from the Received-SPF header that Postmark will have added:
if len(message.get_all("Received-SPF", [])) == 1: # (more than one? someone's up to something weird)
received_spf = message["Received-SPF"].lower()
if received_spf.startswith("pass") or received_spf.startswith("neutral"): # not fail/softfail
message.envelope_sender = message.get_param("envelope-from", None, header="Received-SPF")
message.envelope_recipient = esp_event.get("OriginalRecipient", None)
message.stripped_text = esp_event.get("StrippedTextReply", None)
message.spam_detected = message.get('X-Spam-Status', 'No').lower() == 'yes'
try:
message.spam_score = float(message['X-Spam-Score'])
except (TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # Postmark doesn't provide inbound event timestamp
event_id=esp_event.get("MessageID", None), # Postmark uuid, different from Message-ID mime header
esp_event=esp_event,
message=message,
)
@staticmethod
def _address(full):
"""Return an formatted email address from a Postmark inbound {From,To,Cc}Full dict"""
if full is None:
return ""
return str(EmailAddress(
display_name=full.get('Name', ""),
addr_spec=full.get("Email", ""),
))

View File

@@ -4,25 +4,20 @@ from datetime import datetime
from django.utils.timezone import utc
from .base import AnymailBaseWebhookView
from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
class SendGridBaseWebhookView(AnymailBaseWebhookView):
"""Base view class for SendGrid webhooks"""
class SendGridTrackingWebhookView(AnymailBaseWebhookView):
"""Handler for SendGrid delivery and engagement tracking webhooks"""
esp_name = "SendGrid"
signal = tracking
def parse_events(self, request):
esp_events = json.loads(request.body.decode('utf-8'))
return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
def esp_to_anymail_event(self, esp_event):
raise NotImplementedError()
class SendGridTrackingWebhookView(SendGridBaseWebhookView):
"""Handler for SendGrid delivery and engagement tracking webhooks"""
signal = tracking
event_types = {
# Map SendGrid event: Anymail normalized type
'bounce': EventType.BOUNCED,
@@ -120,3 +115,78 @@ class SendGridTrackingWebhookView(SendGridBaseWebhookView):
'url_offset', # click tracking
'useragent', # click/open tracking
}
class SendGridInboundWebhookView(AnymailBaseWebhookView):
"""Handler for SendGrid inbound webhook"""
esp_name = "SendGrid"
signal = inbound
def parse_events(self, request):
return [self.esp_to_anymail_event(request)]
def esp_to_anymail_event(self, request):
# Inbound uses the entire Django request as esp_event, because we need POST and FILES.
# Note that request.POST is case-sensitive (unlike email.message.Message headers).
esp_event = request
if 'headers' in request.POST:
# Default (not "Send Raw") inbound fields
message = self.message_from_sendgrid_parsed(esp_event)
elif 'email' in request.POST:
# "Send Raw" full MIME
message = AnymailInboundMessage.parse_raw_mime(request.POST['email'])
else:
raise KeyError("Invalid SendGrid inbound event data (missing both 'headers' and 'email' fields)")
try:
envelope = json.loads(request.POST['envelope'])
except (KeyError, TypeError, ValueError):
pass
else:
message.envelope_sender = envelope['from']
message.envelope_recipient = envelope['to'][0]
message.spam_detected = None # no simple boolean field; would need to parse the spam_report
try:
message.spam_score = float(request.POST['spam_score'])
except (KeyError, TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # SendGrid doesn't provide an inbound event timestamp
event_id=None, # SendGrid doesn't provide an idempotent inbound message event id
esp_event=esp_event,
message=message,
)
def message_from_sendgrid_parsed(self, request):
"""Construct a Message from SendGrid's "default" (non-raw) fields"""
try:
charsets = json.loads(request.POST['charsets'])
except (KeyError, ValueError):
charsets = {}
try:
attachment_info = json.loads(request.POST['attachment-info'])
except (KeyError, ValueError):
attachments = None
else:
# Load attachments from posted files
attachments = [
AnymailInboundMessage.construct_attachment_from_uploaded_file(
request.FILES[att_id],
content_id=attachment_info[att_id].get("content-id", None))
for att_id in sorted(attachment_info.keys())
]
return AnymailInboundMessage.construct(
raw_headers=request.POST.get('headers', ""), # includes From, To, Cc, Subject, etc.
text=request.POST.get('text', None),
text_charset=charsets.get('text', 'utf-8'),
html=request.POST.get('html', None),
html_charset=charsets.get('html', 'utf-8'),
attachments=attachments,
)

View File

@@ -1,16 +1,20 @@
import json
from base64 import b64decode
from datetime import datetime
from django.utils.timezone import utc
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailConfigurationError
from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
class SparkPostBaseWebhookView(AnymailBaseWebhookView):
"""Base view class for SparkPost webhooks"""
esp_name = "SparkPost"
def parse_events(self, request):
raw_events = json.loads(request.body.decode('utf-8'))
unwrapped_events = [self.unwrap_event(raw_event) for raw_event in raw_events]
@@ -92,7 +96,7 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
}
def esp_to_anymail_event(self, event_class, event, raw_event):
if event_class == 'relay_event':
if event_class == 'relay_message':
# This is an inbound event
raise AnymailConfigurationError(
"You seem to have set SparkPost's *inbound* relay webhook URL "
@@ -134,3 +138,37 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
user_agent=event.get('user_agent', None),
esp_event=raw_event,
)
class SparkPostInboundWebhookView(SparkPostBaseWebhookView):
"""Handler for SparkPost inbound relay webhook"""
signal = inbound
def esp_to_anymail_event(self, event_class, event, raw_event):
if event_class != 'relay_message':
# This is not an inbound event
raise AnymailConfigurationError(
"You seem to have set SparkPost's *tracking* webhook URL "
"to Anymail's SparkPost *inbound* relay webhook URL.")
if event['protocol'] != 'smtp':
raise AnymailConfigurationError(
"You cannot use Anymail's webhooks for SparkPost '{protocol}' relay events. "
"Anymail only handles the 'smtp' protocol".format(protocol=event['protocol']))
raw_mime = event['content']['email_rfc822']
if event['content']['email_rfc822_is_base64']:
raw_mime = b64decode(raw_mime).decode('utf-8')
message = AnymailInboundMessage.parse_raw_mime(raw_mime)
message.envelope_sender = event.get('msg_from', None)
message.envelope_recipient = event.get('rcpt_to', None)
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # SparkPost does not provide a relay event timestamp
event_id=None, # SparkPost does not provide an idempotent id for relay events
esp_event=raw_event,
message=message,
)