Add inbound mail handling

Add normalized event, signal, and webhooks for inbound mail.

Closes #43
Closes #86

anymail/inbound.py

@@ -0,0 +1,355 @@
+from base64 import b64decode
+from email import message_from_string
+from email.message import Message
+from email.utils import unquote
+
+import six
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from .utils import angle_wrap, get_content_disposition, parse_address_list, parse_rfc2822date
+
+# Python 2/3.*-compatible email.parser.HeaderParser(policy=email.policy.default)
+try:
+    # With Python 3.3+ (email6) package, can use HeaderParser with default policy
+    from email.parser import HeaderParser
+    from email.policy import default as accurate_header_unfolding_policy  # vs. compat32
+
+except ImportError:
+    # Earlier Pythons don't have HeaderParser, and/or try preserve earlier compatibility bugs
+    # by failing to properly unfold headers (see RFC 5322 section 2.2.3)
+    from email.parser import Parser
+    import re
+    accurate_header_unfolding_policy = object()
+
+    class HeaderParser(Parser, object):
+        def __init__(self, _class, policy=None):
+            # This "backport" doesn't actually support policies, but we want to ensure
+            # that callers aren't trying to use HeaderParser's default compat32 policy
+            # (which doesn't properly unfold headers)
+            assert policy is accurate_header_unfolding_policy
+            super(HeaderParser, self).__init__(_class)
+
+        def parsestr(self, text, headersonly=True):
+            unfolded = self._unfold_headers(text)
+            return super(HeaderParser, self).parsestr(unfolded, headersonly=True)
+
+        @staticmethod
+        def _unfold_headers(text):
+            # "Unfolding is accomplished by simply removing any CRLF that is immediately followed by WSP"
+            # (WSP is space or tab, and per email.parser semantics, we allow CRLF, CR, or LF endings)
+            return re.sub(r'(\r\n|\r|\n)(?=[ \t])', "", text)
+
+
+class AnymailInboundMessage(Message, object):  # `object` ensures new-style class in Python 2)
+    """
+    A normalized, parsed inbound email message.
+
+    A subclass of email.message.Message, with some additional
+    convenience properties, plus helpful methods backported
+    from Python 3.6+ email.message.EmailMessage (or really, MIMEPart)
+    """
+
+    # Why Python email.message.Message rather than django.core.mail.EmailMessage?
+    # Django's EmailMessage is really intended for constructing a (limited subset of)
+    # Message to send; Message is better designed for representing arbitrary messages:
+    #
+    # * Message is easily parsed from raw mime (which is an inbound format provided
+    #   by many ESPs), and can accurately represent any mime email that might be received
+    # * Message can represent repeated header fields (e.g., "Received") which
+    #   are common in inbound messages
+    # * Django's EmailMessage defaults a bunch of properties in ways that aren't helpful
+    #   (e.g., from_email from settings)
+
+    def __init__(self, *args, **kwargs):
+        # Note: this must accept zero arguments, for use with message_from_string (email.parser)
+        super(AnymailInboundMessage, self).__init__(*args, **kwargs)
+
+        # Additional attrs provided by some ESPs:
+        self.envelope_sender = None
+        self.envelope_recipient = None
+        self.stripped_text = None
+        self.stripped_html = None
+        self.spam_detected = None
+        self.spam_score = None
+
+    #
+    # Convenience accessors
+    #
+
+    @property
+    def from_email(self):
+        """EmailAddress """
+        # equivalent to Python 3.2+ message['From'].addresses[0]
+        from_email = self.get_address_header('From')
+        if len(from_email) == 1:
+            return from_email[0]
+        elif len(from_email) == 0:
+            return None
+        else:
+            return from_email  # unusual, but technically-legal multiple-From; preserve list
+
+    @property
+    def to(self):
+        """list of EmailAddress objects from To header"""
+        # equivalent to Python 3.2+ message['To'].addresses
+        return self.get_address_header('To')
+
+    @property
+    def cc(self):
+        """list of EmailAddress objects from Cc header"""
+        # equivalent to Python 3.2+ message['Cc'].addresses
+        return self.get_address_header('Cc')
+
+    @property
+    def subject(self):
+        """str value of Subject header, or None"""
+        return self['Subject']
+
+    @property
+    def date(self):
+        """datetime.datetime from Date header, or None if missing/invalid"""
+        # equivalent to Python 3.2+ message['Date'].datetime
+        return self.get_date_header('Date')
+
+    @property
+    def text(self):
+        """Contents of the (first) text/plain body part, or None"""
+        return self._get_body_content('text/plain')
+
+    @property
+    def html(self):
+        """Contents of the (first) text/html body part, or None"""
+        return self._get_body_content('text/html')
+
+    @property
+    def attachments(self):
+        """list of attachments (as MIMEPart objects); excludes inlines"""
+        return [part for part in self.walk() if part.is_attachment()]
+
+    @property
+    def inline_attachments(self):
+        """dict of Content-ID: attachment (as MIMEPart objects)"""
+        return {unquote(part['Content-ID']): part for part in self.walk()
+                if part.is_inline_attachment() and part['Content-ID']}
+
+    def get_address_header(self, header):
+        """Return the value of header parsed into a (possibly-empty) list of EmailAddress objects"""
+        values = self.get_all(header)
+        if values is not None:
+            values = parse_address_list(values)
+        return values or []
+
+    def get_date_header(self, header):
+        """Return the value of header parsed into a datetime.date, or None"""
+        value = self[header]
+        if value is not None:
+            value = parse_rfc2822date(value)
+        return value
+
+    def _get_body_content(self, content_type):
+        # This doesn't handle as many corner cases as Python 3.6 email.message.EmailMessage.get_body,
+        # but should work correctly for nearly all real-world inbound messages.
+        # We're guaranteed to have `is_attachment` available, because all AnymailInboundMessage parts
+        # should themselves be AnymailInboundMessage.
+        for part in self.walk():
+            if part.get_content_type() == content_type and not part.is_attachment():
+                payload = part.get_payload(decode=True)
+                if payload is not None:
+                    return payload.decode('utf-8')
+        return None
+
+    # Backport from Python 3.5 email.message.Message
+    def get_content_disposition(self):
+        try:
+            return super(AnymailInboundMessage, self).get_content_disposition()
+        except AttributeError:
+            return get_content_disposition(self)
+
+    # Backport from Python 3.4.2 email.message.MIMEPart
+    def is_attachment(self):
+        return self.get_content_disposition() == 'attachment'
+
+    # New for Anymail
+    def is_inline_attachment(self):
+        return self.get_content_disposition() == 'inline'
+
+    def get_content_bytes(self):
+        """Return the raw payload bytes"""
+        maintype = self.get_content_maintype()
+        if maintype == 'message':
+            # The attachment's payload is a single (parsed) email Message; flatten it to bytes.
+            # (Note that self.is_multipart() misleadingly returns True in this case.)
+            payload = self.get_payload()
+            assert len(payload) == 1  # should be exactly one message
+            try:
+                return payload[0].as_bytes()  # Python 3
+            except AttributeError:
+                return payload[0].as_string().encode('utf-8')
+        elif maintype == 'multipart':
+            # The attachment itself is multipart; the payload is a list of parts,
+            # and it's not clear which one is the "content".
+            raise ValueError("get_content_bytes() is not valid on multipart messages "
+                             "(perhaps you want as_bytes()?)")
+        return self.get_payload(decode=True)
+
+    def get_content_text(self, charset='utf-8'):
+        """Return the payload decoded to text"""
+        maintype = self.get_content_maintype()
+        if maintype == 'message':
+            # The attachment's payload is a single (parsed) email Message; flatten it to text.
+            # (Note that self.is_multipart() misleadingly returns True in this case.)
+            payload = self.get_payload()
+            assert len(payload) == 1  # should be exactly one message
+            return payload[0].as_string()
+        elif maintype == 'multipart':
+            # The attachment itself is multipart; the payload is a list of parts,
+            # and it's not clear which one is the "content".
+            raise ValueError("get_content_text() is not valid on multipart messages "
+                             "(perhaps you want as_string()?)")
+        return self.get_payload(decode=True).decode(charset)
+
+    def as_uploaded_file(self):
+        """Return the attachment converted to a Django UploadedFile"""
+        if self['Content-Disposition'] is None:
+            return None  # this part is not an attachment
+        name = self.get_filename()
+        content_type = self.get_content_type()
+        content = self.get_content_bytes()
+        return SimpleUploadedFile(name, content, content_type)
+
+    #
+    # Construction
+    #
+    # These methods are intended primarily for internal Anymail use
+    # (in inbound webhook handlers)
+
+    @classmethod
+    def parse_raw_mime(cls, s):
+        """Returns a new AnymailInboundMessage parsed from str s"""
+        return message_from_string(s, cls)
+
+    @classmethod
+    def construct(cls, raw_headers=None, from_email=None, to=None, cc=None, subject=None, headers=None,
+                  text=None, text_charset='utf-8', html=None, html_charset='utf-8',
+                  attachments=None):
+        """
+        Returns a new AnymailInboundMessage constructed from params.
+
+        This is designed to handle the sorts of email fields typically present
+        in ESP parsed inbound messages. (It's not a generalized MIME message constructor.)
+
+        :param raw_headers: {str|None} base (or complete) message headers as a single string
+        :param from_email: {str|None} value for From header
+        :param to: {str|None} value for To header
+        :param cc: {str|None} value for Cc header
+        :param subject: {str|None} value for Subject header
+        :param headers: {sequence[(str, str)]|mapping|None} additional headers
+        :param text: {str|None} plaintext body
+        :param text_charset: {str} charset of plaintext body; default utf-8
+        :param html: {str|None} html body
+        :param html_charset: {str} charset of html body; default utf-8
+        :param attachments: {list[MIMEBase]|None} as returned by construct_attachment
+        :return: {AnymailInboundMessage}
+        """
+        if raw_headers is not None:
+            msg = HeaderParser(cls, policy=accurate_header_unfolding_policy).parsestr(raw_headers)
+            msg.set_payload(None)  # headersonly forces an empty string payload, which breaks things later
+        else:
+            msg = cls()
+
+        if from_email is not None:
+            del msg['From']  # override raw_headers value, if any
+            msg['From'] = from_email
+        if to is not None:
+            del msg['To']
+            msg['To'] = to
+        if cc is not None:
+            del msg['Cc']
+            msg['Cc'] = cc
+        if subject is not None:
+            del msg['Subject']
+            msg['Subject'] = subject
+        if headers is not None:
+            try:
+                header_items = headers.items()  # mapping
+            except AttributeError:
+                header_items = headers  # sequence of (key, value)
+            for name, value in header_items:
+                msg.add_header(name, value)
+
+        # For simplicity, we always build a MIME structure that could support plaintext/html
+        # alternative bodies, inline attachments for the body(ies), and message attachments.
+        # This may be overkill for simpler messages, but the structure is never incorrect.
+        del msg['MIME-Version']  # override raw_headers values, if any
+        del msg['Content-Type']
+        msg['MIME-Version'] = '1.0'
+        msg['Content-Type'] = 'multipart/mixed'
+
+        related = cls()  # container for alternative bodies and inline attachments
+        related['Content-Type'] = 'multipart/related'
+        msg.attach(related)
+
+        alternatives = cls()  # container for text and html bodies
+        alternatives['Content-Type'] = 'multipart/alternative'
+        related.attach(alternatives)
+
+        if text is not None:
+            part = cls()
+            part['Content-Type'] = 'text/plain'
+            part.set_payload(text, charset=text_charset)
+            alternatives.attach(part)
+        if html is not None:
+            part = cls()
+            part['Content-Type'] = 'text/html'
+            part.set_payload(html, charset=html_charset)
+            alternatives.attach(part)
+
+        if attachments is not None:
+            for attachment in attachments:
+                if attachment.is_inline_attachment():
+                    related.attach(attachment)
+                else:
+                    msg.attach(attachment)
+
+        return msg
+
+    @classmethod
+    def construct_attachment_from_uploaded_file(cls, file, content_id=None):
+        # This pulls the entire file into memory; it would be better to implement
+        # some sort of lazy attachment where the content is only pulled in if/when
+        # requested (and then use file.chunks() to minimize memory usage)
+        return cls.construct_attachment(
+            content_type=file.content_type,
+            content=file.read(),
+            filename=file.name,
+            content_id=content_id,
+            charset=file.charset)
+
+    @classmethod
+    def construct_attachment(cls, content_type, content,
+                             charset=None, filename=None, content_id=None, base64=False):
+        part = cls()
+        part['Content-Type'] = content_type
+        part['Content-Disposition'] = 'inline' if content_id is not None else 'attachment'
+
+        if filename is not None:
+            part.set_param('name', filename, header='Content-Type')
+            part.set_param('filename', filename, header='Content-Disposition')
+
+        if content_id is not None:
+            part['Content-ID'] = angle_wrap(content_id)
+
+        if base64:
+            content = b64decode(content)
+
+        payload = content
+        if part.get_content_maintype() == 'message':
+            # email.Message parses message/rfc822 parts as a "multipart" (list) payload
+            # whose single item is the recursively-parsed message attachment
+            if isinstance(content, six.binary_type):
+                content = content.decode()
+            payload = [cls.parse_raw_mime(content)]
+            charset = None
+
+        part.set_payload(payload, charset)
+        return part

anymail/signals.py

@@ -45,6 +45,18 @@ class AnymailInboundEvent(AnymailEvent):
 
     def __init__(self, **kwargs):
         super(AnymailInboundEvent, self).__init__(**kwargs)
+        self.message = kwargs.pop('message', None)  # anymail.inbound.AnymailInboundMessage
+        self.recipient = kwargs.pop('recipient', None)  # str: envelope recipient
+        self.sender = kwargs.pop('sender', None)  # str: envelope sender
+
+        self.stripped_text = kwargs.pop('stripped_text', None)  # cleaned of quotes/signatures (varies by ESP)
+        self.stripped_html = kwargs.pop('stripped_html', None)
+        self.spam_detected = kwargs.pop('spam_detected', None)  # bool
+        self.spam_score = kwargs.pop('spam_score', None)  # float: usually SpamAssassin
+
+        # SPF status?
+        # DKIM status?
+        # DMARC status? (no ESP has documented support yet)
 
 
 class EventType:

anymail/urls.py

@@ -1,19 +1,29 @@
 from django.conf.urls import url
 
-from .webhooks.mailgun import MailgunTrackingWebhookView
-from .webhooks.mailjet import MailjetTrackingWebhookView
-from .webhooks.mandrill import MandrillTrackingWebhookView
-from .webhooks.postmark import PostmarkTrackingWebhookView
-from .webhooks.sendgrid import SendGridTrackingWebhookView
-from .webhooks.sparkpost import SparkPostTrackingWebhookView
+from .webhooks.mailgun import MailgunInboundWebhookView, MailgunTrackingWebhookView
+from .webhooks.mailjet import MailjetInboundWebhookView, MailjetTrackingWebhookView
+from .webhooks.mandrill import MandrillCombinedWebhookView
+from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView
+from .webhooks.sendgrid import SendGridInboundWebhookView, SendGridTrackingWebhookView
+from .webhooks.sparkpost import SparkPostInboundWebhookView, SparkPostTrackingWebhookView
 
 
 app_name = 'anymail'
 urlpatterns = [
+    url(r'^mailgun/inbound(_mime)?/$', MailgunInboundWebhookView.as_view(), name='mailgun_inbound_webhook'),
+    url(r'^mailjet/inbound/$', MailjetInboundWebhookView.as_view(), name='mailjet_inbound_webhook'),
+    url(r'^postmark/inbound/$', PostmarkInboundWebhookView.as_view(), name='postmark_inbound_webhook'),
+    url(r'^sendgrid/inbound/$', SendGridInboundWebhookView.as_view(), name='sendgrid_inbound_webhook'),
+    url(r'^sparkpost/inbound/$', SparkPostInboundWebhookView.as_view(), name='sparkpost_inbound_webhook'),
+
     url(r'^mailgun/tracking/$', MailgunTrackingWebhookView.as_view(), name='mailgun_tracking_webhook'),
     url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'),
-    url(r'^mandrill/tracking/$', MandrillTrackingWebhookView.as_view(), name='mandrill_tracking_webhook'),
     url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'),
     url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'),
     url(r'^sparkpost/tracking/$', SparkPostTrackingWebhookView.as_view(), name='sparkpost_tracking_webhook'),
+
+    # Anymail uses a combined Mandrill webhook endpoint, to simplify Mandrill's key-validation scheme:
+    url(r'^mandrill/$', MandrillCombinedWebhookView.as_view(), name='mandrill_webhook'),
+    # This url is maintained for backwards compatibility with earlier Anymail releases:
+    url(r'^mandrill/tracking/$', MandrillCombinedWebhookView.as_view(), name='mandrill_tracking_webhook'),
 ]

anymail/utils.py

@@ -433,6 +433,18 @@ def rfc2822date(dt):
     return formatdate(timeval, usegmt=True)
 
 
+def angle_wrap(s):
+    """Return s surrounded by angle brackets, added only if necessary"""
+    # This is the inverse behavior of email.utils.unquote
+    # (which you might think email.utils.quote would do, but it doesn't)
+    if len(s) > 0:
+        if s[0] != '<':
+            s = '<' + s
+        if s[-1] != '>':
+            s = s + '>'
+    return s
+
+
 def is_lazy(obj):
     """Return True if obj is a Django lazy object."""
     # See django.utils.functional.lazy. (This appears to be preferred

anymail/webhooks/base.py

@@ -1,4 +1,3 @@
-import re
 import warnings
 
 import six
@@ -128,6 +127,9 @@ class AnymailBaseWebhookView(AnymailBasicAuthMixin, View):
         """
         Read-only name of the ESP for this webhook view.
 
-        (E.g., MailgunTrackingWebhookView will return "Mailgun")
+        Subclasses must override with class attr. E.g.:
+            esp_name = "Postmark"
+            esp_name = "SendGrid"  # (use ESP's preferred capitalization)
         """
-        return re.sub(r'(Tracking|Inbox)WebhookView$', "", self.__class__.__name__)
+        raise NotImplementedError("%s.%s must declare esp_name class attr" %
+                                  (self.__class__.__module__, self.__class__.__name__))

anymail/webhooks/mailgun.py

@@ -8,13 +8,15 @@ from django.utils.timezone import utc
 
 from .base import AnymailBaseWebhookView
 from ..exceptions import AnymailWebhookValidationFailure
-from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+from ..inbound import AnymailInboundMessage
+from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
 from ..utils import get_anymail_setting, combine, querydict_getfirst
 
 
 class MailgunBaseWebhookView(AnymailBaseWebhookView):
     """Base view class for Mailgun webhooks"""
 
+    esp_name = "Mailgun"
     warn_if_no_basic_auth = False  # because we validate against signature
 
     api_key = None  # (Declaring class attr allows override by kwargs in View.as_view.)
@@ -40,12 +42,6 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView):
         if not constant_time_compare(signature, expected_signature):
             raise AnymailWebhookValidationFailure("Mailgun webhook called with incorrect signature")
 
-    def parse_events(self, request):
-        return [self.esp_to_anymail_event(request.POST)]
-
-    def esp_to_anymail_event(self, esp_event):
-        raise NotImplementedError()
-
 
 class MailgunTrackingWebhookView(MailgunBaseWebhookView):
     """Handler for Mailgun delivery and engagement tracking webhooks"""
@@ -75,6 +71,9 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
         607: RejectReason.SPAM,  # previous spam complaint
     }
 
+    def parse_events(self, request):
+        return [self.esp_to_anymail_event(request.POST)]
+
     def esp_to_anymail_event(self, esp_event):
         # esp_event is a Django QueryDict (from request.POST),
         # which has multi-valued fields, but is *not* case-insensitive.
@@ -194,3 +193,69 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
         'opened': _common_event_fields,
         'unsubscribed': _common_event_fields,
     }
+
+
+class MailgunInboundWebhookView(MailgunBaseWebhookView):
+    """Handler for Mailgun inbound (route forward-to-url) webhook"""
+
+    signal = inbound
+
+    def parse_events(self, request):
+        return [self.esp_to_anymail_event(request)]
+
+    def esp_to_anymail_event(self, request):
+        # Inbound uses the entire Django request as esp_event, because we need POST and FILES.
+        # Note that request.POST is case-sensitive (unlike email.message.Message headers).
+        esp_event = request
+        if 'body-mime' in request.POST:
+            # Raw-MIME
+            message = AnymailInboundMessage.parse_raw_mime(request.POST['body-mime'])
+        else:
+            # Fully-parsed
+            message = self.message_from_mailgun_parsed(request)
+
+        message.envelope_sender = request.POST.get('sender', None)
+        message.envelope_recipient = request.POST.get('recipient', None)
+        message.stripped_text = request.POST.get('stripped-text', None)
+        message.stripped_html = request.POST.get('stripped-html', None)
+
+        message.spam_detected = message.get('X-Mailgun-Sflag', 'No').lower() == 'yes'
+        try:
+            message.spam_score = float(message['X-Mailgun-Sscore'])
+        except (TypeError, ValueError):
+            pass
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=datetime.fromtimestamp(int(request.POST['timestamp']), tz=utc),
+            event_id=request.POST.get('token', None),
+            esp_event=esp_event,
+            message=message,
+        )
+
+    def message_from_mailgun_parsed(self, request):
+        """Construct a Message from Mailgun's "fully-parsed" fields"""
+        # Mailgun transcodes all fields to UTF-8 for "fully parsed" messages
+        try:
+            attachment_count = int(request.POST['attachment-count'])
+        except (KeyError, TypeError):
+            attachments = None
+        else:
+            # Load attachments from posted files: Mailgun file field names are 1-based
+            att_ids = ['attachment-%d' % i for i in range(1, attachment_count+1)]
+            att_cids = {  # filename: content-id (invert content-id-map)
+                att_id: cid for cid, att_id
+                in json.loads(request.POST.get('content-id-map', '{}')).items()
+            }
+            attachments = [
+                AnymailInboundMessage.construct_attachment_from_uploaded_file(
+                    request.FILES[att_id], content_id=att_cids.get(att_id, None))
+                for att_id in att_ids
+            ]
+
+        return AnymailInboundMessage.construct(
+            headers=json.loads(request.POST['message-headers']),  # includes From, To, Cc, Subject, etc.
+            text=request.POST.get('body-plain', None),
+            html=request.POST.get('body-html', None),
+            attachments=attachments,
+        )

anymail/webhooks/mailjet.py

@@ -4,12 +4,14 @@ from datetime import datetime
 from django.utils.timezone import utc
 
 from .base import AnymailBaseWebhookView
-from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+from ..inbound import AnymailInboundMessage
+from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
 
 
 class MailjetTrackingWebhookView(AnymailBaseWebhookView):
     """Handler for Mailjet delivery and engagement tracking webhooks"""
 
+    esp_name = "Mailjet"
     signal = tracking
 
     def parse_events(self, request):
@@ -95,3 +97,84 @@ class MailjetTrackingWebhookView(AnymailBaseWebhookView):
             user_agent=esp_event.get('agent', None),
             esp_event=esp_event,
         )
+
+
+class MailjetInboundWebhookView(AnymailBaseWebhookView):
+    """Handler for Mailjet inbound (parse API) webhook"""
+
+    esp_name = "Mailjet"
+    signal = inbound
+
+    def parse_events(self, request):
+        esp_event = json.loads(request.body.decode('utf-8'))
+        return [self.esp_to_anymail_event(esp_event)]
+
+    def esp_to_anymail_event(self, esp_event):
+        # You could _almost_ reconstruct the raw mime message from Mailjet's Headers and Parts fields,
+        # but it's not clear which multipart boundary to use on each individual Part. Although each Part's
+        # Content-Type header still has the multipart boundary, not knowing the parent part means typical
+        # nested multipart structures can't be reliably recovered from the data Mailjet provides.
+        # We'll just use our standarized multipart inbound constructor.
+
+        headers = self._flatten_mailjet_headers(esp_event.get("Headers", {}))
+        attachments = [
+            self._construct_mailjet_attachment(part, esp_event)
+            for part in esp_event.get("Parts", [])
+            if "Attachment" in part.get("ContentRef", "")  # Attachment<N> or InlineAttachment<N>
+        ]
+        message = AnymailInboundMessage.construct(
+            headers=headers,
+            text=esp_event.get("Text-part", None),
+            html=esp_event.get("Html-part", None),
+            attachments=attachments,
+        )
+
+        message.envelope_sender = esp_event.get("Sender", None)
+        message.envelope_recipient = esp_event.get("Recipient", None)
+
+        message.spam_detected = None  # Mailjet doesn't provide a boolean; you'll have to interpret spam_score
+        try:
+            message.spam_score = float(esp_event['SpamAssassinScore'])
+        except (KeyError, TypeError, ValueError):
+            pass
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=None,  # Mailjet doesn't provide inbound event timestamp (esp_event['Date'] is time sent)
+            event_id=None,  # Mailjet doesn't provide an idempotent inbound event id
+            esp_event=esp_event,
+            message=message,
+        )
+
+    @staticmethod
+    def _flatten_mailjet_headers(headers):
+        """Convert Mailjet's dict-of-strings-and/or-lists header format to our list-of-name-value-pairs
+
+        {'name1': 'value', 'name2': ['value1', 'value2']}
+          --> [('name1', 'value'), ('name2', 'value1'), ('name2', 'value2')]
+        """
+        result = []
+        for name, values in headers.items():
+            if isinstance(values, list):  # Mailjet groups repeated headers together as a list of values
+                for value in values:
+                    result.append((name, value))
+            else:
+                result.append((name, values))  # single-valued (non-list) header
+        return result
+
+    def _construct_mailjet_attachment(self, part, esp_event):
+        # Mailjet includes unparsed attachment headers in each part; it's easiest to temporarily
+        # attach them to a MIMEPart for parsing. (We could just turn this into the attachment,
+        # but we want to use the payload handling from AnymailInboundMessage.construct_attachment later.)
+        part_headers = AnymailInboundMessage()  # temporary container for parsed attachment headers
+        for name, value in self._flatten_mailjet_headers(part.get("Headers", {})):
+            part_headers.add_header(name, value)
+
+        content_base64 = esp_event[part["ContentRef"]]  # Mailjet *always* base64-encodes attachments
+
+        return AnymailInboundMessage.construct_attachment(
+            content_type=part_headers.get_content_type(),
+            content=content_base64, base64=True,
+            filename=part_headers.get_filename(None),
+            content_id=part_headers.get("Content-ID", "") or None,
+        )

anymail/webhooks/mandrill.py

@@ -8,8 +8,9 @@ from django.utils.crypto import constant_time_compare
 from django.utils.timezone import utc
 
 from .base import AnymailBaseWebhookView
-from ..exceptions import AnymailWebhookValidationFailure, AnymailConfigurationError
-from ..signals import tracking, AnymailTrackingEvent, EventType
+from ..exceptions import AnymailWebhookValidationFailure
+from ..inbound import AnymailInboundMessage
+from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType
 from ..utils import get_anymail_setting, getfirst, get_request_uri
 
 
@@ -59,22 +60,34 @@ class MandrillSignatureMixin(object):
                 "Mandrill webhook called with incorrect signature (for url %r)" % url)
 
 
-class MandrillBaseWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView):
-    """Base view class for Mandrill webhooks"""
+class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView):
+    """Unified view class for Mandrill tracking and inbound webhooks"""
+
+    esp_name = "Mandrill"
 
     warn_if_no_basic_auth = False  # because we validate against signature
+    signal = None  # set in esp_to_anymail_event
 
     def parse_events(self, request):
         esp_events = json.loads(request.POST['mandrill_events'])
         return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
 
     def esp_to_anymail_event(self, esp_event):
-        raise NotImplementedError()
+        """Route events to the inbound or tracking handler"""
+        esp_type = getfirst(esp_event, ['event', 'type'], 'unknown')
 
+        if esp_type == 'inbound':
+            assert self.signal is not tracking  # Mandrill should never mix event types in the same batch
+            self.signal = inbound
+            return self.mandrill_inbound_to_anymail_event(esp_event)
+        else:
+            assert self.signal is not inbound  # Mandrill should never mix event types in the same batch
+            self.signal = tracking
+            return self.mandrill_tracking_to_anymail_event(esp_event)
 
-class MandrillTrackingWebhookView(MandrillBaseWebhookView):
-
-    signal = tracking
+    #
+    #  Tracking events
+    #
 
     event_types = {
         # Message events:
@@ -94,13 +107,9 @@ class MandrillTrackingWebhookView(MandrillBaseWebhookView):
         'inbound': EventType.INBOUND,
     }
 
-    def esp_to_anymail_event(self, esp_event):
+    def mandrill_tracking_to_anymail_event(self, esp_event):
         esp_type = getfirst(esp_event, ['event', 'type'], None)
         event_type = self.event_types.get(esp_type, EventType.UNKNOWN)
-        if event_type == EventType.INBOUND:
-            raise AnymailConfigurationError(
-                "You seem to have set Mandrill's *inbound* webhook URL "
-                "to Anymail's Mandrill *tracking* webhook URL.")
 
         try:
             timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc)
@@ -149,3 +158,33 @@ class MandrillTrackingWebhookView(MandrillBaseWebhookView):
             timestamp=timestamp,
             user_agent=esp_event.get('user_agent', None),
         )
+
+    #
+    # Inbound events
+    #
+
+    def mandrill_inbound_to_anymail_event(self, esp_event):
+        # It's easier (and more accurate) to just work from the original raw mime message
+        message = AnymailInboundMessage.parse_raw_mime(esp_event['msg']['raw_msg'])
+        message.envelope_sender = None  # (Mandrill's 'sender' field only applies to outbound messages)
+        message.envelope_recipient = esp_event['msg'].get('email', None)
+
+        message.spam_detected = None  # no simple boolean field; would need to parse the spam_report
+        message.spam_score = esp_event['msg'].get('spam_report', {}).get('score', None)
+
+        try:
+            timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc)
+        except (KeyError, ValueError):
+            timestamp = None
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=timestamp,
+            event_id=None,  # Mandrill doesn't provide an idempotent inbound message event id
+            esp_event=esp_event,
+            message=message,
+        )
+
+
+# Backwards-compatibility: earlier Anymail versions had only MandrillTrackingWebhookView:
+MandrillTrackingWebhookView = MandrillCombinedWebhookView

anymail/webhooks/postmark.py

@@ -4,13 +4,16 @@ from django.utils.dateparse import parse_datetime
 
 from .base import AnymailBaseWebhookView
 from ..exceptions import AnymailConfigurationError
-from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
-from ..utils import getfirst
+from ..inbound import AnymailInboundMessage
+from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
+from ..utils import getfirst, EmailAddress
 
 
 class PostmarkBaseWebhookView(AnymailBaseWebhookView):
     """Base view class for Postmark webhooks"""
 
+    esp_name = "Postmark"
+
     def parse_events(self, request):
         esp_event = json.loads(request.body.decode('utf-8'))
         return [self.esp_to_anymail_event(esp_event)]
@@ -107,3 +110,72 @@ class PostmarkTrackingWebhookView(PostmarkBaseWebhookView):
             user_agent=esp_event.get('UserAgent', None),
             click_url=esp_event.get('OriginalLink', None),
         )
+
+
+class PostmarkInboundWebhookView(PostmarkBaseWebhookView):
+    """Handler for Postmark inbound webhook"""
+
+    signal = inbound
+
+    def esp_to_anymail_event(self, esp_event):
+        attachments = [
+            AnymailInboundMessage.construct_attachment(
+                content_type=attachment["ContentType"],
+                content=attachment["Content"], base64=True,
+                filename=attachment.get("Name", "") or None,
+                content_id=attachment.get("ContentID", "") or None,
+            )
+            for attachment in esp_event.get("Attachments", [])
+        ]
+
+        message = AnymailInboundMessage.construct(
+            from_email=self._address(esp_event.get("FromFull")),
+            to=', '.join([self._address(to) for to in esp_event.get("ToFull", [])]),
+            cc=', '.join([self._address(cc) for cc in esp_event.get("CcFull", [])]),
+            # bcc? Postmark specs this for inbound events, but it's unclear how it could occur
+            subject=esp_event.get("Subject", ""),
+            headers=[(header["Name"], header["Value"]) for header in esp_event.get("Headers", [])],
+            text=esp_event.get("TextBody", ""),
+            html=esp_event.get("HtmlBody", ""),
+            attachments=attachments,
+        )
+
+        # Postmark strips these headers and provides them as separate event fields:
+        if "Date" in esp_event and "Date" not in message:
+            message["Date"] = esp_event["Date"]
+        if "ReplyTo" in esp_event and "Reply-To" not in message:
+            message["Reply-To"] = esp_event["ReplyTo"]
+
+        # Postmark doesn't have a separate envelope-sender field, but it can be extracted
+        # from the Received-SPF header that Postmark will have added:
+        if len(message.get_all("Received-SPF", [])) == 1:  # (more than one? someone's up to something weird)
+            received_spf = message["Received-SPF"].lower()
+            if received_spf.startswith("pass") or received_spf.startswith("neutral"):  # not fail/softfail
+                message.envelope_sender = message.get_param("envelope-from", None, header="Received-SPF")
+
+        message.envelope_recipient = esp_event.get("OriginalRecipient", None)
+        message.stripped_text = esp_event.get("StrippedTextReply", None)
+
+        message.spam_detected = message.get('X-Spam-Status', 'No').lower() == 'yes'
+        try:
+            message.spam_score = float(message['X-Spam-Score'])
+        except (TypeError, ValueError):
+            pass
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=None,  # Postmark doesn't provide inbound event timestamp
+            event_id=esp_event.get("MessageID", None),  # Postmark uuid, different from Message-ID mime header
+            esp_event=esp_event,
+            message=message,
+        )
+
+    @staticmethod
+    def _address(full):
+        """Return an formatted email address from a Postmark inbound {From,To,Cc}Full dict"""
+        if full is None:
+            return ""
+        return str(EmailAddress(
+            display_name=full.get('Name', ""),
+            addr_spec=full.get("Email", ""),
+        ))

anymail/webhooks/sendgrid.py

@@ -4,25 +4,20 @@ from datetime import datetime
 from django.utils.timezone import utc
 
 from .base import AnymailBaseWebhookView
-from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+from ..inbound import AnymailInboundMessage
+from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
 
 
-class SendGridBaseWebhookView(AnymailBaseWebhookView):
-    """Base view class for SendGrid webhooks"""
+class SendGridTrackingWebhookView(AnymailBaseWebhookView):
+    """Handler for SendGrid delivery and engagement tracking webhooks"""
+
+    esp_name = "SendGrid"
+    signal = tracking
 
     def parse_events(self, request):
         esp_events = json.loads(request.body.decode('utf-8'))
         return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
 
-    def esp_to_anymail_event(self, esp_event):
-        raise NotImplementedError()
-
-
-class SendGridTrackingWebhookView(SendGridBaseWebhookView):
-    """Handler for SendGrid delivery and engagement tracking webhooks"""
-
-    signal = tracking
-
     event_types = {
         # Map SendGrid event: Anymail normalized type
         'bounce': EventType.BOUNCED,
@@ -120,3 +115,78 @@ class SendGridTrackingWebhookView(SendGridBaseWebhookView):
         'url_offset',  # click tracking
         'useragent',  # click/open tracking
     }
+
+
+class SendGridInboundWebhookView(AnymailBaseWebhookView):
+    """Handler for SendGrid inbound webhook"""
+
+    esp_name = "SendGrid"
+    signal = inbound
+
+    def parse_events(self, request):
+        return [self.esp_to_anymail_event(request)]
+
+    def esp_to_anymail_event(self, request):
+        # Inbound uses the entire Django request as esp_event, because we need POST and FILES.
+        # Note that request.POST is case-sensitive (unlike email.message.Message headers).
+        esp_event = request
+        if 'headers' in request.POST:
+            # Default (not "Send Raw") inbound fields
+            message = self.message_from_sendgrid_parsed(esp_event)
+        elif 'email' in request.POST:
+            # "Send Raw" full MIME
+            message = AnymailInboundMessage.parse_raw_mime(request.POST['email'])
+        else:
+            raise KeyError("Invalid SendGrid inbound event data (missing both 'headers' and 'email' fields)")
+
+        try:
+            envelope = json.loads(request.POST['envelope'])
+        except (KeyError, TypeError, ValueError):
+            pass
+        else:
+            message.envelope_sender = envelope['from']
+            message.envelope_recipient = envelope['to'][0]
+
+        message.spam_detected = None  # no simple boolean field; would need to parse the spam_report
+        try:
+            message.spam_score = float(request.POST['spam_score'])
+        except (KeyError, TypeError, ValueError):
+            pass
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=None,  # SendGrid doesn't provide an inbound event timestamp
+            event_id=None,  # SendGrid doesn't provide an idempotent inbound message event id
+            esp_event=esp_event,
+            message=message,
+        )
+
+    def message_from_sendgrid_parsed(self, request):
+        """Construct a Message from SendGrid's "default" (non-raw) fields"""
+
+        try:
+            charsets = json.loads(request.POST['charsets'])
+        except (KeyError, ValueError):
+            charsets = {}
+
+        try:
+            attachment_info = json.loads(request.POST['attachment-info'])
+        except (KeyError, ValueError):
+            attachments = None
+        else:
+            # Load attachments from posted files
+            attachments = [
+                AnymailInboundMessage.construct_attachment_from_uploaded_file(
+                    request.FILES[att_id],
+                    content_id=attachment_info[att_id].get("content-id", None))
+                for att_id in sorted(attachment_info.keys())
+            ]
+
+        return AnymailInboundMessage.construct(
+            raw_headers=request.POST.get('headers', ""),  # includes From, To, Cc, Subject, etc.
+            text=request.POST.get('text', None),
+            text_charset=charsets.get('text', 'utf-8'),
+            html=request.POST.get('html', None),
+            html_charset=charsets.get('html', 'utf-8'),
+            attachments=attachments,
+        )

anymail/webhooks/sparkpost.py

@@ -1,16 +1,20 @@
 import json
+from base64 import b64decode
 from datetime import datetime
 
 from django.utils.timezone import utc
 
 from .base import AnymailBaseWebhookView
 from ..exceptions import AnymailConfigurationError
-from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+from ..inbound import AnymailInboundMessage
+from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
 
 
 class SparkPostBaseWebhookView(AnymailBaseWebhookView):
     """Base view class for SparkPost webhooks"""
 
+    esp_name = "SparkPost"
+
     def parse_events(self, request):
         raw_events = json.loads(request.body.decode('utf-8'))
         unwrapped_events = [self.unwrap_event(raw_event) for raw_event in raw_events]
@@ -92,7 +96,7 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
     }
 
     def esp_to_anymail_event(self, event_class, event, raw_event):
-        if event_class == 'relay_event':
+        if event_class == 'relay_message':
             # This is an inbound event
             raise AnymailConfigurationError(
                 "You seem to have set SparkPost's *inbound* relay webhook URL "
@@ -134,3 +138,37 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
             user_agent=event.get('user_agent', None),
             esp_event=raw_event,
         )
+
+
+class SparkPostInboundWebhookView(SparkPostBaseWebhookView):
+    """Handler for SparkPost inbound relay webhook"""
+
+    signal = inbound
+
+    def esp_to_anymail_event(self, event_class, event, raw_event):
+        if event_class != 'relay_message':
+            # This is not an inbound event
+            raise AnymailConfigurationError(
+                "You seem to have set SparkPost's *tracking* webhook URL "
+                "to Anymail's SparkPost *inbound* relay webhook URL.")
+
+        if event['protocol'] != 'smtp':
+            raise AnymailConfigurationError(
+                "You cannot use Anymail's webhooks for SparkPost '{protocol}' relay events. "
+                "Anymail only handles the 'smtp' protocol".format(protocol=event['protocol']))
+
+        raw_mime = event['content']['email_rfc822']
+        if event['content']['email_rfc822_is_base64']:
+            raw_mime = b64decode(raw_mime).decode('utf-8')
+        message = AnymailInboundMessage.parse_raw_mime(raw_mime)
+
+        message.envelope_sender = event.get('msg_from', None)
+        message.envelope_recipient = event.get('rcpt_to', None)
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=None,  # SparkPost does not provide a relay event timestamp
+            event_id=None,  # SparkPost does not provide an idempotent id for relay events
+            esp_event=raw_event,
+            message=message,
+        )