diff --git a/README.rst b/README.rst index db225b5..6f44491 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,8 @@ built-in `django.core.mail` package. It includes: * Normalized sent-message status and tracking notification, by connecting your ESP's webhooks to Django signals * "Batch transactional" sends using your ESP's merge and template features +* Inbound message support, to receive email through your ESP's webhooks, + with simplified, portable access to attachments and other inbound content Anymail is released under the BSD license. It is extensively tested against Django 1.8--2.0 (including Python 2.7, Python 3 and PyPy). @@ -67,6 +69,7 @@ Anymail 1-2-3 .. This quickstart section is also included in docs/quickstart.rst +Here's how to send a message. This example uses Mailgun, but you can substitute Mailjet or Postmark or SendGrid or SparkPost or any other supported ESP where you see "mailgun": @@ -144,4 +147,5 @@ or SparkPost or any other supported ESP where you see "mailgun": See the `full documentation `_ -for more features and options. +for more features and options, including receiving messages and tracking +sent message status. diff --git a/anymail/inbound.py b/anymail/inbound.py new file mode 100644 index 0000000..3cce925 --- /dev/null +++ b/anymail/inbound.py @@ -0,0 +1,355 @@ +from base64 import b64decode +from email import message_from_string +from email.message import Message +from email.utils import unquote + +import six +from django.core.files.uploadedfile import SimpleUploadedFile + +from .utils import angle_wrap, get_content_disposition, parse_address_list, parse_rfc2822date + +# Python 2/3.*-compatible email.parser.HeaderParser(policy=email.policy.default) +try: + # With Python 3.3+ (email6) package, can use HeaderParser with default policy + from email.parser import HeaderParser + from email.policy import default as accurate_header_unfolding_policy # vs. compat32 + +except ImportError: + # Earlier Pythons don't have HeaderParser, and/or try preserve earlier compatibility bugs + # by failing to properly unfold headers (see RFC 5322 section 2.2.3) + from email.parser import Parser + import re + accurate_header_unfolding_policy = object() + + class HeaderParser(Parser, object): + def __init__(self, _class, policy=None): + # This "backport" doesn't actually support policies, but we want to ensure + # that callers aren't trying to use HeaderParser's default compat32 policy + # (which doesn't properly unfold headers) + assert policy is accurate_header_unfolding_policy + super(HeaderParser, self).__init__(_class) + + def parsestr(self, text, headersonly=True): + unfolded = self._unfold_headers(text) + return super(HeaderParser, self).parsestr(unfolded, headersonly=True) + + @staticmethod + def _unfold_headers(text): + # "Unfolding is accomplished by simply removing any CRLF that is immediately followed by WSP" + # (WSP is space or tab, and per email.parser semantics, we allow CRLF, CR, or LF endings) + return re.sub(r'(\r\n|\r|\n)(?=[ \t])', "", text) + + +class AnymailInboundMessage(Message, object): # `object` ensures new-style class in Python 2) + """ + A normalized, parsed inbound email message. + + A subclass of email.message.Message, with some additional + convenience properties, plus helpful methods backported + from Python 3.6+ email.message.EmailMessage (or really, MIMEPart) + """ + + # Why Python email.message.Message rather than django.core.mail.EmailMessage? + # Django's EmailMessage is really intended for constructing a (limited subset of) + # Message to send; Message is better designed for representing arbitrary messages: + # + # * Message is easily parsed from raw mime (which is an inbound format provided + # by many ESPs), and can accurately represent any mime email that might be received + # * Message can represent repeated header fields (e.g., "Received") which + # are common in inbound messages + # * Django's EmailMessage defaults a bunch of properties in ways that aren't helpful + # (e.g., from_email from settings) + + def __init__(self, *args, **kwargs): + # Note: this must accept zero arguments, for use with message_from_string (email.parser) + super(AnymailInboundMessage, self).__init__(*args, **kwargs) + + # Additional attrs provided by some ESPs: + self.envelope_sender = None + self.envelope_recipient = None + self.stripped_text = None + self.stripped_html = None + self.spam_detected = None + self.spam_score = None + + # + # Convenience accessors + # + + @property + def from_email(self): + """EmailAddress """ + # equivalent to Python 3.2+ message['From'].addresses[0] + from_email = self.get_address_header('From') + if len(from_email) == 1: + return from_email[0] + elif len(from_email) == 0: + return None + else: + return from_email # unusual, but technically-legal multiple-From; preserve list + + @property + def to(self): + """list of EmailAddress objects from To header""" + # equivalent to Python 3.2+ message['To'].addresses + return self.get_address_header('To') + + @property + def cc(self): + """list of EmailAddress objects from Cc header""" + # equivalent to Python 3.2+ message['Cc'].addresses + return self.get_address_header('Cc') + + @property + def subject(self): + """str value of Subject header, or None""" + return self['Subject'] + + @property + def date(self): + """datetime.datetime from Date header, or None if missing/invalid""" + # equivalent to Python 3.2+ message['Date'].datetime + return self.get_date_header('Date') + + @property + def text(self): + """Contents of the (first) text/plain body part, or None""" + return self._get_body_content('text/plain') + + @property + def html(self): + """Contents of the (first) text/html body part, or None""" + return self._get_body_content('text/html') + + @property + def attachments(self): + """list of attachments (as MIMEPart objects); excludes inlines""" + return [part for part in self.walk() if part.is_attachment()] + + @property + def inline_attachments(self): + """dict of Content-ID: attachment (as MIMEPart objects)""" + return {unquote(part['Content-ID']): part for part in self.walk() + if part.is_inline_attachment() and part['Content-ID']} + + def get_address_header(self, header): + """Return the value of header parsed into a (possibly-empty) list of EmailAddress objects""" + values = self.get_all(header) + if values is not None: + values = parse_address_list(values) + return values or [] + + def get_date_header(self, header): + """Return the value of header parsed into a datetime.date, or None""" + value = self[header] + if value is not None: + value = parse_rfc2822date(value) + return value + + def _get_body_content(self, content_type): + # This doesn't handle as many corner cases as Python 3.6 email.message.EmailMessage.get_body, + # but should work correctly for nearly all real-world inbound messages. + # We're guaranteed to have `is_attachment` available, because all AnymailInboundMessage parts + # should themselves be AnymailInboundMessage. + for part in self.walk(): + if part.get_content_type() == content_type and not part.is_attachment(): + payload = part.get_payload(decode=True) + if payload is not None: + return payload.decode('utf-8') + return None + + # Backport from Python 3.5 email.message.Message + def get_content_disposition(self): + try: + return super(AnymailInboundMessage, self).get_content_disposition() + except AttributeError: + return get_content_disposition(self) + + # Backport from Python 3.4.2 email.message.MIMEPart + def is_attachment(self): + return self.get_content_disposition() == 'attachment' + + # New for Anymail + def is_inline_attachment(self): + return self.get_content_disposition() == 'inline' + + def get_content_bytes(self): + """Return the raw payload bytes""" + maintype = self.get_content_maintype() + if maintype == 'message': + # The attachment's payload is a single (parsed) email Message; flatten it to bytes. + # (Note that self.is_multipart() misleadingly returns True in this case.) + payload = self.get_payload() + assert len(payload) == 1 # should be exactly one message + try: + return payload[0].as_bytes() # Python 3 + except AttributeError: + return payload[0].as_string().encode('utf-8') + elif maintype == 'multipart': + # The attachment itself is multipart; the payload is a list of parts, + # and it's not clear which one is the "content". + raise ValueError("get_content_bytes() is not valid on multipart messages " + "(perhaps you want as_bytes()?)") + return self.get_payload(decode=True) + + def get_content_text(self, charset='utf-8'): + """Return the payload decoded to text""" + maintype = self.get_content_maintype() + if maintype == 'message': + # The attachment's payload is a single (parsed) email Message; flatten it to text. + # (Note that self.is_multipart() misleadingly returns True in this case.) + payload = self.get_payload() + assert len(payload) == 1 # should be exactly one message + return payload[0].as_string() + elif maintype == 'multipart': + # The attachment itself is multipart; the payload is a list of parts, + # and it's not clear which one is the "content". + raise ValueError("get_content_text() is not valid on multipart messages " + "(perhaps you want as_string()?)") + return self.get_payload(decode=True).decode(charset) + + def as_uploaded_file(self): + """Return the attachment converted to a Django UploadedFile""" + if self['Content-Disposition'] is None: + return None # this part is not an attachment + name = self.get_filename() + content_type = self.get_content_type() + content = self.get_content_bytes() + return SimpleUploadedFile(name, content, content_type) + + # + # Construction + # + # These methods are intended primarily for internal Anymail use + # (in inbound webhook handlers) + + @classmethod + def parse_raw_mime(cls, s): + """Returns a new AnymailInboundMessage parsed from str s""" + return message_from_string(s, cls) + + @classmethod + def construct(cls, raw_headers=None, from_email=None, to=None, cc=None, subject=None, headers=None, + text=None, text_charset='utf-8', html=None, html_charset='utf-8', + attachments=None): + """ + Returns a new AnymailInboundMessage constructed from params. + + This is designed to handle the sorts of email fields typically present + in ESP parsed inbound messages. (It's not a generalized MIME message constructor.) + + :param raw_headers: {str|None} base (or complete) message headers as a single string + :param from_email: {str|None} value for From header + :param to: {str|None} value for To header + :param cc: {str|None} value for Cc header + :param subject: {str|None} value for Subject header + :param headers: {sequence[(str, str)]|mapping|None} additional headers + :param text: {str|None} plaintext body + :param text_charset: {str} charset of plaintext body; default utf-8 + :param html: {str|None} html body + :param html_charset: {str} charset of html body; default utf-8 + :param attachments: {list[MIMEBase]|None} as returned by construct_attachment + :return: {AnymailInboundMessage} + """ + if raw_headers is not None: + msg = HeaderParser(cls, policy=accurate_header_unfolding_policy).parsestr(raw_headers) + msg.set_payload(None) # headersonly forces an empty string payload, which breaks things later + else: + msg = cls() + + if from_email is not None: + del msg['From'] # override raw_headers value, if any + msg['From'] = from_email + if to is not None: + del msg['To'] + msg['To'] = to + if cc is not None: + del msg['Cc'] + msg['Cc'] = cc + if subject is not None: + del msg['Subject'] + msg['Subject'] = subject + if headers is not None: + try: + header_items = headers.items() # mapping + except AttributeError: + header_items = headers # sequence of (key, value) + for name, value in header_items: + msg.add_header(name, value) + + # For simplicity, we always build a MIME structure that could support plaintext/html + # alternative bodies, inline attachments for the body(ies), and message attachments. + # This may be overkill for simpler messages, but the structure is never incorrect. + del msg['MIME-Version'] # override raw_headers values, if any + del msg['Content-Type'] + msg['MIME-Version'] = '1.0' + msg['Content-Type'] = 'multipart/mixed' + + related = cls() # container for alternative bodies and inline attachments + related['Content-Type'] = 'multipart/related' + msg.attach(related) + + alternatives = cls() # container for text and html bodies + alternatives['Content-Type'] = 'multipart/alternative' + related.attach(alternatives) + + if text is not None: + part = cls() + part['Content-Type'] = 'text/plain' + part.set_payload(text, charset=text_charset) + alternatives.attach(part) + if html is not None: + part = cls() + part['Content-Type'] = 'text/html' + part.set_payload(html, charset=html_charset) + alternatives.attach(part) + + if attachments is not None: + for attachment in attachments: + if attachment.is_inline_attachment(): + related.attach(attachment) + else: + msg.attach(attachment) + + return msg + + @classmethod + def construct_attachment_from_uploaded_file(cls, file, content_id=None): + # This pulls the entire file into memory; it would be better to implement + # some sort of lazy attachment where the content is only pulled in if/when + # requested (and then use file.chunks() to minimize memory usage) + return cls.construct_attachment( + content_type=file.content_type, + content=file.read(), + filename=file.name, + content_id=content_id, + charset=file.charset) + + @classmethod + def construct_attachment(cls, content_type, content, + charset=None, filename=None, content_id=None, base64=False): + part = cls() + part['Content-Type'] = content_type + part['Content-Disposition'] = 'inline' if content_id is not None else 'attachment' + + if filename is not None: + part.set_param('name', filename, header='Content-Type') + part.set_param('filename', filename, header='Content-Disposition') + + if content_id is not None: + part['Content-ID'] = angle_wrap(content_id) + + if base64: + content = b64decode(content) + + payload = content + if part.get_content_maintype() == 'message': + # email.Message parses message/rfc822 parts as a "multipart" (list) payload + # whose single item is the recursively-parsed message attachment + if isinstance(content, six.binary_type): + content = content.decode() + payload = [cls.parse_raw_mime(content)] + charset = None + + part.set_payload(payload, charset) + return part diff --git a/anymail/signals.py b/anymail/signals.py index c5231a8..533e5b0 100644 --- a/anymail/signals.py +++ b/anymail/signals.py @@ -45,6 +45,18 @@ class AnymailInboundEvent(AnymailEvent): def __init__(self, **kwargs): super(AnymailInboundEvent, self).__init__(**kwargs) + self.message = kwargs.pop('message', None) # anymail.inbound.AnymailInboundMessage + self.recipient = kwargs.pop('recipient', None) # str: envelope recipient + self.sender = kwargs.pop('sender', None) # str: envelope sender + + self.stripped_text = kwargs.pop('stripped_text', None) # cleaned of quotes/signatures (varies by ESP) + self.stripped_html = kwargs.pop('stripped_html', None) + self.spam_detected = kwargs.pop('spam_detected', None) # bool + self.spam_score = kwargs.pop('spam_score', None) # float: usually SpamAssassin + + # SPF status? + # DKIM status? + # DMARC status? (no ESP has documented support yet) class EventType: diff --git a/anymail/urls.py b/anymail/urls.py index e3e6cb5..75a3f77 100644 --- a/anymail/urls.py +++ b/anymail/urls.py @@ -1,19 +1,29 @@ from django.conf.urls import url -from .webhooks.mailgun import MailgunTrackingWebhookView -from .webhooks.mailjet import MailjetTrackingWebhookView -from .webhooks.mandrill import MandrillTrackingWebhookView -from .webhooks.postmark import PostmarkTrackingWebhookView -from .webhooks.sendgrid import SendGridTrackingWebhookView -from .webhooks.sparkpost import SparkPostTrackingWebhookView +from .webhooks.mailgun import MailgunInboundWebhookView, MailgunTrackingWebhookView +from .webhooks.mailjet import MailjetInboundWebhookView, MailjetTrackingWebhookView +from .webhooks.mandrill import MandrillCombinedWebhookView +from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView +from .webhooks.sendgrid import SendGridInboundWebhookView, SendGridTrackingWebhookView +from .webhooks.sparkpost import SparkPostInboundWebhookView, SparkPostTrackingWebhookView app_name = 'anymail' urlpatterns = [ + url(r'^mailgun/inbound(_mime)?/$', MailgunInboundWebhookView.as_view(), name='mailgun_inbound_webhook'), + url(r'^mailjet/inbound/$', MailjetInboundWebhookView.as_view(), name='mailjet_inbound_webhook'), + url(r'^postmark/inbound/$', PostmarkInboundWebhookView.as_view(), name='postmark_inbound_webhook'), + url(r'^sendgrid/inbound/$', SendGridInboundWebhookView.as_view(), name='sendgrid_inbound_webhook'), + url(r'^sparkpost/inbound/$', SparkPostInboundWebhookView.as_view(), name='sparkpost_inbound_webhook'), + url(r'^mailgun/tracking/$', MailgunTrackingWebhookView.as_view(), name='mailgun_tracking_webhook'), url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'), - url(r'^mandrill/tracking/$', MandrillTrackingWebhookView.as_view(), name='mandrill_tracking_webhook'), url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'), url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'), url(r'^sparkpost/tracking/$', SparkPostTrackingWebhookView.as_view(), name='sparkpost_tracking_webhook'), + + # Anymail uses a combined Mandrill webhook endpoint, to simplify Mandrill's key-validation scheme: + url(r'^mandrill/$', MandrillCombinedWebhookView.as_view(), name='mandrill_webhook'), + # This url is maintained for backwards compatibility with earlier Anymail releases: + url(r'^mandrill/tracking/$', MandrillCombinedWebhookView.as_view(), name='mandrill_tracking_webhook'), ] diff --git a/anymail/utils.py b/anymail/utils.py index 69ead1b..4ed5bb5 100644 --- a/anymail/utils.py +++ b/anymail/utils.py @@ -433,6 +433,18 @@ def rfc2822date(dt): return formatdate(timeval, usegmt=True) +def angle_wrap(s): + """Return s surrounded by angle brackets, added only if necessary""" + # This is the inverse behavior of email.utils.unquote + # (which you might think email.utils.quote would do, but it doesn't) + if len(s) > 0: + if s[0] != '<': + s = '<' + s + if s[-1] != '>': + s = s + '>' + return s + + def is_lazy(obj): """Return True if obj is a Django lazy object.""" # See django.utils.functional.lazy. (This appears to be preferred diff --git a/anymail/webhooks/base.py b/anymail/webhooks/base.py index 1f98bc6..2838031 100644 --- a/anymail/webhooks/base.py +++ b/anymail/webhooks/base.py @@ -1,4 +1,3 @@ -import re import warnings import six @@ -128,6 +127,9 @@ class AnymailBaseWebhookView(AnymailBasicAuthMixin, View): """ Read-only name of the ESP for this webhook view. - (E.g., MailgunTrackingWebhookView will return "Mailgun") + Subclasses must override with class attr. E.g.: + esp_name = "Postmark" + esp_name = "SendGrid" # (use ESP's preferred capitalization) """ - return re.sub(r'(Tracking|Inbox)WebhookView$', "", self.__class__.__name__) + raise NotImplementedError("%s.%s must declare esp_name class attr" % + (self.__class__.__module__, self.__class__.__name__)) diff --git a/anymail/webhooks/mailgun.py b/anymail/webhooks/mailgun.py index 396a871..7515119 100644 --- a/anymail/webhooks/mailgun.py +++ b/anymail/webhooks/mailgun.py @@ -8,13 +8,15 @@ from django.utils.timezone import utc from .base import AnymailBaseWebhookView from ..exceptions import AnymailWebhookValidationFailure -from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason +from ..inbound import AnymailInboundMessage +from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason from ..utils import get_anymail_setting, combine, querydict_getfirst class MailgunBaseWebhookView(AnymailBaseWebhookView): """Base view class for Mailgun webhooks""" + esp_name = "Mailgun" warn_if_no_basic_auth = False # because we validate against signature api_key = None # (Declaring class attr allows override by kwargs in View.as_view.) @@ -40,12 +42,6 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView): if not constant_time_compare(signature, expected_signature): raise AnymailWebhookValidationFailure("Mailgun webhook called with incorrect signature") - def parse_events(self, request): - return [self.esp_to_anymail_event(request.POST)] - - def esp_to_anymail_event(self, esp_event): - raise NotImplementedError() - class MailgunTrackingWebhookView(MailgunBaseWebhookView): """Handler for Mailgun delivery and engagement tracking webhooks""" @@ -75,6 +71,9 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): 607: RejectReason.SPAM, # previous spam complaint } + def parse_events(self, request): + return [self.esp_to_anymail_event(request.POST)] + def esp_to_anymail_event(self, esp_event): # esp_event is a Django QueryDict (from request.POST), # which has multi-valued fields, but is *not* case-insensitive. @@ -194,3 +193,69 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): 'opened': _common_event_fields, 'unsubscribed': _common_event_fields, } + + +class MailgunInboundWebhookView(MailgunBaseWebhookView): + """Handler for Mailgun inbound (route forward-to-url) webhook""" + + signal = inbound + + def parse_events(self, request): + return [self.esp_to_anymail_event(request)] + + def esp_to_anymail_event(self, request): + # Inbound uses the entire Django request as esp_event, because we need POST and FILES. + # Note that request.POST is case-sensitive (unlike email.message.Message headers). + esp_event = request + if 'body-mime' in request.POST: + # Raw-MIME + message = AnymailInboundMessage.parse_raw_mime(request.POST['body-mime']) + else: + # Fully-parsed + message = self.message_from_mailgun_parsed(request) + + message.envelope_sender = request.POST.get('sender', None) + message.envelope_recipient = request.POST.get('recipient', None) + message.stripped_text = request.POST.get('stripped-text', None) + message.stripped_html = request.POST.get('stripped-html', None) + + message.spam_detected = message.get('X-Mailgun-Sflag', 'No').lower() == 'yes' + try: + message.spam_score = float(message['X-Mailgun-Sscore']) + except (TypeError, ValueError): + pass + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=datetime.fromtimestamp(int(request.POST['timestamp']), tz=utc), + event_id=request.POST.get('token', None), + esp_event=esp_event, + message=message, + ) + + def message_from_mailgun_parsed(self, request): + """Construct a Message from Mailgun's "fully-parsed" fields""" + # Mailgun transcodes all fields to UTF-8 for "fully parsed" messages + try: + attachment_count = int(request.POST['attachment-count']) + except (KeyError, TypeError): + attachments = None + else: + # Load attachments from posted files: Mailgun file field names are 1-based + att_ids = ['attachment-%d' % i for i in range(1, attachment_count+1)] + att_cids = { # filename: content-id (invert content-id-map) + att_id: cid for cid, att_id + in json.loads(request.POST.get('content-id-map', '{}')).items() + } + attachments = [ + AnymailInboundMessage.construct_attachment_from_uploaded_file( + request.FILES[att_id], content_id=att_cids.get(att_id, None)) + for att_id in att_ids + ] + + return AnymailInboundMessage.construct( + headers=json.loads(request.POST['message-headers']), # includes From, To, Cc, Subject, etc. + text=request.POST.get('body-plain', None), + html=request.POST.get('body-html', None), + attachments=attachments, + ) diff --git a/anymail/webhooks/mailjet.py b/anymail/webhooks/mailjet.py index bb8c5da..ce536ea 100644 --- a/anymail/webhooks/mailjet.py +++ b/anymail/webhooks/mailjet.py @@ -4,12 +4,14 @@ from datetime import datetime from django.utils.timezone import utc from .base import AnymailBaseWebhookView -from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason +from ..inbound import AnymailInboundMessage +from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason class MailjetTrackingWebhookView(AnymailBaseWebhookView): """Handler for Mailjet delivery and engagement tracking webhooks""" + esp_name = "Mailjet" signal = tracking def parse_events(self, request): @@ -95,3 +97,84 @@ class MailjetTrackingWebhookView(AnymailBaseWebhookView): user_agent=esp_event.get('agent', None), esp_event=esp_event, ) + + +class MailjetInboundWebhookView(AnymailBaseWebhookView): + """Handler for Mailjet inbound (parse API) webhook""" + + esp_name = "Mailjet" + signal = inbound + + def parse_events(self, request): + esp_event = json.loads(request.body.decode('utf-8')) + return [self.esp_to_anymail_event(esp_event)] + + def esp_to_anymail_event(self, esp_event): + # You could _almost_ reconstruct the raw mime message from Mailjet's Headers and Parts fields, + # but it's not clear which multipart boundary to use on each individual Part. Although each Part's + # Content-Type header still has the multipart boundary, not knowing the parent part means typical + # nested multipart structures can't be reliably recovered from the data Mailjet provides. + # We'll just use our standarized multipart inbound constructor. + + headers = self._flatten_mailjet_headers(esp_event.get("Headers", {})) + attachments = [ + self._construct_mailjet_attachment(part, esp_event) + for part in esp_event.get("Parts", []) + if "Attachment" in part.get("ContentRef", "") # Attachment or InlineAttachment + ] + message = AnymailInboundMessage.construct( + headers=headers, + text=esp_event.get("Text-part", None), + html=esp_event.get("Html-part", None), + attachments=attachments, + ) + + message.envelope_sender = esp_event.get("Sender", None) + message.envelope_recipient = esp_event.get("Recipient", None) + + message.spam_detected = None # Mailjet doesn't provide a boolean; you'll have to interpret spam_score + try: + message.spam_score = float(esp_event['SpamAssassinScore']) + except (KeyError, TypeError, ValueError): + pass + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=None, # Mailjet doesn't provide inbound event timestamp (esp_event['Date'] is time sent) + event_id=None, # Mailjet doesn't provide an idempotent inbound event id + esp_event=esp_event, + message=message, + ) + + @staticmethod + def _flatten_mailjet_headers(headers): + """Convert Mailjet's dict-of-strings-and/or-lists header format to our list-of-name-value-pairs + + {'name1': 'value', 'name2': ['value1', 'value2']} + --> [('name1', 'value'), ('name2', 'value1'), ('name2', 'value2')] + """ + result = [] + for name, values in headers.items(): + if isinstance(values, list): # Mailjet groups repeated headers together as a list of values + for value in values: + result.append((name, value)) + else: + result.append((name, values)) # single-valued (non-list) header + return result + + def _construct_mailjet_attachment(self, part, esp_event): + # Mailjet includes unparsed attachment headers in each part; it's easiest to temporarily + # attach them to a MIMEPart for parsing. (We could just turn this into the attachment, + # but we want to use the payload handling from AnymailInboundMessage.construct_attachment later.) + part_headers = AnymailInboundMessage() # temporary container for parsed attachment headers + for name, value in self._flatten_mailjet_headers(part.get("Headers", {})): + part_headers.add_header(name, value) + + content_base64 = esp_event[part["ContentRef"]] # Mailjet *always* base64-encodes attachments + + return AnymailInboundMessage.construct_attachment( + content_type=part_headers.get_content_type(), + content=content_base64, base64=True, + filename=part_headers.get_filename(None), + content_id=part_headers.get("Content-ID", "") or None, + ) diff --git a/anymail/webhooks/mandrill.py b/anymail/webhooks/mandrill.py index 3d5257a..ee2dcef 100644 --- a/anymail/webhooks/mandrill.py +++ b/anymail/webhooks/mandrill.py @@ -8,8 +8,9 @@ from django.utils.crypto import constant_time_compare from django.utils.timezone import utc from .base import AnymailBaseWebhookView -from ..exceptions import AnymailWebhookValidationFailure, AnymailConfigurationError -from ..signals import tracking, AnymailTrackingEvent, EventType +from ..exceptions import AnymailWebhookValidationFailure +from ..inbound import AnymailInboundMessage +from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType from ..utils import get_anymail_setting, getfirst, get_request_uri @@ -59,22 +60,34 @@ class MandrillSignatureMixin(object): "Mandrill webhook called with incorrect signature (for url %r)" % url) -class MandrillBaseWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView): - """Base view class for Mandrill webhooks""" +class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView): + """Unified view class for Mandrill tracking and inbound webhooks""" + + esp_name = "Mandrill" warn_if_no_basic_auth = False # because we validate against signature + signal = None # set in esp_to_anymail_event def parse_events(self, request): esp_events = json.loads(request.POST['mandrill_events']) return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events] def esp_to_anymail_event(self, esp_event): - raise NotImplementedError() + """Route events to the inbound or tracking handler""" + esp_type = getfirst(esp_event, ['event', 'type'], 'unknown') + if esp_type == 'inbound': + assert self.signal is not tracking # Mandrill should never mix event types in the same batch + self.signal = inbound + return self.mandrill_inbound_to_anymail_event(esp_event) + else: + assert self.signal is not inbound # Mandrill should never mix event types in the same batch + self.signal = tracking + return self.mandrill_tracking_to_anymail_event(esp_event) -class MandrillTrackingWebhookView(MandrillBaseWebhookView): - - signal = tracking + # + # Tracking events + # event_types = { # Message events: @@ -94,13 +107,9 @@ class MandrillTrackingWebhookView(MandrillBaseWebhookView): 'inbound': EventType.INBOUND, } - def esp_to_anymail_event(self, esp_event): + def mandrill_tracking_to_anymail_event(self, esp_event): esp_type = getfirst(esp_event, ['event', 'type'], None) event_type = self.event_types.get(esp_type, EventType.UNKNOWN) - if event_type == EventType.INBOUND: - raise AnymailConfigurationError( - "You seem to have set Mandrill's *inbound* webhook URL " - "to Anymail's Mandrill *tracking* webhook URL.") try: timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc) @@ -149,3 +158,33 @@ class MandrillTrackingWebhookView(MandrillBaseWebhookView): timestamp=timestamp, user_agent=esp_event.get('user_agent', None), ) + + # + # Inbound events + # + + def mandrill_inbound_to_anymail_event(self, esp_event): + # It's easier (and more accurate) to just work from the original raw mime message + message = AnymailInboundMessage.parse_raw_mime(esp_event['msg']['raw_msg']) + message.envelope_sender = None # (Mandrill's 'sender' field only applies to outbound messages) + message.envelope_recipient = esp_event['msg'].get('email', None) + + message.spam_detected = None # no simple boolean field; would need to parse the spam_report + message.spam_score = esp_event['msg'].get('spam_report', {}).get('score', None) + + try: + timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc) + except (KeyError, ValueError): + timestamp = None + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=timestamp, + event_id=None, # Mandrill doesn't provide an idempotent inbound message event id + esp_event=esp_event, + message=message, + ) + + +# Backwards-compatibility: earlier Anymail versions had only MandrillTrackingWebhookView: +MandrillTrackingWebhookView = MandrillCombinedWebhookView diff --git a/anymail/webhooks/postmark.py b/anymail/webhooks/postmark.py index 5f19585..ec1c8e7 100644 --- a/anymail/webhooks/postmark.py +++ b/anymail/webhooks/postmark.py @@ -4,13 +4,16 @@ from django.utils.dateparse import parse_datetime from .base import AnymailBaseWebhookView from ..exceptions import AnymailConfigurationError -from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason -from ..utils import getfirst +from ..inbound import AnymailInboundMessage +from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason +from ..utils import getfirst, EmailAddress class PostmarkBaseWebhookView(AnymailBaseWebhookView): """Base view class for Postmark webhooks""" + esp_name = "Postmark" + def parse_events(self, request): esp_event = json.loads(request.body.decode('utf-8')) return [self.esp_to_anymail_event(esp_event)] @@ -107,3 +110,72 @@ class PostmarkTrackingWebhookView(PostmarkBaseWebhookView): user_agent=esp_event.get('UserAgent', None), click_url=esp_event.get('OriginalLink', None), ) + + +class PostmarkInboundWebhookView(PostmarkBaseWebhookView): + """Handler for Postmark inbound webhook""" + + signal = inbound + + def esp_to_anymail_event(self, esp_event): + attachments = [ + AnymailInboundMessage.construct_attachment( + content_type=attachment["ContentType"], + content=attachment["Content"], base64=True, + filename=attachment.get("Name", "") or None, + content_id=attachment.get("ContentID", "") or None, + ) + for attachment in esp_event.get("Attachments", []) + ] + + message = AnymailInboundMessage.construct( + from_email=self._address(esp_event.get("FromFull")), + to=', '.join([self._address(to) for to in esp_event.get("ToFull", [])]), + cc=', '.join([self._address(cc) for cc in esp_event.get("CcFull", [])]), + # bcc? Postmark specs this for inbound events, but it's unclear how it could occur + subject=esp_event.get("Subject", ""), + headers=[(header["Name"], header["Value"]) for header in esp_event.get("Headers", [])], + text=esp_event.get("TextBody", ""), + html=esp_event.get("HtmlBody", ""), + attachments=attachments, + ) + + # Postmark strips these headers and provides them as separate event fields: + if "Date" in esp_event and "Date" not in message: + message["Date"] = esp_event["Date"] + if "ReplyTo" in esp_event and "Reply-To" not in message: + message["Reply-To"] = esp_event["ReplyTo"] + + # Postmark doesn't have a separate envelope-sender field, but it can be extracted + # from the Received-SPF header that Postmark will have added: + if len(message.get_all("Received-SPF", [])) == 1: # (more than one? someone's up to something weird) + received_spf = message["Received-SPF"].lower() + if received_spf.startswith("pass") or received_spf.startswith("neutral"): # not fail/softfail + message.envelope_sender = message.get_param("envelope-from", None, header="Received-SPF") + + message.envelope_recipient = esp_event.get("OriginalRecipient", None) + message.stripped_text = esp_event.get("StrippedTextReply", None) + + message.spam_detected = message.get('X-Spam-Status', 'No').lower() == 'yes' + try: + message.spam_score = float(message['X-Spam-Score']) + except (TypeError, ValueError): + pass + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=None, # Postmark doesn't provide inbound event timestamp + event_id=esp_event.get("MessageID", None), # Postmark uuid, different from Message-ID mime header + esp_event=esp_event, + message=message, + ) + + @staticmethod + def _address(full): + """Return an formatted email address from a Postmark inbound {From,To,Cc}Full dict""" + if full is None: + return "" + return str(EmailAddress( + display_name=full.get('Name', ""), + addr_spec=full.get("Email", ""), + )) diff --git a/anymail/webhooks/sendgrid.py b/anymail/webhooks/sendgrid.py index 0bd22d2..13f8384 100644 --- a/anymail/webhooks/sendgrid.py +++ b/anymail/webhooks/sendgrid.py @@ -4,25 +4,20 @@ from datetime import datetime from django.utils.timezone import utc from .base import AnymailBaseWebhookView -from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason +from ..inbound import AnymailInboundMessage +from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason -class SendGridBaseWebhookView(AnymailBaseWebhookView): - """Base view class for SendGrid webhooks""" +class SendGridTrackingWebhookView(AnymailBaseWebhookView): + """Handler for SendGrid delivery and engagement tracking webhooks""" + + esp_name = "SendGrid" + signal = tracking def parse_events(self, request): esp_events = json.loads(request.body.decode('utf-8')) return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events] - def esp_to_anymail_event(self, esp_event): - raise NotImplementedError() - - -class SendGridTrackingWebhookView(SendGridBaseWebhookView): - """Handler for SendGrid delivery and engagement tracking webhooks""" - - signal = tracking - event_types = { # Map SendGrid event: Anymail normalized type 'bounce': EventType.BOUNCED, @@ -120,3 +115,78 @@ class SendGridTrackingWebhookView(SendGridBaseWebhookView): 'url_offset', # click tracking 'useragent', # click/open tracking } + + +class SendGridInboundWebhookView(AnymailBaseWebhookView): + """Handler for SendGrid inbound webhook""" + + esp_name = "SendGrid" + signal = inbound + + def parse_events(self, request): + return [self.esp_to_anymail_event(request)] + + def esp_to_anymail_event(self, request): + # Inbound uses the entire Django request as esp_event, because we need POST and FILES. + # Note that request.POST is case-sensitive (unlike email.message.Message headers). + esp_event = request + if 'headers' in request.POST: + # Default (not "Send Raw") inbound fields + message = self.message_from_sendgrid_parsed(esp_event) + elif 'email' in request.POST: + # "Send Raw" full MIME + message = AnymailInboundMessage.parse_raw_mime(request.POST['email']) + else: + raise KeyError("Invalid SendGrid inbound event data (missing both 'headers' and 'email' fields)") + + try: + envelope = json.loads(request.POST['envelope']) + except (KeyError, TypeError, ValueError): + pass + else: + message.envelope_sender = envelope['from'] + message.envelope_recipient = envelope['to'][0] + + message.spam_detected = None # no simple boolean field; would need to parse the spam_report + try: + message.spam_score = float(request.POST['spam_score']) + except (KeyError, TypeError, ValueError): + pass + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=None, # SendGrid doesn't provide an inbound event timestamp + event_id=None, # SendGrid doesn't provide an idempotent inbound message event id + esp_event=esp_event, + message=message, + ) + + def message_from_sendgrid_parsed(self, request): + """Construct a Message from SendGrid's "default" (non-raw) fields""" + + try: + charsets = json.loads(request.POST['charsets']) + except (KeyError, ValueError): + charsets = {} + + try: + attachment_info = json.loads(request.POST['attachment-info']) + except (KeyError, ValueError): + attachments = None + else: + # Load attachments from posted files + attachments = [ + AnymailInboundMessage.construct_attachment_from_uploaded_file( + request.FILES[att_id], + content_id=attachment_info[att_id].get("content-id", None)) + for att_id in sorted(attachment_info.keys()) + ] + + return AnymailInboundMessage.construct( + raw_headers=request.POST.get('headers', ""), # includes From, To, Cc, Subject, etc. + text=request.POST.get('text', None), + text_charset=charsets.get('text', 'utf-8'), + html=request.POST.get('html', None), + html_charset=charsets.get('html', 'utf-8'), + attachments=attachments, + ) diff --git a/anymail/webhooks/sparkpost.py b/anymail/webhooks/sparkpost.py index b958646..22f412e 100644 --- a/anymail/webhooks/sparkpost.py +++ b/anymail/webhooks/sparkpost.py @@ -1,16 +1,20 @@ import json +from base64 import b64decode from datetime import datetime from django.utils.timezone import utc from .base import AnymailBaseWebhookView from ..exceptions import AnymailConfigurationError -from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason +from ..inbound import AnymailInboundMessage +from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason class SparkPostBaseWebhookView(AnymailBaseWebhookView): """Base view class for SparkPost webhooks""" + esp_name = "SparkPost" + def parse_events(self, request): raw_events = json.loads(request.body.decode('utf-8')) unwrapped_events = [self.unwrap_event(raw_event) for raw_event in raw_events] @@ -92,7 +96,7 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView): } def esp_to_anymail_event(self, event_class, event, raw_event): - if event_class == 'relay_event': + if event_class == 'relay_message': # This is an inbound event raise AnymailConfigurationError( "You seem to have set SparkPost's *inbound* relay webhook URL " @@ -134,3 +138,37 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView): user_agent=event.get('user_agent', None), esp_event=raw_event, ) + + +class SparkPostInboundWebhookView(SparkPostBaseWebhookView): + """Handler for SparkPost inbound relay webhook""" + + signal = inbound + + def esp_to_anymail_event(self, event_class, event, raw_event): + if event_class != 'relay_message': + # This is not an inbound event + raise AnymailConfigurationError( + "You seem to have set SparkPost's *tracking* webhook URL " + "to Anymail's SparkPost *inbound* relay webhook URL.") + + if event['protocol'] != 'smtp': + raise AnymailConfigurationError( + "You cannot use Anymail's webhooks for SparkPost '{protocol}' relay events. " + "Anymail only handles the 'smtp' protocol".format(protocol=event['protocol'])) + + raw_mime = event['content']['email_rfc822'] + if event['content']['email_rfc822_is_base64']: + raw_mime = b64decode(raw_mime).decode('utf-8') + message = AnymailInboundMessage.parse_raw_mime(raw_mime) + + message.envelope_sender = event.get('msg_from', None) + message.envelope_recipient = event.get('rcpt_to', None) + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=None, # SparkPost does not provide a relay event timestamp + event_id=None, # SparkPost does not provide an idempotent id for relay events + esp_event=raw_event, + message=message, + ) diff --git a/docs/esps/index.rst b/docs/esps/index.rst index 3df65a5..851d146 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -48,6 +48,10 @@ Email Service Provider |Mailgun| |Mailjet| |Mandrill| --------------------------------------------------------------------------------------------------------------------- :attr:`~AnymailMessage.anymail_status` Yes Yes Yes Yes Yes Yes |AnymailTrackingEvent| from webhooks Yes Yes Yes Yes Yes Yes + +.. rubric:: :ref:`Inbound handling ` +--------------------------------------------------------------------------------------------------------------------- +|AnymailInboundEvent| from webhooks Yes Yes Yes Yes Yes Yes ============================================ ========== ========== ========== ========== ========== =========== @@ -63,6 +67,7 @@ meaningless. (And even specific features don't matter if you don't plan to use t .. |SendGrid| replace:: :ref:`sendgrid-backend` .. |SparkPost| replace:: :ref:`sparkpost-backend` .. |AnymailTrackingEvent| replace:: :class:`~anymail.signals.AnymailTrackingEvent` +.. |AnymailInboundEvent| replace:: :class:`~anymail.signals.AnymailInboundEvent` Other ESPs diff --git a/docs/esps/mailgun.rst b/docs/esps/mailgun.rst index a5395d4..3802e50 100644 --- a/docs/esps/mailgun.rst +++ b/docs/esps/mailgun.rst @@ -215,3 +215,36 @@ a Django :class:`~django.http.QueryDict` object of `Mailgun event fields`_. .. _Mailgun dashboard: https://mailgun.com/app/dashboard .. _Mailgun event fields: https://documentation.mailgun.com/user_manual.html#webhooks + + +.. _mailgun-inbound: + +Inbound webhook +--------------- + +If you want to receive email from Mailgun through Anymail's normalized :ref:`inbound ` +handling, follow Mailgun's `Receiving, Storing and Fowarding Messages`_ guide to set up +an inbound route that forwards to Anymail's inbound webhook. (You can configure routes +using Mailgun's API, or simply using the "Routes" tab in your `Mailgun dashboard`_.) + +The *action* for your route will be either: + + :samp:`forward("https://{random}:{random}@{yoursite.example.com}/anymail/mailgun/inbound/")` + :samp:`forward("https://{random}:{random}@{yoursite.example.com}/anymail/mailgun/inbound_mime/")` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_AUTHORIZATION` shared secret + * *yoursite.example.com* is your Django site + +Anymail accepts either of Mailgun's "fully-parsed" (.../inbound/) and "raw MIME" (.../inbound_mime/) +formats; the URL tells Mailgun which you want. Because Anymail handles parsing and normalizing the data, +both are equally easy to use. The raw MIME option will give the most accurate representation of *any* +received email (including complex forms like multi-message mailing list digests). The fully-parsed option +*may* use less memory while processing messages with many large attachments. + +If you want to use Anymail's normalized :attr:`~anymail.inbound.AnymailInboundMessage.spam_detected` and +:attr:`~anymail.inbound.AnymailInboundMessage.spam_score` attributes, you'll need to set your Mailgun +domain's inbound spam filter to "Deliver spam, but add X-Mailgun-SFlag and X-Mailgun-SScore headers" +(in the `Mailgun dashboard`_ on the "Domains" tab). + +.. _Receiving, Storing and Fowarding Messages: + https://documentation.mailgun.com/en/latest/user_manual.html#receiving-forwarding-and-storing-messages diff --git a/docs/esps/mailjet.rst b/docs/esps/mailjet.rst index b4dde00..06b8784 100644 --- a/docs/esps/mailjet.rst +++ b/docs/esps/mailjet.rst @@ -249,3 +249,26 @@ for each event in the batch.) .. _Event tracking (triggers): https://app.mailjet.com/account/triggers .. _Mailjet event: https://dev.mailjet.com/guides/#events + + +.. _mailjet-inbound: + +Inbound webhook +--------------- + +If you want to receive email from Mailjet through Anymail's normalized :ref:`inbound ` +handling, follow Mailjet's `Parse API inbound emails`_ guide to set up Anymail's inbound webhook. + +The parseroute Url parameter will be: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/mailjet/inbound/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_AUTHORIZATION` shared secret + * *yoursite.example.com* is your Django site + +Once you've done Mailjet's "basic setup" to configure the Parse API webhook, you can skip +ahead to the "use your own domain" section of their guide. (Anymail normalizes the inbound +event for you, so you won't need to worry about Mailjet's event and attachment formats.) + +.. _Parse API inbound emails: + https://dev.mailjet.com/guides/#parse-api-inbound-emails diff --git a/docs/esps/mandrill.rst b/docs/esps/mandrill.rst index 24c4b38..d3ac498 100644 --- a/docs/esps/mandrill.rst +++ b/docs/esps/mandrill.rst @@ -185,27 +185,31 @@ See the `Mandrill's template docs`_ for more information. .. _mandrill-webhooks: +.. _mandrill-inbound: -Status tracking webhooks ------------------------- +Status tracking and inbound webhooks +------------------------------------ -If you are using Anymail's normalized :ref:`status tracking `, -setting up Anymail's webhook URL requires deploying your Django project twice: +If you are using Anymail's normalized :ref:`status tracking ` +and/or :ref:`inbound ` handling, setting up Anymail's webhook URL +requires deploying your Django project twice: 1. First, follow the instructions to - :ref:`configure Anymail's webhooks `. You *must* - deploy before adding the webhook URL to Mandrill, because it will attempt + :ref:`configure Anymail's webhooks `. You *must deploy* + before adding the webhook URL to Mandrill, because Mandrill will attempt to verify the URL against your production server. - Follow `Mandrill's instructions`_ to add Anymail's webhook URL in their settings: + Once you've deployed, then set Anymail's webhook URL in Mandrill, following their + instructions for `tracking event webhooks`_ (be sure to check the boxes for the + events you want to receive) and/or `inbound route webhooks`_. + In either case, the webhook url is: - :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/mandrill/tracking/` + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/mandrill/` * *random:random* is an :setting:`ANYMAIL_WEBHOOK_AUTHORIZATION` shared secret * *yoursite.example.com* is your Django site - - Be sure to check the boxes in the Mandrill settings for the event types you want to receive. - The same Anymail tracking URL can handle all Mandrill "message" and "change" events. + * (Note: Unlike Anymail's other supported ESPs, the Mandrill webhook uses this + single url for both tracking and inbound events.) 2. Mandrill will provide you a "webhook authentication key" once it verifies the URL is working. Add this to your Django project's Anymail settings under @@ -226,7 +230,7 @@ else fails, you can set Anymail's :setting:`MANDRILL_WEBHOOK_URL ` + to figure out the complete webhook url that Mandrill called. + + If you are experiencing webhook authorization errors, the best solution is to adjust + your Django :setting:`SECURE_PROXY_SSL_HEADER`, :setting:`USE_X_FORWARDED_HOST`, and/or + :setting:`USE_X_FORWARDED_PORT` settings to work with your proxy server. + If that's not possible, you can set :setting:`ANYMAIL_MANDRILL_WEBHOOK_URL` to explicitly + declare the webhook url. Changes to EmailMessage attributes @@ -393,14 +414,17 @@ parameters is that most logging and analytics systems are aware of the need to keep auth secret.) Anymail replaces `djrill.signals.webhook_event` with -`anymail.signals.tracking` for delivery tracking events. -(It does not currently handle inbound message webhooks.) +`anymail.signals.tracking` for delivery tracking events, +and `anymail.signals.inbound` for inbound events. Anymail parses and normalizes -the event data passed to the signal receiver: see :ref:`event-tracking`. +the event data passed to the signal receiver: see :ref:`event-tracking` +and :ref:`inbound`. The equivalent of Djrill's ``data`` parameter is available to your signal receiver as :attr:`event.esp_event `, and for most events, the equivalent of Djrill's ``event_type`` parameter is `event.esp_event['event']`. But consider working with Anymail's -normalized :class:`~anymail.signals.AnymailTrackingEvent` instead. +normalized :class:`~anymail.signals.AnymailTrackingEvent` and +:class:`~anymail.signals.AnymailInboundEvent` instead for easy portability +to other ESPs. diff --git a/docs/esps/postmark.rst b/docs/esps/postmark.rst index 2bd1db9..7ec25d7 100644 --- a/docs/esps/postmark.rst +++ b/docs/esps/postmark.rst @@ -201,3 +201,26 @@ a `dict` of Postmark `delivery `_ webhook data. .. _Postmark account settings: https://account.postmarkapp.com/servers + + +.. _postmark-inbound: + +Inbound webhook +--------------- + +If you want to receive email from Postmark through Anymail's normalized :ref:`inbound ` +handling, follow Postmark's `Inbound Processing`_ guide to configure +an inbound server pointing to Anymail's inbound webhook. + +The InboundHookUrl setting will be: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/postmark/inbound/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_AUTHORIZATION` shared secret + * *yoursite.example.com* is your Django site + +Anymail handles the "parse an email" part of Postmark's instructions for you, but you'll +likely want to work through the other sections to set up a custom inbound domain, and +perhaps configure inbound spam blocking. + +.. _Inbound Processing: https://postmarkapp.com/developer/user-guide/inbound diff --git a/docs/esps/sendgrid.rst b/docs/esps/sendgrid.rst index 8e0def9..7a1708c 100644 --- a/docs/esps/sendgrid.rst +++ b/docs/esps/sendgrid.rst @@ -302,6 +302,37 @@ for each event in the batch.) .. _Sendgrid event: https://sendgrid.com/docs/API_Reference/Webhooks/event.html +.. _sendgrid-inbound: + +Inbound webhook +--------------- + +If you want to receive email from SendGrid through Anymail's normalized :ref:`inbound ` +handling, follow SendGrid's `Inbound Parse Webhook`_ guide to set up +Anymail's inbound webhook. + +The Destination URL setting will be: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/sendgrid/inbound/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_AUTHORIZATION` shared secret + * *yoursite.example.com* is your Django site + +Be sure the URL has a trailing slash. (SendGrid's inbound processing won't follow Django's +:setting:`APPEND_SLASH` redirect.) + +If you want to use Anymail's normalized :attr:`~anymail.inbound.AnymailInboundMessage.spam_detected` and +:attr:`~anymail.inbound.AnymailInboundMessage.spam_score` attributes, be sure to enable the "Check +incoming emails for spam" checkbox. + +You have a choice for SendGrid's "POST the raw, full MIME message" checkbox. Anymail will handle +either option (and you can change it at any time). Enabling raw MIME will give the most accurate +representation of *any* received email (including complex forms like multi-message mailing list +digests). But disabling it *may* use less memory while processing messages with many large attachments. + +.. _Inbound Parse Webhook: + https://sendgrid.com/docs/Classroom/Basics/Inbound_Parse_Webhook/setting_up_the_inbound_parse_webhook.html + .. _sendgrid-v3-upgrade: diff --git a/docs/esps/sparkpost.rst b/docs/esps/sparkpost.rst index a7a6a78..47ad8ed 100644 --- a/docs/esps/sparkpost.rst +++ b/docs/esps/sparkpost.rst @@ -220,3 +220,23 @@ The esp_event is the raw, `wrapped json event structure`_ as provided by SparkPo https://support.sparkpost.com/customer/portal/articles/1976204-webhook-event-reference .. _wrapped json event structure: https://support.sparkpost.com/customer/en/portal/articles/2311698-comparing-webhook-and-message-event-data + + +.. _sparkpost-inbound: + +Inbound webhook +--------------- + +If you want to receive email from SparkPost through Anymail's normalized :ref:`inbound ` +handling, follow SparkPost's `Enabling Inbound Email Relaying`_ guide to set up +Anymail's inbound webhook. + +The target parameter for the Relay Webhook will be: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/sparkpost/inbound/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_AUTHORIZATION` shared secret + * *yoursite.example.com* is your Django site + +.. _Enabling Inbound Email Relaying: + https://www.sparkpost.com/docs/tech-resources/inbound-email-relay-webhook/ diff --git a/docs/inbound.rst b/docs/inbound.rst new file mode 100644 index 0000000..189b569 --- /dev/null +++ b/docs/inbound.rst @@ -0,0 +1,454 @@ +.. _inbound: + +Receiving mail +============== + +.. versionadded:: 1.3 + +For ESPs that support receiving inbound email, Anymail offers normalized handling +of inbound events. + +If you didn't set up webhooks when first installing Anymail, you'll need to +:ref:`configure webhooks ` to get started with inbound email. +(You should also review :ref:`securing-webhooks`.) + +Once you've enabled webhooks, Anymail will send a ``anymail.signals.inbound`` +custom Django :mod:`signal ` for each ESP inbound message it receives. +You can connect your own receiver function to this signal for further processing. +(This is very much like how Anymail handles :ref:`status tracking ` +events for sent messages. Inbound events just use a different signal receiver +and have different event parameters.) + +Be sure to read Django's :doc:`listening to signals ` docs +for information on defining and connecting signal receivers. + +Example: + +.. code-block:: python + + from anymail.signals import inbound + from django.dispatch import receiver + + @receiver(inbound) # add weak=False if inside some other function/class + def handle_inbound(sender, event, esp_name, **kwargs): + message = event.message + print("Received message from %s (envelope sender %s) with subject '%s'" % ( + message.from_email, message.envelope_sender, message.subject)) + +Some ESPs batch up multiple inbound messages into a single webhook call. Anymail will +invoke your signal receiver once, separately, for each message in the batch. + +.. _inbound-security: + +.. warning:: **Be careful with inbound email** + + Inbound email is user-supplied content. There are all kinds of ways a + malicious sender can abuse the email format to give your app misleading + or dangerous data. Treat inbound email content with the same suspicion + you'd apply to any user-submitted data. Among other concerns: + + * Senders can spoof the From header. An inbound message's + :attr:`~anymail.inbound.AnymailInboundMessage.from_email` may + or may not match the actual address that sent the message. (There are both + legitimate and malicious uses for this capability.) + + * Most other fields in email can be falsified. E.g., an inbound message's + :attr:`~anymail.inbound.AnymailInboundMessage.date` may or may not accurately + reflect when the message was sent. + + * Inbound attachments have the same security concerns as user-uploaded files. + If you process inbound attachments, you'll need to verify that the + attachment content is valid. + + This is particularly important if you publish the attachment content + through your app. For example, an "image" attachment could actually contain an + executable file or raw HTML. You wouldn't want to serve that as a user's avatar. + + It's *not* sufficient to check the attachment's content-type or + filename extension---senders can falsify both of those. + Consider `using python-magic`_ or a similar approach + to validate the *actual attachment content*. + + The Django docs have additional notes on + :ref:`user-supplied content security `. + +.. _using python-magic: + http://blog.hayleyanderson.us/2015/07/18/validating-file-types-in-django/ + + +.. _inbound-event: + +Normalized inbound event +------------------------ + +.. class:: anymail.signals.AnymailInboundEvent + + The `event` parameter to Anymail's `inbound` + :ref:`signal receiver ` is an object + with the following attributes: + + .. attribute:: message + + An :class:`~anymail.inbound.AnymailInboundMessage` representing the email + that was received. Most of what you're interested in will be on this `message` + attribute. See the full details :ref:`below `. + + .. attribute:: event_type + + A normalized `str` identifying the type of event. For inbound events, + this is always `'inbound'`. + + .. attribute:: timestamp + + A `~datetime.datetime` indicating when the inbound event was generated + by the ESP, if available; otherwise `None`. (Very few ESPs provide this info.) + + This is typically when the ESP received the message or shortly + thereafter. (Use :attr:`event.message.date ` + if you're interested in when the message was sent.) + + (The timestamp's timezone is often UTC, but the exact behavior depends + on your ESP and account settings. Anymail ensures that this value is + an *aware* datetime with an accurate timezone.) + + .. attribute:: event_id + + A `str` unique identifier for the event, if available; otherwise `None`. + Can be used to avoid processing the same event twice. The exact format varies + by ESP, and very few ESPs provide an event_id for inbound messages. + + An alternative approach to avoiding duplicate processing is to use the + inbound message's :mailheader:`Message-ID` header (``event.message['Message-ID']``). + + .. attribute:: esp_event + + The "raw" event data from the ESP, deserialized into a python data structure. + For most ESPs this is either parsed JSON (as a `dict`), or sometimes the + complete Django :class:`~django.http.HttpRequest` received by the webhook. + + This gives you (non-portable) access to original event provided by your ESP, + which can be helpful if you need to access data Anymail doesn't normalize. + + +.. _inbound-message: + +Normalized inbound message +-------------------------- + +.. class:: anymail.inbound.AnymailInboundMessage + + The :attr:`~AnymailInboundEvent.message` attribute of an :class:`AnymailInboundEvent` + is an AnymailInboundMessage---an extension of Python's standard :class:`email.message.Message` + with additional features to simplify inbound handling. + + In addition to the base :class:`~email.message.Message` functionality, it includes these attributes: + + .. attribute:: envelope_sender + + The actual sending address of the inbound message, as determined by your ESP. + This is a `str` "addr-spec"---just the email address portion without any display + name (``"sender@example.com"``)---or `None` if the ESP didn't provide a value. + + The envelope sender often won't match the message's From header---for example, + messages sent on someone's behalf (mailing lists, invitations) or when a spammer + deliberately falsifies the From address. + + .. attribute:: envelope_recipient + + The actual destination address the inbound message was delivered to. + This is a `str` "addr-spec"---just the email address portion without any display + name (``"recipient@example.com"``)---or `None` if the ESP didn't provide a value. + + The envelope recipient may not appear in the To or Cc recipient lists---for example, + if your inbound address is bcc'd on a message. + + .. attribute:: from_email + + The value of the message's From header. Anymail converts this to an + :class:`~anymail.utils.EmailAddress` object, which makes it easier to access + the parsed address fields: + + .. code-block:: python + + >>> str(message.from_email) # the fully-formatted address + '"Dr. Justin Customer, CPA" ' + >>> message.from_email.addr_spec # the "email" portion of the address + 'jcustomer@example.com' + >>> message.from_email.display_name # empty string if no display name + 'Dr. Justin Customer, CPA' + >>> message.from_email.domain + 'example.com' + >>> message.from_email.username + 'jcustomer' + + (This API is borrowed from Python 3.6's :class:`email.headerregistry.Address`.) + + If the message has an invalid or missing From header, this property will be `None`. + Note that From headers can be misleading; see :attr:`envelope_sender`. + + .. attribute:: to + + A `list` of of parsed :class:`~anymail.utils.EmailAddress` objects from the To header, + or an empty list if that header is missing or invalid. Each address in the list + has the same properties as shown above for :attr:`from_email`. + + See :attr:`envelope_recipient` if you need to know the actual inbound address + that received the inbound message. + + .. attribute:: cc + + A `list` of of parsed :class:`~anymail.utils.EmailAddress` objects, like :attr:`to`, + but from the Cc headers. + + .. attribute:: subject + + The value of the message's Subject header, as a `str`, or `None` if there is no Subject + header. + + .. attribute:: date + + The value of the message's Date header, as a `~datetime.datetime` object, or `None` + if the Date header is missing or invalid. This attribute will almost always be an + aware datetime (with a timezone); in rare cases it can be naive if the sending mailer + indicated that it had no timezone information available. + + The Date header is the sender's claim about when it sent the message, which isn't + necessarily accurate. (If you need to know when the message was received at your ESP, + that might be available in :attr:`event.timestamp `. + If not, you'd need to parse the messages's :mailheader:`Received` headers, + which can be non-trivial.) + + .. attribute:: text + + The message's plaintext message body as a `str`, or `None` if the + message doesn't include a plaintext body. + + .. attribute:: html + + The message's HTML message body as a `str`, or `None` if the + message doesn't include an HTML body. + + .. attribute:: attachments + + A `list` of all (non-inline) attachments to the message, or an empty list if there are + no attachments. See :ref:`inbound-attachments` below for the contents of each list item. + + .. attribute:: inline_attachments + + A `dict` mapping inline Content-ID references to attachment content. Each key is an + "unquoted" cid without angle brackets. E.g., if the :attr:`html` body contains + ````, you could get that inline image using + ``message.inline_attachments["abc123..."]``. + + The content of each attachment is described in :ref:`inbound-attachments` below. + + .. attribute:: spam_score + + A `float` spam score (usually from SpamAssassin) if your ESP provides it; otherwise `None`. + The range of values varies by ESP and spam-filtering configuration, so you may need to + experiment to find a useful threshold. + + .. attribute:: spam_detected + + If your ESP provides a simple yes/no spam determination, a `bool` indicating whether the + ESP thinks the inbound message is probably spam. Otherwise `None`. (Most ESPs just assign + a :attr:`spam_score` and leave its interpretation up to you.) + + .. attribute:: stripped_text + + If provided by your ESP, a simplified version the inbound message's plaintext body; + otherwise `None`. + + What exactly gets "stripped" varies by ESP, but it often omits quoted replies + and sometimes signature blocks. (And ESPs who do offer stripped bodies + usually consider the feature experimental.) + + .. attribute:: stripped_html + + Like :attr:`stripped_text`, but for the HTML body. (Very few ESPs support this.) + + .. rubric:: Other headers, complex messages, etc. + + You can use all of Python's :class:`email.message.Message` features with an + AnymailInboundMessage. For example, you can access message headers using + Message's :meth:`mapping interface `: + + .. code-block:: python + + message['reply-to'] # the Reply-To header (header keys are case-insensitive) + message.getall('DKIM-Signature') # list of all DKIM-Signature headers + + And you can use Message methods like :meth:`~email.message.Message.walk` and + :meth:`~email.message.Message.get_content_type` to examine more-complex + multipart MIME messages (digests, delivery reports, or whatever). + + +.. _inbound-attachments: + +Handling Inbound Attachments +---------------------------- + +Anymail converts each inbound attachment to a specialized MIME object with +additional methods for handling attachments and integrating with Django. +It also backports some helpful MIME methods from newer versions of Python +to all versions supported by Anymail. + +The attachment objects in an AnymailInboundMessage's +:attr:`~AnymailInboundMessage.attachments` list and +:attr:`~AnymailInboundMessage.inline_attachments` dict +have these methods: + +.. class:: AnymailInboundMessage + + .. method:: as_uploaded_file() + + Returns the attachment converted to a Django :class:`~django.core.files.uploadedfile.UploadedFile` + object. This is suitable for assigning to a model's :class:`~django.db.models.FileField` + or :class:`~django.db.models.ImageField`: + + .. code-block:: python + + # allow users to mail in jpeg attachments to set their profile avatars... + if attachment.get_content_type() == "image/jpeg": + # for security, you must verify the content is really a jpeg + # (you'll need to supply the is_valid_jpeg function) + if is_valid_jpeg(attachment.get_content_bytes()): + user.profile.avatar_image = attachment.as_uploaded_file() + + See Django's docs on :doc:`django:topics/files` for more information + on working with uploaded files. + + .. method:: get_content_type() + .. method:: get_content_maintype() + .. method:: get_content_subtype() + + The type of attachment content, as specified by the sender. (But remember + attachments are essentially user-uploaded content, so you should + :ref:`never trust the sender `.) + + See the Python docs for more info on :meth:`email.message.Message.get_content_type`, + :meth:`~email.message.Message.get_content_maintype`, and + :meth:`~email.message.Message.get_content_subtype`. + + (Note that you *cannot* determine the attachment type using code like + ``issubclass(attachment, email.mime.image.MIMEImage)``. You should instead use something + like ``attachment.get_content_maintype() == 'image'``. The email package's specialized + MIME subclasses are designed for constructing new messages, and aren't used + for parsing existing, inbound email messages.) + + .. method:: get_filename() + + The original filename of the attachment, as specified by the sender. + + *Never* use this filename directly to write files---that would be a huge security hole. + (What would your app do if the sender gave the filename "/etc/passwd" or "../settings.py"?) + + .. method:: is_attachment() + + Returns `True` for a (non-inline) attachment, `False` otherwise. + (Anymail back-ports Python 3.4.2's :meth:`~email.message.EmailMessage.is_attachment` method + to all supported versions.) + + .. method:: is_inline_attachment() + + Returns `True` for an inline attachment (one with :mailheader:`Content-Disposition` "inline"), + `False` otherwise. + + .. method:: get_content_disposition() + + Returns the lowercased value (without parameters) of the attachment's + :mailheader:`Content-Disposition` header. The return value should be either "inline" + or "attachment", or `None` if the attachment is somehow missing that header. + + (Anymail back-ports Python 3.5's :meth:`~email.message.Message.get_content_disposition` + method to all supported versions.) + + .. method:: get_content_text(charset='utf-8') + + Returns the content of the attachment decoded to a `str` in the given charset. + (This is generally only appropriate for text or message-type attachments.) + + .. method:: get_content_bytes() + + Returns the raw content of the attachment as bytes. (This will automatically decode + any base64-encoded attachment data.) + + .. rubric:: Complex attachments + + An Anymail inbound attachment is actually just an :class:`AnymailInboundMessage` instance, + following the Python email package's usual recursive representation of MIME messages. + All :class:`AnymailInboundMessage` and :class:`email.message.Message` functionality + is available on attachment objects (though of course not all features are meaningful in all contexts). + + This can be helpful for, e.g., parsing email messages that are forwarded as attachments + to an inbound message. + + +Anymail loads all attachment content into memory as it processes each inbound +message. This may limit the size of attachments your app can handle, beyond +any attachment size limits imposed by your ESP. Depending on how your ESP transmits +attachments, you may also need to adjust Django's :setting:`DATA_UPLOAD_MAX_MEMORY_SIZE` +setting to successfully receive larger attachments. + + +.. _inbound-signal-receivers: + +Inbound signal receiver functions +--------------------------------- + +Your Anymail inbound signal receiver must be a function with this signature: + +.. function:: def my_handler(sender, event, esp_name, **kwargs): + + (You can name it anything you want.) + + :param class sender: The source of the event. (One of the + :mod:`anymail.webhook.*` View classes, but you + generally won't examine this parameter; it's + required by Django's signal mechanism.) + :param AnymailInboundEvent event: The normalized inbound event. + Almost anything you'd be interested in + will be in here---usually in the + :class:`~anymail.inbound.AnymailInboundMessage` + found in `event.message`. + :param str esp_name: e.g., "SendMail" or "Postmark". If you are working + with multiple ESPs, you can use this to distinguish + ESP-specific handling in your shared event processing. + :param \**kwargs: Required by Django's signal mechanism + (to support future extensions). + + :returns: nothing + :raises: any exceptions in your signal receiver will result + in a 400 HTTP error to the webhook. See discussion + below. + +.. TODO: this section is almost exactly duplicated from tracking. Combine somehow? + +If (any of) your signal receivers raise an exception, Anymail +will discontinue processing the current batch of events and return +an HTTP 400 error to the ESP. Most ESPs respond to this by re-sending +the event(s) later, a limited number of times. + +This is the desired behavior for transient problems (e.g., your +Django database being unavailable), but can cause confusion in other +error cases. You may want to catch some (or all) exceptions +in your signal receiver, log the problem for later follow up, +and allow Anymail to return the normal 200 success response +to your ESP. + +Some ESPs impose strict time limits on webhooks, and will consider +them failed if they don't respond within (say) five seconds. +And they may then retry sending these "failed" events, which could +cause duplicate processing in your code. +If your signal receiver code might be slow, you should instead +queue the event for later, asynchronous processing (e.g., using +something like `Celery`_). + +If your signal receiver function is defined within some other +function or instance method, you *must* use the `weak=False` +option when connecting it. Otherwise, it might seem to work at first, +but will unpredictably stop being called at some point---typically +on your production server, in a hard-to-debug way. See Django's +docs on :doc:`signals ` for more information. + +.. _Celery: http://www.celeryproject.org/ diff --git a/docs/index.rst b/docs/index.rst index 466410b..ecd7ec9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,6 +21,7 @@ Documentation quickstart installation sending/index + inbound esps/index tips/index troubleshooting diff --git a/docs/installation.rst b/docs/installation.rst index 485e932..fff5a18 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,28 +6,23 @@ Installation and configuration Installing Anymail ------------------ -It's easiest to install Anymail from PyPI using pip. +To use Anymail in your Django project: + +1. Install the django-anymail app. It's easiest to install from PyPI using pip: .. code-block:: console $ pip install django-anymail[sendgrid,sparkpost] -The `[sendgrid,sparkpost]` part of that command tells pip you also -want to install additional packages required for those ESPs. -You can give one or more comma-separated, lowercase ESP names. -(Most ESPs don't have additional requirements, so you can often -just skip this. Or change your mind later. Anymail will let you know -if there are any missing dependencies when you try to use it.) + The `[sendgrid,sparkpost]` part of that command tells pip you also + want to install additional packages required for those ESPs. + You can give one or more comma-separated, lowercase ESP names. + (Most ESPs don't have additional requirements, so you can often + just skip this. Or change your mind later. Anymail will let you know + if there are any missing dependencies when you try to use it.) - -.. _backend-configuration: - -Configuring Django's email backend ----------------------------------- - -To use Anymail for sending email, edit your Django project's :file:`settings.py`: - -1. Add :mod:`anymail` to your :setting:`INSTALLED_APPS` (anywhere in the list): +2. Edit your Django project's :file:`settings.py`, and add :mod:`anymail` + to your :setting:`INSTALLED_APPS` (anywhere in the list): .. code-block:: python @@ -37,8 +32,8 @@ To use Anymail for sending email, edit your Django project's :file:`settings.py` # ... ] -2. Add an :setting:`ANYMAIL` settings dict, substituting the appropriate settings for - your ESP: +3. Also in :file:`settings.py`, add an :setting:`ANYMAIL` settings dict, + substituting the appropriate settings for your ESP. E.g.: .. code-block:: python @@ -49,7 +44,20 @@ To use Anymail for sending email, edit your Django project's :file:`settings.py` The exact settings vary by ESP. See the :ref:`supported ESPs ` section for specifics. -3. Change your existing Django :setting:`EMAIL_BACKEND` to the Anymail backend +Then continue with either or both of the next two sections, depending +on which Anymail features you want to use. + + +.. _backend-configuration: + +Configuring Django's email backend +---------------------------------- + +To use Anymail for *sending* email from Django, make additional changes +in your project's :file:`settings.py`. (Skip this section if you are only +planning to *receive* email.) + +1. Change your existing Django :setting:`EMAIL_BACKEND` to the Anymail backend for your ESP. For example, to send using Mailgun by default: .. code-block:: python @@ -60,25 +68,27 @@ To use Anymail for sending email, edit your Django project's :file:`settings.py` use :ref:`multiple Anymail backends ` to send particular messages through different ESPs.) -Finally, if you don't already have a :setting:`DEFAULT_FROM_EMAIL` in your settings, -this is a good time to add one. (Django's default is "webmaster\@localhost", -which some ESPs will reject.) +2. If you don't already have a :setting:`DEFAULT_FROM_EMAIL` in your settings, + this is a good time to add one. (Django's default is "webmaster\@localhost", + which some ESPs will reject.) With the settings above, you are ready to send outgoing email through your ESP. -If you also want to enable status tracking, continue with the -optional settings below. Otherwise, skip ahead to :ref:`sending-email`. +If you also want to enable status tracking or inbound handling, continue with the +settings below. Otherwise, skip ahead to :ref:`sending-email`. .. _webhooks-configuration: -Configuring status tracking webhooks (optional) ------------------------------------------------ +Configuring tracking and inbound webhooks +----------------------------------------- -Anymail can optionally connect to your ESP's event webhooks to notify your app -of status like bounced and rejected emails, successful delivery, message opens -and clicks, and other tracking. +Anymail can optionally connect to your ESP's event webhooks to notify your app of: -If you aren't using Anymail's webhooks, skip this section. +* status tracking events for sent email, like bounced or rejected messages, + successful delivery, message opens and clicks, etc. +* inbound message events, if you are set up to receive email through your ESP + +Skip this section if you won't be using Anymail's webhooks. .. warning:: @@ -87,14 +97,13 @@ If you aren't using Anymail's webhooks, skip this section. that could expose your users' emails and other private information, or subject your app to malicious input data. - At a minimum, your site should **use SSL** (https), and you should + At a minimum, your site should **use https** and you should configure **webhook authorization** as described below. See :ref:`securing-webhooks` for additional information. -If you want to use Anymail's status tracking webhooks, follow the steps above -to :ref:`configure an Anymail backend `, and then: +If you want to use Anymail's inbound or tracking webhooks: 1. In your :file:`settings.py`, add :setting:`WEBHOOK_AUTHORIZATION ` @@ -148,31 +157,33 @@ to :ref:`configure an Anymail backend `, and then: 3. Enter the webhook URL(s) into your ESP's dashboard or control panel. In most cases, the URL will be: - :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/{esp}/tracking/` + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/{esp}/{type}/` * "https" (rather than http) is *strongly recommended* * *random:random* is the WEBHOOK_AUTHORIZATION string you created in step 1 * *yoursite.example.com* is your Django site * "anymail" is the url prefix (from step 2) * *esp* is the lowercase name of your ESP (e.g., "sendgrid" or "mailgun") - * "tracking" is used for Anymail's sent-mail event tracking webhooks + * *type* is either "tracking" for Anymail's sent-mail event tracking webhooks, + or "inbound" for receiving email Some ESPs support different webhooks for different tracking events. You can - usually enter the same Anymail webhook URL for all of them (or all that you - want to receive). But be sure to check the specific details for your ESP - under :ref:`supported-esps`. + usually enter the same Anymail *tracking* webhook URL for all of them (or all that you + want to receive)---but be sure to use the separate *inbound* URL for inbound webhooks. + And always check the specific details for your ESP under :ref:`supported-esps`. Also, some ESPs try to validate the webhook URL immediately when you enter it. If so, you'll need to deploy your Django project to your live server before you can complete this step. - Some WSGI servers may need additional settings to pass HTTP authorization headers - through to Django. For example, Apache with `mod_wsgi`_ requires - `WSGIPassAuthorization On`, else Anymail will complain about "missing or invalid - basic auth" when your webhook is called. +Some WSGI servers may need additional settings to pass HTTP authorization headers +through to Django. For example, Apache with `mod_wsgi`_ requires +`WSGIPassAuthorization On`, else Anymail will complain about "missing or invalid +basic auth" when your webhook is called. See :ref:`event-tracking` for information on creating signal handlers and the -status tracking events you can receive. +status tracking events you can receive. See :ref:`inbound` for information on +receiving inbound message events. .. _mod_wsgi: http://modwsgi.readthedocs.io/en/latest/configuration-directives/WSGIPassAuthorization.html diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 91ad1e7..3053166 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -17,5 +17,6 @@ Problems? We have some :ref:`troubleshooting` info that may help. Now that you've got Anymail working, you might be interested in: * :ref:`Sending email with Anymail ` +* :ref:`Receiving inbound email ` * :ref:`ESP-specific information ` * :ref:`All the docs ` diff --git a/tests/test_files/sample_email.txt b/tests/test_files/sample_email.txt index 5cd2614..1f92112 100644 --- a/tests/test_files/sample_email.txt +++ b/tests/test_files/sample_email.txt @@ -1,13 +1,16 @@ Received: by luna.mailgun.net with SMTP mgrt 8734663311733; Fri, 03 May 2013 18:26:27 +0000 -Content-Type: multipart/alternative; boundary="eb663d73ae0a4d6c9153cc0aec8b7520" +Content-Type: multipart/alternative; + boundary="eb663d73ae0a4d6c9153cc0aec8b7520" Mime-Version: 1.0 Subject: Test email From: Someone To: someoneelse@example.com Reply-To: reply.to@example.com Message-Id: <20130503182626.18666.16540@example.com> -List-Unsubscribe: +List-Unsubscribe: X-Mailgun-Sid: WyIwNzI5MCIsICJhbGljZUBleGFtcGxlLmNvbSIsICI2Il0= X-Mailgun-Variables: {"my_var_1": "Mailgun Variable #1", "my-var-2": "awesome"} Date: Fri, 03 May 2013 18:26:27 +0000 diff --git a/tests/test_inbound.py b/tests/test_inbound.py new file mode 100644 index 0000000..d2af891 --- /dev/null +++ b/tests/test_inbound.py @@ -0,0 +1,389 @@ +from __future__ import unicode_literals + +from base64 import b64encode +from textwrap import dedent + +from django.test import SimpleTestCase + +from anymail.inbound import AnymailInboundMessage +from .utils import SAMPLE_IMAGE_FILENAME, sample_image_content + +SAMPLE_IMAGE_CONTENT = sample_image_content() + + +class AnymailInboundMessageConstructionTests(SimpleTestCase): + def test_construct_params(self): + msg = AnymailInboundMessage.construct( + from_email="from@example.com", to="to@example.com", cc="cc@example.com", + subject="test subject") + self.assertEqual(msg['From'], "from@example.com") + self.assertEqual(msg['To'], "to@example.com") + self.assertEqual(msg['Cc'], "cc@example.com") + self.assertEqual(msg['Subject'], "test subject") + + self.assertEqual(msg.defects, []) # ensures email.message.Message.__init__ ran + self.assertIsNone(msg.envelope_recipient) # ensures AnymailInboundMessage.__init__ ran + + def test_construct_headers_from_mapping(self): + msg = AnymailInboundMessage.construct( + headers={'Reply-To': "reply@example.com", 'X-Test': "anything"}) + self.assertEqual(msg['reply-to'], "reply@example.com") # headers are case-insensitive + self.assertEqual(msg['X-TEST'], "anything") + + def test_construct_headers_from_pairs(self): + # allows multiple instances of a header + msg = AnymailInboundMessage.construct( + headers=[['Reply-To', "reply@example.com"], + ['Received', "by 10.1.1.4 with SMTP id q4csp; Sun, 22 Oct 2017 00:23:22 -0700 (PDT)"], + ['Received', "from mail.example.com (mail.example.com. [10.10.1.9])" + " by mx.example.com with SMTPS id 93s8iok for ;" + " Sun, 22 Oct 2017 00:23:21 -0700 (PDT)"], + ]) + self.assertEqual(msg['Reply-To'], "reply@example.com") + self.assertEqual(msg.get_all('Received'), [ + "by 10.1.1.4 with SMTP id q4csp; Sun, 22 Oct 2017 00:23:22 -0700 (PDT)", + "from mail.example.com (mail.example.com. [10.10.1.9])" + " by mx.example.com with SMTPS id 93s8iok for ;" + " Sun, 22 Oct 2017 00:23:21 -0700 (PDT)"]) + + def test_construct_headers_from_raw(self): + # (note header "folding" in second Received header) + msg = AnymailInboundMessage.construct( + raw_headers=dedent("""\ + Reply-To: reply@example.com + Subject: raw subject + Content-Type: x-custom/custom + Received: by 10.1.1.4 with SMTP id q4csp; Sun, 22 Oct 2017 00:23:22 -0700 (PDT) + Received: from mail.example.com (mail.example.com. [10.10.1.9]) + by mx.example.com with SMTPS id 93s8iok for ; + Sun, 22 Oct 2017 00:23:21 -0700 (PDT) + """), + subject="Explicit subject overrides raw") + self.assertEqual(msg['Reply-To'], "reply@example.com") + self.assertEqual(msg.get_all('Received'), [ + "by 10.1.1.4 with SMTP id q4csp; Sun, 22 Oct 2017 00:23:22 -0700 (PDT)", + "from mail.example.com (mail.example.com. [10.10.1.9])" # unfolding should have stripped newlines + " by mx.example.com with SMTPS id 93s8iok for ;" + " Sun, 22 Oct 2017 00:23:21 -0700 (PDT)"]) + self.assertEqual(msg.get_all('Subject'), ["Explicit subject overrides raw"]) + self.assertEqual(msg.get_all('Content-Type'), ["multipart/mixed"]) # Content-Type in raw header ignored + + def test_construct_bodies(self): + # this verifies we construct the expected MIME structure; + # see the `text` and `html` props (in the ConveniencePropTests below) + # for an easier way to get to these fields (that works however constructed) + msg = AnymailInboundMessage.construct(text="Plaintext body", html="HTML body") + self.assertEqual(msg['Content-Type'], "multipart/mixed") + self.assertEqual(len(msg.get_payload()), 1) + + related = msg.get_payload(0) + self.assertEqual(related['Content-Type'], "multipart/related") + self.assertEqual(len(related.get_payload()), 1) + + alternative = related.get_payload(0) + self.assertEqual(alternative['Content-Type'], "multipart/alternative") + self.assertEqual(len(alternative.get_payload()), 2) + + plaintext = alternative.get_payload(0) + self.assertEqual(plaintext['Content-Type'], 'text/plain; charset="utf-8"') + self.assertEqual(plaintext.get_content_text(), "Plaintext body") + + html = alternative.get_payload(1) + self.assertEqual(html['Content-Type'], 'text/html; charset="utf-8"') + self.assertEqual(html.get_content_text(), "HTML body") + + def test_construct_attachments(self): + att1 = AnymailInboundMessage.construct_attachment( + 'text/csv', "One,Two\n1,2".encode('iso-8859-1'), charset="iso-8859-1", filename="test.csv") + + att2 = AnymailInboundMessage.construct_attachment( + 'image/png', SAMPLE_IMAGE_CONTENT, filename=SAMPLE_IMAGE_FILENAME, content_id="abc123") + + msg = AnymailInboundMessage.construct(attachments=[att1, att2]) + self.assertEqual(msg['Content-Type'], "multipart/mixed") + self.assertEqual(len(msg.get_payload()), 2) # bodies (related), att1 + + att1_part = msg.get_payload(1) + self.assertEqual(att1_part['Content-Type'], 'text/csv; name="test.csv"; charset="iso-8859-1"') + self.assertEqual(att1_part['Content-Disposition'], 'attachment; filename="test.csv"') + self.assertNotIn('Content-ID', att1_part) + self.assertEqual(att1_part.get_content_text(), "One,Two\n1,2") + + related = msg.get_payload(0) + self.assertEqual(len(related.get_payload()), 2) # alternatives (with no bodies in this test); att2 + att2_part = related.get_payload(1) + self.assertEqual(att2_part['Content-Type'], 'image/png; name="sample_image.png"') + self.assertEqual(att2_part['Content-Disposition'], 'inline; filename="sample_image.png"') + self.assertEqual(att2_part['Content-ID'], '') + self.assertEqual(att2_part.get_content_bytes(), SAMPLE_IMAGE_CONTENT) + + def test_construct_attachments_from_uploaded_files(self): + from django.core.files.uploadedfile import SimpleUploadedFile + file = SimpleUploadedFile(SAMPLE_IMAGE_FILENAME, SAMPLE_IMAGE_CONTENT, 'image/png') + att = AnymailInboundMessage.construct_attachment_from_uploaded_file(file, content_id="abc123") + self.assertEqual(att['Content-Type'], 'image/png; name="sample_image.png"') + self.assertEqual(att['Content-Disposition'], 'inline; filename="sample_image.png"') + self.assertEqual(att['Content-ID'], '') + self.assertEqual(att.get_content_bytes(), SAMPLE_IMAGE_CONTENT) + + def test_construct_attachments_from_base64_data(self): + # This is a fairly common way for ESPs to provide attachment content to webhooks + from base64 import b64encode + content = b64encode(SAMPLE_IMAGE_CONTENT) + att = AnymailInboundMessage.construct_attachment(content_type="image/png", content=content, base64=True) + self.assertEqual(att.get_content_bytes(), SAMPLE_IMAGE_CONTENT) + + def test_parse_raw_mime(self): + # (we're not trying to exhaustively test email.parser MIME handling here; + # just that AnymailInboundMessage.parse_raw_mime calls it correctly) + raw = dedent("""\ + Content-Type: text/plain + Subject: This is a test message + + This is a test body. + """) + msg = AnymailInboundMessage.parse_raw_mime(raw) + self.assertEqual(msg['Subject'], "This is a test message") + self.assertEqual(msg.get_content_text(), "This is a test body.\n") + self.assertEqual(msg.defects, []) + + # (see test_attachment_as_uploaded_file below for parsing basic attachment from raw mime) + + +class AnymailInboundMessageConveniencePropTests(SimpleTestCase): + # AnymailInboundMessage defines several properties to simplify reading + # commonly-used items in an email.message.Message + + def test_address_props(self): + msg = AnymailInboundMessage.construct( + from_email='"Sender, Inc." ', + to='First To , to2@example.com', + cc='First Cc , cc2@example.com', + ) + self.assertEqual(str(msg.from_email), '"Sender, Inc." ') + self.assertEqual(msg.from_email.addr_spec, 'sender@example.com') + self.assertEqual(msg.from_email.display_name, 'Sender, Inc.') + self.assertEqual(msg.from_email.username, 'sender') + self.assertEqual(msg.from_email.domain, 'example.com') + + self.assertEqual(len(msg.to), 2) + self.assertEqual(msg.to[0].addr_spec, 'to1@example.com') + self.assertEqual(msg.to[0].display_name, 'First To') + self.assertEqual(msg.to[1].addr_spec, 'to2@example.com') + self.assertEqual(msg.to[1].display_name, '') + + self.assertEqual(len(msg.cc), 2) + self.assertEqual(msg.cc[0].address, 'First Cc ') + self.assertEqual(msg.cc[1].address, 'cc2@example.com') + + # Default None/empty lists + msg = AnymailInboundMessage() + self.assertIsNone(msg.from_email) + self.assertEqual(msg.to, []) + self.assertEqual(msg.cc, []) + + def test_body_props(self): + msg = AnymailInboundMessage.construct(text="Test plaintext", html="Test HTML") + self.assertEqual(msg.text, "Test plaintext") + self.assertEqual(msg.html, "Test HTML") + + # Make sure attachments don't confuse it + att_text = AnymailInboundMessage.construct_attachment('text/plain', "text attachment") + att_html = AnymailInboundMessage.construct_attachment('text/html', "html attachment") + + msg = AnymailInboundMessage.construct(text="Test plaintext", attachments=[att_text, att_html]) + self.assertEqual(msg.text, "Test plaintext") + self.assertIsNone(msg.html) # no html body (the html attachment doesn't count) + + msg = AnymailInboundMessage.construct(html="Test HTML", attachments=[att_text, att_html]) + self.assertIsNone(msg.text) # no plaintext body (the text attachment doesn't count) + self.assertEqual(msg.html, "Test HTML") + + # Default None + msg = AnymailInboundMessage() + self.assertIsNone(msg.text) + self.assertIsNone(msg.html) + + def test_date_props(self): + msg = AnymailInboundMessage.construct(headers={ + 'Date': "Mon, 23 Oct 2017 17:50:55 -0700" + }) + self.assertEqual(msg.date.isoformat(), "2017-10-23T17:50:55-07:00") + + # Default None + self.assertIsNone(AnymailInboundMessage().date) + + def test_attachments_prop(self): + att = AnymailInboundMessage.construct_attachment( + 'image/png', SAMPLE_IMAGE_CONTENT, filename=SAMPLE_IMAGE_FILENAME) + + msg = AnymailInboundMessage.construct(attachments=[att]) + self.assertEqual(msg.attachments, [att]) + + # Default empty list + self.assertEqual(AnymailInboundMessage().attachments, []) + + def test_inline_attachments_prop(self): + att = AnymailInboundMessage.construct_attachment( + 'image/png', SAMPLE_IMAGE_CONTENT, filename=SAMPLE_IMAGE_FILENAME, content_id="abc123") + + msg = AnymailInboundMessage.construct(attachments=[att]) + self.assertEqual(msg.inline_attachments, {'abc123': att}) + + # Default empty dict + self.assertEqual(AnymailInboundMessage().inline_attachments, {}) + + def test_attachment_as_uploaded_file(self): + raw = dedent("""\ + MIME-Version: 1.0 + Subject: Attachment test + Content-Type: multipart/mixed; boundary="this_is_a_boundary" + + --this_is_a_boundary + Content-Type: text/plain; charset="UTF-8" + + The test sample image is attached below. + + --this_is_a_boundary + Content-Type: image/png; name="sample_image.png" + Content-Disposition: attachment; filename="sample_image.png" + Content-Transfer-Encoding: base64 + + iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz + AAALEgAACxIB0t1+/AAAABR0RVh0Q3JlYXRpb24gVGltZQAzLzEvMTNoZNRjAAAAHHRFWHRTb2Z0 + d2FyZQBBZG9iZSBGaXJld29ya3MgQ1M1cbXjNgAAAZ1JREFUWIXtl7FKA0EQhr+TgIFgo5BXyBUp + fIGksLawUNAXWFFfwCJgBAtfIJFMLXgQn8BSwdpCiPcKAdOIoI2x2Dmyd7kYwXhp9odluX/uZv6d + nZu7DXowxiKZi0IAUHKCvxcsoAIEpST4IawVGb0Hb0BlpcigefACvAAvwAsoTTGGlwwzBAyivLUP + EZrOM10AhGOH2wWugVVlHoAdhJHrPC8DNR0JGsAAQ9mxNzBOMNjS4Qrq69U5EKmf12ywWVsQI4QI + IbCn3Gnmnk7uk1bokfooI7QRDlQIGCdzPwiYh0idtXNs2zq3UqwVEiDcu/R0DVjUnFpItuPSscfA + FXCGSfEAdZ2fVeQ68OjYWwi3ycVvMhABGwgfKXZScHeZ+4c6VzN8FbuYukvOykCs+z8PJ0xqIXYE + d4ALoKlVH2IIgUHWwd/6gNAFPjPcCPvKNTDcYAj1lXzKc7GIRrSZI6yJzcQ+dtV9bD+IkHThBj34 + 4j9/yYxupaQbXPJLNqsGFgeZ6qwpLP1b4AV4AV5AoKfjpR5OwR6VKwULCAC+AQV4W9Ps4uZQAAAA + AElFTkSuQmCC + --this_is_a_boundary-- + """) + + msg = AnymailInboundMessage.parse_raw_mime(raw) + attachment = msg.attachments[0] + attachment_file = attachment.as_uploaded_file() + + self.assertEqual(attachment_file.name, "sample_image.png") + self.assertEqual(attachment_file.content_type, "image/png") + self.assertEqual(attachment_file.read(), SAMPLE_IMAGE_CONTENT) + + def test_attachment_as_uploaded_file_security(self): + # Raw attachment filenames can be malicious; we want to make sure that + # our Django file converter sanitizes them (as much as any uploaded filename) + raw = dedent("""\ + MIME-Version: 1.0 + Subject: Attachment test + Content-Type: multipart/mixed; boundary="this_is_a_boundary" + + --this_is_a_boundary + Content-Type: text/plain; charset="UTF-8" + + The malicious attachment filenames below need to get sanitized + + --this_is_a_boundary + Content-Type: text/plain; name="report.txt" + Content-Disposition: attachment; filename="/etc/passwd" + + # (not that overwriting /etc/passwd is actually a thing + # anymore, but you get the point) + --this_is_a_boundary + Content-Type: text/html + Content-Disposition: attachment; filename="../static/index.html" + + Hey, did I overwrite your site? + --this_is_a_boundary-- + """) + msg = AnymailInboundMessage.parse_raw_mime(raw) + attachments = msg.attachments + + self.assertEqual(attachments[0].get_filename(), "/etc/passwd") # you wouldn't want to actually write here + self.assertEqual(attachments[0].as_uploaded_file().name, "passwd") # path removed - good! + + self.assertEqual(attachments[1].get_filename(), "../static/index.html") + self.assertEqual(attachments[1].as_uploaded_file().name, "index.html") # ditto for relative paths + + +class AnymailInboundMessageAttachedMessageTests(SimpleTestCase): + # message/rfc822 attachments should get parsed recursively + + original_raw_message = dedent("""\ + MIME-Version: 1.0 + From: sender@example.com + Subject: Original message + Return-Path: bounces@inbound.example.com + Content-Type: multipart/related; boundary="boundary-orig" + + --boundary-orig + Content-Type: text/html; charset="UTF-8" + + Here is your message! + + --boundary-orig + Content-Type: image/png; name="sample_image.png" + Content-Disposition: inline + Content-ID: + Content-Transfer-Encoding: base64 + + {image_content_base64} + --boundary-orig-- + """).format(image_content_base64=b64encode(SAMPLE_IMAGE_CONTENT).decode('ascii')) + + def test_parse_rfc822_attachment_from_raw_mime(self): + # message/rfc822 attachments should be parsed recursively + raw = dedent("""\ + MIME-Version: 1.0 + From: mailer-demon@example.org + Subject: Undeliverable + To: bounces@inbound.example.com + Content-Type: multipart/mixed; boundary="boundary-bounce" + + --boundary-bounce + Content-Type: text/plain + + Your message was undeliverable due to carrier pigeon strike. + The original message is attached. + + --boundary-bounce + Content-Type: message/rfc822 + Content-Disposition: attachment + + {original_raw_message} + --boundary-bounce-- + """).format(original_raw_message=self.original_raw_message) + + msg = AnymailInboundMessage.parse_raw_mime(raw) + self.assertIsInstance(msg, AnymailInboundMessage) + + att = msg.get_payload(1) + self.assertIsInstance(att, AnymailInboundMessage) + self.assertEqual(att.get_content_type(), "message/rfc822") + self.assertTrue(att.is_attachment()) + + orig_msg = att.get_payload(0) + self.assertIsInstance(orig_msg, AnymailInboundMessage) + self.assertEqual(orig_msg['Subject'], "Original message") + self.assertEqual(orig_msg.get_content_type(), "multipart/related") + self.assertEqual(att.get_content_text(), self.original_raw_message) + + orig_inline_att = orig_msg.get_payload(1) + self.assertEqual(orig_inline_att.get_content_type(), "image/png") + self.assertTrue(orig_inline_att.is_inline_attachment()) + self.assertEqual(orig_inline_att.get_payload(decode=True), SAMPLE_IMAGE_CONTENT) + + def test_construct_rfc822_attachment_from_data(self): + # constructed message/rfc822 attachment should end up as parsed message + # (same as if attachment was parsed from raw mime, as in previous test) + att = AnymailInboundMessage.construct_attachment('message/rfc822', self.original_raw_message) + self.assertIsInstance(att, AnymailInboundMessage) + self.assertEqual(att.get_content_type(), "message/rfc822") + self.assertTrue(att.is_attachment()) + self.assertEqual(att.get_content_text(), self.original_raw_message) + + orig_msg = att.get_payload(0) + self.assertIsInstance(orig_msg, AnymailInboundMessage) + self.assertEqual(orig_msg['Subject'], "Original message") + self.assertEqual(orig_msg.get_content_type(), "multipart/related") diff --git a/tests/test_mailgun_backend.py b/tests/test_mailgun_backend.py index c53bb7b..5973f69 100644 --- a/tests/test_mailgun_backend.py +++ b/tests/test_mailgun_backend.py @@ -159,7 +159,7 @@ class MailgunBackendStandardEmailTests(MailgunBackendMockAPITestCase): # Email messages can get a bit changed with respect to whitespace characters # in headers, without breaking the message, so we tolerate that: self.assertEqual(attachments[3][0], None) - self.assertEqualIgnoringWhitespace( + self.assertEqualIgnoringHeaderFolding( attachments[3][1], b'Content-Type: message/rfc822\nMIME-Version: 1.0\n\n' + forwarded_email_content) self.assertEqual(attachments[3][2], 'message/rfc822') diff --git a/tests/test_mailgun_inbound.py b/tests/test_mailgun_inbound.py new file mode 100644 index 0000000..192e6ee --- /dev/null +++ b/tests/test_mailgun_inbound.py @@ -0,0 +1,176 @@ +import json +from datetime import datetime +from textwrap import dedent + +import six +from django.test import override_settings +from django.utils.timezone import utc +from mock import ANY + +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.mailgun import MailgunInboundWebhookView + +from .test_mailgun_webhooks import TEST_API_KEY, mailgun_sign, querydict_to_postdict +from .utils import sample_image_content, sample_email_content +from .webhook_cases import WebhookTestCase + + +@override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY) +class MailgunInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + raw_event = mailgun_sign({ + 'token': '06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0', + 'timestamp': '1461261330', + 'recipient': 'test@inbound.example.com', + 'sender': 'envelope-from@example.org', + 'message-headers': json.dumps([ + ["X-Mailgun-Spam-Rules", "DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, ..."], + ["X-Mailgun-Dkim-Check-Result", "Pass"], + ["X-Mailgun-Spf", "Pass"], + ["X-Mailgun-Sscore", "1.7"], + ["X-Mailgun-Sflag", "No"], + ["X-Mailgun-Incoming", "Yes"], + ["X-Envelope-From", ""], + ["Received", "from mail.example.org by mxa.mailgun.org ..."], + ["Received", "by mail.example.org for ..."], + ["Dkim-Signature", "v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.org; ..."], + ["Mime-Version", "1.0"], + ["Received", "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)"], + ["From", "\"Displayed From\" "], + ["Date", "Wed, 11 Oct 2017 18:31:04 -0700"], + ["Message-Id", ""], + ["Subject", "Test subject"], + ["To", "\"Test Inbound\" , other@example.com"], + ["Cc", "cc@example.com"], + ["Content-Type", "multipart/mixed; boundary=\"089e0825ccf874a0bb055b4f7e23\""], + ]), + 'body-plain': 'Test body plain', + 'body-html': '
Test body html
', + 'stripped-html': 'stripped html body', + 'stripped-text': 'stripped plaintext body', + }) + response = self.client.post('/anymail/mailgun/inbound/', data=raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=MailgunInboundWebhookView, + event=ANY, esp_name='Mailgun') + # AnymailInboundEvent + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertEqual(event.timestamp, datetime(2016, 4, 21, 17, 55, 30, tzinfo=utc)) + self.assertEqual(event.event_id, "06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0") + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(querydict_to_postdict(event.esp_event.POST), raw_event) + + # AnymailInboundMessage - convenience properties + message = event.message + + self.assertEqual(message.from_email.display_name, 'Displayed From') + self.assertEqual(message.from_email.addr_spec, 'from+test@example.org') + self.assertEqual([str(e) for e in message.to], + ['Test Inbound ', 'other@example.com']) + self.assertEqual([str(e) for e in message.cc], + ['cc@example.com']) + self.assertEqual(message.subject, 'Test subject') + self.assertEqual(message.date.isoformat(" "), "2017-10-11 18:31:04-07:00") + self.assertEqual(message.text, 'Test body plain') + self.assertEqual(message.html, '
Test body html
') + + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertEqual(message.stripped_text, 'stripped plaintext body') + self.assertEqual(message.stripped_html, 'stripped html body') + self.assertIs(message.spam_detected, False) + self.assertEqual(message.spam_score, 1.7) + + # AnymailInboundMessage - other headers + self.assertEqual(message['Message-ID'], "") + self.assertEqual(message.get_all('Received'), [ + "from mail.example.org by mxa.mailgun.org ...", + "by mail.example.org for ...", + "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)", + ]) + + def test_attachments(self): + att1 = six.BytesIO('test attachment'.encode('utf-8')) + att1.name = 'test.txt' + image_content = sample_image_content() + att2 = six.BytesIO(image_content) + att2.name = 'image.png' + email_content = sample_email_content() + att3 = six.BytesIO(email_content) + att3.content_type = 'message/rfc822; charset="us-ascii"' + raw_event = mailgun_sign({ + 'message-headers': '[]', + 'attachment-count': '3', + 'content-id-map': """{"": "attachment-2"}""", + 'attachment-1': att1, + 'attachment-2': att2, # inline + 'attachment-3': att3, + }) + + response = self.client.post('/anymail/mailgun/inbound/', data=raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=MailgunInboundWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + message = event.message + attachments = message.attachments # AnymailInboundMessage convenience accessor + self.assertEqual(len(attachments), 2) + self.assertEqual(attachments[0].get_filename(), 'test.txt') + self.assertEqual(attachments[0].get_content_type(), 'text/plain') + self.assertEqual(attachments[0].get_content_text(), u'test attachment') + self.assertEqual(attachments[1].get_content_type(), 'message/rfc822') + self.assertEqualIgnoringHeaderFolding(attachments[1].get_content_bytes(), email_content) + + inlines = message.inline_attachments + self.assertEqual(len(inlines), 1) + inline = inlines['abc123'] + self.assertEqual(inline.get_filename(), 'image.png') + self.assertEqual(inline.get_content_type(), 'image/png') + self.assertEqual(inline.get_content_bytes(), image_content) + + def test_inbound_mime(self): + # Mailgun provides the full, raw MIME message if the webhook url ends in 'mime' + raw_event = mailgun_sign({ + 'token': '06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0', + 'timestamp': '1461261330', + 'recipient': 'test@inbound.example.com', + 'sender': 'envelope-from@example.org', + 'body-mime': dedent("""\ + From: A tester + Date: Thu, 12 Oct 2017 18:03:30 -0700 + Message-ID: + Subject: Raw MIME test + To: test@inbound.example.com + MIME-Version: 1.0 + Content-Type: multipart/alternative; boundary="94eb2c05e174adb140055b6339c5" + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/plain; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + + It's a body=E2=80=A6 + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/html; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + +
It's a body=E2=80=A6
+ + --94eb2c05e174adb140055b6339c5-- + """), + }) + + response = self.client.post('/anymail/mailgun/inbound_mime/', data=raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=MailgunInboundWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + message = event.message + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertEqual(message.subject, 'Raw MIME test') + self.assertEqual(message.text, u"It's a body\N{HORIZONTAL ELLIPSIS}\n") + self.assertEqual(message.html, u"""
It's a body\N{HORIZONTAL ELLIPSIS}
\n""") diff --git a/tests/test_mailjet_inbound.py b/tests/test_mailjet_inbound.py new file mode 100644 index 0000000..3036aea --- /dev/null +++ b/tests/test_mailjet_inbound.py @@ -0,0 +1,170 @@ +import json +from base64 import b64encode + +from mock import ANY + +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.mailjet import MailjetInboundWebhookView + +from .utils import sample_image_content, sample_email_content +from .webhook_cases import WebhookTestCase + + +class MailjetInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + raw_event = { + "Sender": "envelope-from@example.org", + "Recipient": "test@inbound.example.com", + "Date": "20171012T013104", # this is just the Date header from the sender, parsed to UTC + "From": '"Displayed From" ', + "Subject": "Test subject", + "Headers": { + "Return-Path": [""], + "Received": [ + "from mail.example.org by parse.mailjet.com ..." + "by mail.example.org for ...", + "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)", + ], + "MIME-Version": ["1.0"], + "From": '"Displayed From" ', + "Date": "Wed, 11 Oct 2017 18:31:04 -0700", + "Message-ID": "", + "Subject": "Test subject", + "To": "Test Inbound , other@example.com", + "Cc": "cc@example.com", + "Reply-To": "from+test@milter.example.org", + "Content-Type": ["multipart/alternative; boundary=\"boundary0\""], + }, + "Parts": [{ + "Headers": { + "Content-Type": ['text/plain; charset="UTF-8"'] + }, + "ContentRef": "Text-part" + }, { + "Headers": { + "Content-Type": ['text/html; charset="UTF-8"'], + "Content-Transfer-Encoding": ["quoted-printable"] + }, + "ContentRef": "Html-part" + }], + "Text-part": "Test body plain", + "Html-part": "
Test body html
", + "SpamAssassinScore": "1.7" + } + + response = self.client.post('/anymail/mailjet/inbound/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=MailjetInboundWebhookView, + event=ANY, esp_name='Mailjet') + # AnymailInboundEvent + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertIsNone(event.timestamp) # Mailjet doesn't provide inbound event timestamp + self.assertIsNone(event.event_id) # Mailjet doesn't provide inbound event id + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_event) + + # AnymailInboundMessage - convenience properties + message = event.message + + self.assertEqual(message.from_email.display_name, 'Displayed From') + self.assertEqual(message.from_email.addr_spec, 'from+test@example.org') + self.assertEqual([str(e) for e in message.to], + ['Test Inbound ', 'other@example.com']) + self.assertEqual([str(e) for e in message.cc], + ['cc@example.com']) + self.assertEqual(message.subject, 'Test subject') + self.assertEqual(message.date.isoformat(" "), "2017-10-11 18:31:04-07:00") + self.assertEqual(message.text, 'Test body plain') + self.assertEqual(message.html, '
Test body html
') + + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertIsNone(message.stripped_text) # Mailjet doesn't provide stripped plaintext body + self.assertIsNone(message.stripped_html) # Mailjet doesn't provide stripped html + self.assertIsNone(message.spam_detected) # Mailjet doesn't provide spam boolean + self.assertEqual(message.spam_score, 1.7) + + # AnymailInboundMessage - other headers + self.assertEqual(message['Message-ID'], "") + self.assertEqual(message['Reply-To'], "from+test@milter.example.org") + self.assertEqual(message.get_all('Received'), [ + "from mail.example.org by parse.mailjet.com ..." + "by mail.example.org for ...", + "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)", + ]) + + def test_attachments(self): + image_content = sample_image_content() + email_content = sample_email_content() + raw_event = { + "Headers": { + "MIME-Version": ["1.0"], + "Content-Type": ["multipart/mixed; boundary=\"boundary0\""], + }, + "Parts": [{ + "Headers": {"Content-Type": ['multipart/related; boundary="boundary1"']} + }, { + "Headers": {"Content-Type": ['multipart/alternative; boundary="boundary2"']} + }, { + "Headers": {"Content-Type": ['text/plain; charset="UTF-8"']}, + "ContentRef": "Text-part" + }, { + "Headers": { + "Content-Type": ['text/html; charset="UTF-8"'], + "Content-Transfer-Encoding": ["quoted-printable"] + }, + "ContentRef": "Html-part" + }, { + "Headers": { + "Content-Type": ['text/plain'], + "Content-Disposition": ['attachment; filename="test.txt"'], + "Content-Transfer-Encoding": ["quoted-printable"], + }, + "ContentRef": "Attachment1" + }, { + "Headers": { + "Content-Type": ['image/png; name="image.png"'], + "Content-Disposition": ['inline; filename="image.png"'], + "Content-Transfer-Encoding": ["base64"], + "Content-ID": [""], + }, + "ContentRef": "InlineAttachment1" + }, { + "Headers": { + "Content-Type": ['message/rfc822; charset="US-ASCII"'], + "Content-Disposition": ['attachment'], + }, + "ContentRef": "Attachment2" + }], + "Text-part": "Test body plain", + "Html-part": "
Test body html
", + "InlineAttachment1": b64encode(image_content).decode('ascii'), + "Attachment1": b64encode('test attachment'.encode('utf-8')).decode('ascii'), + "Attachment2": b64encode(email_content).decode('ascii'), + } + + response = self.client.post('/anymail/mailjet/inbound/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=MailjetInboundWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + message = event.message + attachments = message.attachments # AnymailInboundMessage convenience accessor + self.assertEqual(len(attachments), 2) + self.assertEqual(attachments[0].get_filename(), 'test.txt') + self.assertEqual(attachments[0].get_content_type(), 'text/plain') + self.assertEqual(attachments[0].get_content_text(), u'test attachment') + self.assertEqual(attachments[1].get_content_type(), 'message/rfc822') + self.assertEqualIgnoringHeaderFolding(attachments[1].get_content_bytes(), email_content) + + inlines = message.inline_attachments + self.assertEqual(len(inlines), 1) + inline = inlines['abc123'] + self.assertEqual(inline.get_filename(), 'image.png') + self.assertEqual(inline.get_content_type(), 'image/png') + self.assertEqual(inline.get_content_bytes(), image_content) diff --git a/tests/test_mandrill_inbound.py b/tests/test_mandrill_inbound.py new file mode 100644 index 0000000..d554e96 --- /dev/null +++ b/tests/test_mandrill_inbound.py @@ -0,0 +1,88 @@ +from textwrap import dedent + +from django.test import override_settings +from mock import ANY + +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.mandrill import MandrillCombinedWebhookView + +from .test_mandrill_webhooks import TEST_WEBHOOK_KEY, mandrill_args +from .webhook_cases import WebhookTestCase + + +@override_settings(ANYMAIL_MANDRILL_WEBHOOK_KEY=TEST_WEBHOOK_KEY) +class MandrillInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + raw_event = { + "event": "inbound", + "ts": 1507856722, + "msg": { + "raw_msg": dedent("""\ + From: A tester + Date: Thu, 12 Oct 2017 18:03:30 -0700 + Message-ID: + Subject: Test subject + To: "Test, Inbound" , other@example.com + MIME-Version: 1.0 + Content-Type: multipart/alternative; boundary="94eb2c05e174adb140055b6339c5" + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/plain; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + + It's a body=E2=80=A6 + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/html; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + +
It's a body=E2=80=A6
+ + --94eb2c05e174adb140055b6339c5-- + """), + "email": "delivered-to@example.com", + "sender": None, # Mandrill populates "sender" only for outbound message events + "spam_report": { + "score": 1.7, + }, + # Anymail ignores Mandrill's other inbound event fields + # (which are all redundant with raw_msg) + }, + } + + response = self.client.post(**mandrill_args(events=[raw_event], path='/anymail/mandrill/')) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=MandrillCombinedWebhookView, + event=ANY, esp_name='Mandrill') + self.assertEqual(self.tracking_handler.call_count, 0) # Inbound should not dispatch tracking signal + + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, "inbound") + self.assertEqual(event.timestamp.isoformat(), "2017-10-13T01:05:22+00:00") + self.assertIsNone(event.event_id) # Mandrill doesn't provide inbound event id + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_event) + + message = event.message + self.assertEqual(message.from_email.display_name, 'A tester') + self.assertEqual(message.from_email.addr_spec, 'test@example.org') + self.assertEqual(len(message.to), 2) + self.assertEqual(message.to[0].display_name, 'Test, Inbound') + self.assertEqual(message.to[0].addr_spec, 'test@inbound.example.com') + self.assertEqual(message.to[1].addr_spec, 'other@example.com') + self.assertEqual(message.subject, 'Test subject') + self.assertEqual(message.date.isoformat(" "), "2017-10-12 18:03:30-07:00") + self.assertEqual(message.text, u"It's a body\N{HORIZONTAL ELLIPSIS}\n") + self.assertEqual(message.html, u"""
It's a body\N{HORIZONTAL ELLIPSIS}
\n""") + + self.assertIsNone(message.envelope_sender) # Mandrill doesn't provide sender + self.assertEqual(message.envelope_recipient, 'delivered-to@example.com') + self.assertIsNone(message.stripped_text) # Mandrill doesn't provide stripped plaintext body + self.assertIsNone(message.stripped_html) # Mandrill doesn't provide stripped html + self.assertIsNone(message.spam_detected) # Mandrill doesn't provide spam boolean + self.assertEqual(message.spam_score, 1.7) + + # Anymail will also parse attachments (if any) from the raw mime. + # We don't bother testing that here; see test_inbound for examples. diff --git a/tests/test_mandrill_webhooks.py b/tests/test_mandrill_webhooks.py index b413fe3..1fc19f3 100644 --- a/tests/test_mandrill_webhooks.py +++ b/tests/test_mandrill_webhooks.py @@ -1,6 +1,5 @@ import json from datetime import datetime -# noinspection PyUnresolvedReferences from six.moves.urllib.parse import urljoin import hashlib @@ -12,7 +11,7 @@ from django.utils.timezone import utc from mock import ANY from anymail.signals import AnymailTrackingEvent -from anymail.webhooks.mandrill import MandrillTrackingWebhookView +from anymail.webhooks.mandrill import MandrillCombinedWebhookView, MandrillTrackingWebhookView from .webhook_cases import WebhookTestCase, WebhookBasicAuthTestsMixin @@ -21,7 +20,7 @@ TEST_WEBHOOK_KEY = 'TEST_WEBHOOK_KEY' def mandrill_args(events=None, host="http://testserver/", # Django test-client default - path='/anymail/mandrill/tracking/', # Anymail urlconf default + path='/anymail/mandrill/', # Anymail urlconf default auth="username:password", # WebhookTestCase default key=TEST_WEBHOOK_KEY): """Returns TestClient.post kwargs for Mandrill webhook call with events @@ -30,7 +29,7 @@ def mandrill_args(events=None, """ if events is None: events = [] - test_client_path = urljoin(host, path) # https://testserver/anymail/mandrill/tracking/ + test_client_path = urljoin(host, path) # https://testserver/anymail/mandrill/ if auth: # we can get away with this simplification in these controlled tests, # but don't ever construct urls like this in production code -- it's not safe! @@ -52,14 +51,14 @@ def mandrill_args(events=None, class MandrillWebhookSettingsTestCase(WebhookTestCase): def test_requires_webhook_key(self): with self.assertRaisesRegex(ImproperlyConfigured, r'MANDRILL_WEBHOOK_KEY'): - self.client.post('/anymail/mandrill/tracking/', + self.client.post('/anymail/mandrill/', data={'mandrill_events': '[]'}) def test_head_does_not_require_webhook_key(self): # Mandrill issues an unsigned HEAD request to verify the wehbook url. # Only *after* that succeeds will Mandrill will tell you the webhook key. # So make sure that HEAD request will go through without any key set: - response = self.client.head('/anymail/mandrill/tracking/') + response = self.client.head('/anymail/mandrill/') self.assertEqual(response.status_code, 200) @@ -79,7 +78,7 @@ class MandrillWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixi self.assertEqual(response.status_code, 200) def test_verifies_missing_signature(self): - response = self.client.post('/anymail/mandrill/tracking/', + response = self.client.post('/anymail/mandrill/', data={'mandrill_events': '[{"event":"send"}]'}) self.assertEqual(response.status_code, 400) @@ -99,7 +98,7 @@ class MandrillWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixi @override_settings( ALLOWED_HOSTS=['127.0.0.1', '.example.com'], ANYMAIL={ - "MANDRILL_WEBHOOK_URL": "https://abcde:12345@example.com/anymail/mandrill/tracking/", + "MANDRILL_WEBHOOK_URL": "https://abcde:12345@example.com/anymail/mandrill/", "WEBHOOK_AUTHORIZATION": "abcde:12345", }) def test_webhook_url_setting(self): @@ -133,6 +132,7 @@ class MandrillTrackingTestCase(WebhookTestCase): def test_head_request(self): # Mandrill verifies webhooks at config time with a HEAD request + # (See MandrillWebhookSettingsTestCase above for equivalent without the key yet set) response = self.client.head('/anymail/mandrill/tracking/') self.assertEqual(response.status_code, 200) @@ -159,7 +159,7 @@ class MandrillTrackingTestCase(WebhookTestCase): }] response = self.client.post(**mandrill_args(events=raw_events)) self.assertEqual(response.status_code, 200) - kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillTrackingWebhookView, + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillCombinedWebhookView, event=ANY, esp_name='Mandrill') event = kwargs['event'] self.assertIsInstance(event, AnymailTrackingEvent) @@ -189,7 +189,7 @@ class MandrillTrackingTestCase(WebhookTestCase): }] response = self.client.post(**mandrill_args(events=raw_events)) self.assertEqual(response.status_code, 200) - kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillTrackingWebhookView, + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillCombinedWebhookView, event=ANY, esp_name='Mandrill') event = kwargs['event'] self.assertIsInstance(event, AnymailTrackingEvent) @@ -219,7 +219,7 @@ class MandrillTrackingTestCase(WebhookTestCase): }] response = self.client.post(**mandrill_args(events=raw_events)) self.assertEqual(response.status_code, 200) - kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillTrackingWebhookView, + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillCombinedWebhookView, event=ANY, esp_name='Mandrill') event = kwargs['event'] self.assertIsInstance(event, AnymailTrackingEvent) @@ -241,9 +241,31 @@ class MandrillTrackingTestCase(WebhookTestCase): }] response = self.client.post(**mandrill_args(events=raw_events)) self.assertEqual(response.status_code, 200) - kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillTrackingWebhookView, + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillCombinedWebhookView, event=ANY, esp_name='Mandrill') event = kwargs['event'] self.assertEqual(event.event_type, "unknown") self.assertEqual(event.recipient, "recipient@example.com") self.assertEqual(event.description, "manual edit") + + def test_old_tracking_url(self): + # Earlier versions of Anymail used /mandrill/tracking/ (and didn't support inbound); + # make sure that URL continues to work. + raw_events = [{ + "event": "send", + "msg": { + "ts": 1461095211, # time send called + "subject": "Webhook Test", + "email": "recipient@example.com", + "sender": "sender@example.com", + "tags": ["tag1", "tag2"], + "metadata": {"custom1": "value1", "custom2": "value2"}, + "_id": "abcdef012345789abcdef012345789" + }, + "_id": "abcdef012345789abcdef012345789", + "ts": 1461095246 # time of event + }] + response = self.client.post(**mandrill_args(events=raw_events, path='/anymail/mandrill/tracking/')) + self.assertEqual(response.status_code, 200) + self.assert_handler_called_once_with(self.tracking_handler, sender=MandrillTrackingWebhookView, + event=ANY, esp_name='Mandrill') diff --git a/tests/test_postmark_inbound.py b/tests/test_postmark_inbound.py new file mode 100644 index 0000000..af77615 --- /dev/null +++ b/tests/test_postmark_inbound.py @@ -0,0 +1,224 @@ +import json +from base64 import b64encode + +from mock import ANY + +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.postmark import PostmarkInboundWebhookView + +from .utils import sample_image_content, sample_email_content +from .webhook_cases import WebhookTestCase + + +class PostmarkInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + raw_event = { + "FromFull": { + "Email": "from+test@example.org", + "Name": "Displayed From", + "MailboxHash": "test" + }, + "ToFull": [{ + "Email": "test@inbound.example.com", + "Name": "Test Inbound", + "MailboxHash": "" + }, { + "Email": "other@example.com", + "Name": "", + "MailboxHash": "" + }], + "CcFull": [{ + "Email": "cc@example.com", + "Name": "", + "MailboxHash": "" + }], + "BccFull": [{ + "Email": "bcc@example.com", + "Name": "Postmark documents blind cc on inbound email (?)", + "MailboxHash": "" + }], + "OriginalRecipient": "test@inbound.example.com", + "ReplyTo": "from+test@milter.example.org", + "Subject": "Test subject", + "MessageID": "22c74902-a0c1-4511-804f2-341342852c90", + "Date": "Wed, 11 Oct 2017 18:31:04 -0700", + "TextBody": "Test body plain", + "HtmlBody": "
Test body html
", + "StrippedTextReply": "stripped plaintext body", + "Tag": "", + "Headers": [{ + "Name": "Received", + "Value": "from mail.example.org by inbound.postmarkapp.com ..." + }, { + "Name": "X-Spam-Checker-Version", + "Value": "SpamAssassin 3.4.0 (2014-02-07) onp-pm-smtp-inbound01b-aws-useast2b" + }, { + "Name": "X-Spam-Status", + "Value": "No" + }, { + "Name": "X-Spam-Score", + "Value": "1.7" + }, { + "Name": "X-Spam-Tests", + "Value": "SPF_PASS" + }, { + "Name": "Received-SPF", + "Value": "Pass (sender SPF authorized) identity=mailfrom; client-ip=333.3.3.3;" + " helo=mail-02.example.org; envelope-from=envelope-from@example.org;" + " receiver=test@inbound.example.com" + }, { + "Name": "Received", + "Value": "by mail.example.org for ..." + }, { + "Name": "Received", + "Value": "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)" + }, { + "Name": "MIME-Version", + "Value": "1.0" + }, { + "Name": "Message-ID", + "Value": "" + }], + } + + response = self.client.post('/anymail/postmark/inbound/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=PostmarkInboundWebhookView, + event=ANY, esp_name='Postmark') + # AnymailInboundEvent + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertIsNone(event.timestamp) # Postmark doesn't provide inbound event timestamp + self.assertEqual(event.event_id, "22c74902-a0c1-4511-804f2-341342852c90") + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_event) + + # AnymailInboundMessage - convenience properties + message = event.message + + self.assertEqual(message.from_email.display_name, 'Displayed From') + self.assertEqual(message.from_email.addr_spec, 'from+test@example.org') + self.assertEqual([str(e) for e in message.to], + ['Test Inbound ', 'other@example.com']) + self.assertEqual([str(e) for e in message.cc], + ['cc@example.com']) + self.assertEqual(message.subject, 'Test subject') + self.assertEqual(message.date.isoformat(" "), "2017-10-11 18:31:04-07:00") + self.assertEqual(message.text, 'Test body plain') + self.assertEqual(message.html, '
Test body html
') + + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertEqual(message.stripped_text, 'stripped plaintext body') + self.assertIsNone(message.stripped_html) # Postmark doesn't provide stripped html + self.assertIs(message.spam_detected, False) + self.assertEqual(message.spam_score, 1.7) + + # AnymailInboundMessage - other headers + self.assertEqual(message['Message-ID'], "") + self.assertEqual(message['Reply-To'], "from+test@milter.example.org") + self.assertEqual(message.get_all('Received'), [ + "from mail.example.org by inbound.postmarkapp.com ...", + "by mail.example.org for ...", + "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)", + ]) + + def test_attachments(self): + image_content = sample_image_content() + email_content = sample_email_content() + raw_event = { + "Attachments": [{ + "Name": "test.txt", + "Content": b64encode('test attachment'.encode('utf-8')).decode('ascii'), + "ContentType": "text/plain", + "ContentLength": len('test attachment') + }, { + "Name": "image.png", + "Content": b64encode(image_content).decode('ascii'), + "ContentType": "image/png", + "ContentID": "abc123", + "ContentLength": len(image_content) + }, { + "Name": "bounce.txt", + "Content": b64encode(email_content).decode('ascii'), + "ContentType": 'message/rfc822; charset="us-ascii"', + "ContentLength": len(email_content) + }] + } + + response = self.client.post('/anymail/postmark/inbound/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=PostmarkInboundWebhookView, + event=ANY, esp_name='Postmark') + event = kwargs['event'] + message = event.message + attachments = message.attachments # AnymailInboundMessage convenience accessor + self.assertEqual(len(attachments), 2) + self.assertEqual(attachments[0].get_filename(), 'test.txt') + self.assertEqual(attachments[0].get_content_type(), 'text/plain') + self.assertEqual(attachments[0].get_content_text(), u'test attachment') + self.assertEqual(attachments[1].get_content_type(), 'message/rfc822') + self.assertEqualIgnoringHeaderFolding(attachments[1].get_content_bytes(), email_content) + + inlines = message.inline_attachments + self.assertEqual(len(inlines), 1) + inline = inlines['abc123'] + self.assertEqual(inline.get_filename(), 'image.png') + self.assertEqual(inline.get_content_type(), 'image/png') + self.assertEqual(inline.get_content_bytes(), image_content) + + def test_envelope_sender(self): + # Anymail extracts envelope-sender from Postmark Received-SPF header + raw_event = { + "Headers": [{ + "Name": "Received-SPF", + "Value": "Pass (sender SPF authorized) identity=mailfrom; client-ip=333.3.3.3;" + " helo=mail-02.example.org; envelope-from=envelope-from@example.org;" + " receiver=test@inbound.example.com" + }], + } + response = self.client.post('/anymail/postmark/inbound/', content_type='application/json', + data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.get_kwargs(self.inbound_handler)['event'].message.envelope_sender, + "envelope-from@example.org") + + # Allow neutral SPF response + self.client.post( + '/anymail/postmark/inbound/', content_type='application/json', data=json.dumps({"Headers": [{ + "Name": "Received-SPF", + "Value": "Neutral (no SPF record exists) identity=mailfrom; envelope-from=envelope-from@example.org" + }]})) + self.assertEqual(self.get_kwargs(self.inbound_handler)['event'].message.envelope_sender, + "envelope-from@example.org") + + # Ignore fail/softfail + self.client.post( + '/anymail/postmark/inbound/', content_type='application/json', data=json.dumps({"Headers": [{ + "Name": "Received-SPF", + "Value": "Fail (sender not SPF authorized) identity=mailfrom; envelope-from=spoofed@example.org" + }]})) + self.assertIsNone(self.get_kwargs(self.inbound_handler)['event'].message.envelope_sender) + + # Ignore garbage + self.client.post( + '/anymail/postmark/inbound/', content_type='application/json', data=json.dumps({"Headers": [{ + "Name": "Received-SPF", + "Value": "ThisIsNotAValidReceivedSPFHeader@example.org" + }]})) + self.assertIsNone(self.get_kwargs(self.inbound_handler)['event'].message.envelope_sender) + + # Ignore multiple Received-SPF headers + self.client.post( + '/anymail/postmark/inbound/', content_type='application/json', data=json.dumps({"Headers": [{ + "Name": "Received-SPF", + "Value": "Fail (sender not SPF authorized) identity=mailfrom; envelope-from=spoofed@example.org" + }, { + "Name": "Received-SPF", + "Value": "Pass (malicious sender added this) identity=mailfrom; envelope-from=spoofed@example.org" + }]})) + self.assertIsNone(self.get_kwargs(self.inbound_handler)['event'].message.envelope_sender) diff --git a/tests/test_sendgrid_inbound.py b/tests/test_sendgrid_inbound.py new file mode 100644 index 0000000..f2595c0 --- /dev/null +++ b/tests/test_sendgrid_inbound.py @@ -0,0 +1,183 @@ +import json +from textwrap import dedent + +import six +from mock import ANY + +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.sendgrid import SendGridInboundWebhookView + +from .utils import sample_image_content, sample_email_content +from .webhook_cases import WebhookTestCase + + +class SendgridInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + raw_event = { + 'headers': dedent("""\ + Received: from mail.example.org by mx987654321.sendgrid.net ... + Received: by mail.example.org for ... + DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.org; ... + MIME-Version: 1.0 + Received: by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT) + From: "Displayed From" + Date: Wed, 11 Oct 2017 18:31:04 -0700 + Message-ID: + Subject: Test subject + To: "Test Inbound" , other@example.com + Cc: cc@example.com + Content-Type: multipart/mixed; boundary="94eb2c115edcf35387055b61f849" + """), + 'from': 'Displayed From ', + 'to': 'Test Inbound , other@example.com', + 'subject': "Test subject", + 'text': "Test body plain", + 'html': "
Test body html
", + 'attachments': "0", + 'charsets': '{"to":"UTF-8","html":"UTF-8","subject":"UTF-8","from":"UTF-8","text":"UTF-8"}', + 'envelope': '{"to":["test@inbound.example.com"],"from":"envelope-from@example.org"}', + 'sender_ip': "10.10.1.71", + 'dkim': "{@example.org : pass}", # yep, SendGrid uses not-exactly-json for this field + 'SPF': "pass", + 'spam_score': "1.7", + 'spam_report': 'Spam detection software, running on the system "mx987654321.sendgrid.net", ' + 'has identified this incoming email as possible spam...', + } + response = self.client.post('/anymail/sendgrid/inbound/', data=raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=SendGridInboundWebhookView, + event=ANY, esp_name='SendGrid') + # AnymailInboundEvent + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertIsNone(event.timestamp) + self.assertIsNone(event.event_id) + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event.POST.dict(), raw_event) # esp_event is a Django HttpRequest + + # AnymailInboundMessage - convenience properties + message = event.message + + self.assertEqual(message.from_email.display_name, 'Displayed From') + self.assertEqual(message.from_email.addr_spec, 'from+test@example.org') + self.assertEqual([str(e) for e in message.to], + ['Test Inbound ', 'other@example.com']) + self.assertEqual([str(e) for e in message.cc], + ['cc@example.com']) + self.assertEqual(message.subject, 'Test subject') + self.assertEqual(message.date.isoformat(" "), "2017-10-11 18:31:04-07:00") + self.assertEqual(message.text, 'Test body plain') + self.assertEqual(message.html, '
Test body html
') + + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertIsNone(message.stripped_text) + self.assertIsNone(message.stripped_html) + self.assertIsNone(message.spam_detected) # SendGrid doesn't give a simple yes/no; check the score yourself + self.assertEqual(message.spam_score, 1.7) + + # AnymailInboundMessage - other headers + self.assertEqual(message['Message-ID'], "") + self.assertEqual(message.get_all('Received'), [ + "from mail.example.org by mx987654321.sendgrid.net ...", + "by mail.example.org for ...", + "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)", + ]) + + def test_attachments(self): + att1 = six.BytesIO('test attachment'.encode('utf-8')) + att1.name = 'test.txt' + image_content = sample_image_content() + att2 = six.BytesIO(image_content) + att2.name = 'image.png' + email_content = sample_email_content() + att3 = six.BytesIO(email_content) + att3.content_type = 'message/rfc822; charset="us-ascii"' + raw_event = { + 'headers': '', + 'attachments': '3', + 'attachment-info': json.dumps({ + "attachment3": {"filename": "", "name": "", "charset": "US-ASCII", "type": "message/rfc822"}, + "attachment2": {"filename": "image.png", "name": "image.png", "type": "image/png", + "content-id": "abc123"}, + "attachment1": {"filename": "test.txt", "name": "test.txt", "type": "text/plain"}, + }), + 'content-ids': '{"abc123": "attachment2"}', + 'attachment1': att1, + 'attachment2': att2, # inline + 'attachment3': att3, + } + + response = self.client.post('/anymail/sendgrid/inbound/', data=raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=SendGridInboundWebhookView, + event=ANY, esp_name='SendGrid') + event = kwargs['event'] + message = event.message + attachments = message.attachments # AnymailInboundMessage convenience accessor + self.assertEqual(len(attachments), 2) + self.assertEqual(attachments[0].get_filename(), 'test.txt') + self.assertEqual(attachments[0].get_content_type(), 'text/plain') + self.assertEqual(attachments[0].get_content_text(), u'test attachment') + self.assertEqual(attachments[1].get_content_type(), 'message/rfc822') + self.assertEqualIgnoringHeaderFolding(attachments[1].get_content_bytes(), email_content) + + inlines = message.inline_attachments + self.assertEqual(len(inlines), 1) + inline = inlines['abc123'] + self.assertEqual(inline.get_filename(), 'image.png') + self.assertEqual(inline.get_content_type(), 'image/png') + self.assertEqual(inline.get_content_bytes(), image_content) + + def test_inbound_mime(self): + # SendGrid has an option to send the full, raw MIME message + raw_event = { + 'email': dedent("""\ + From: A tester + Date: Thu, 12 Oct 2017 18:03:30 -0700 + Message-ID: + Subject: Raw MIME test + To: test@inbound.example.com + MIME-Version: 1.0 + Content-Type: multipart/alternative; boundary="94eb2c05e174adb140055b6339c5" + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/plain; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + + It's a body=E2=80=A6 + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/html; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + +
It's a body=E2=80=A6
+ + --94eb2c05e174adb140055b6339c5-- + """), + 'from': 'A tester ', + 'to': 'test@inbound.example.com', + 'subject': "Raw MIME test", + 'charsets': '{"to":"UTF-8","subject":"UTF-8","from":"UTF-8"}', + 'envelope': '{"to":["test@inbound.example.com"],"from":"envelope-from@example.org"}', + 'sender_ip': "10.10.1.71", + 'dkim': "{@example.org : pass}", # yep, SendGrid uses not-exactly-json for this field + 'SPF': "pass", + 'spam_score': "1.7", + 'spam_report': 'Spam detection software, running on the system "mx987654321.sendgrid.net", ' + 'has identified this incoming email as possible spam...', + } + + response = self.client.post('/anymail/sendgrid/inbound/', data=raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=SendGridInboundWebhookView, + event=ANY, esp_name='SendGrid') + event = kwargs['event'] + message = event.message + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertEqual(message.subject, 'Raw MIME test') + self.assertEqual(message.text, u"It's a body\N{HORIZONTAL ELLIPSIS}\n") + self.assertEqual(message.html, u"""
It's a body\N{HORIZONTAL ELLIPSIS}
\n""") diff --git a/tests/test_sparkpost_inbound.py b/tests/test_sparkpost_inbound.py new file mode 100644 index 0000000..037e9c3 --- /dev/null +++ b/tests/test_sparkpost_inbound.py @@ -0,0 +1,175 @@ +import json +from base64 import b64encode +from textwrap import dedent + +from mock import ANY + +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.sparkpost import SparkPostInboundWebhookView + +from .utils import sample_image_content, sample_email_content +from .webhook_cases import WebhookTestCase + + +class SparkpostInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + event = { + 'protocol': "smtp", + 'rcpt_to': "test@inbound.example.com", + 'msg_from': "envelope-from@example.org", + 'content': { + # Anymail just parses the raw rfc822 email. SparkPost's other content fields are ignored. + 'email_rfc822_is_base64': False, + 'email_rfc822': dedent("""\ + Received: from mail.example.org by c.mta1vsmtp.cc.prd.sparkpost ... + Received: by mail.example.org for ... + DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.org; ... + MIME-Version: 1.0 + Received: by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT) + From: "Displayed From" + Date: Wed, 11 Oct 2017 18:31:04 -0700 + Message-ID: + Subject: Test subject + To: "Test Inbound" , other@example.com + Cc: cc@example.com + Content-Type: multipart/alternative; boundary="94eb2c05e174adb140055b6339c5" + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/plain; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + + It's a body=E2=80=A6 + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/html; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + +
It's a body=E2=80=A6
+ + --94eb2c05e174adb140055b6339c5-- + """), + }, + } + raw_event = {'msys': {'relay_message': event}} + + response = self.client.post('/anymail/sparkpost/inbound/', + content_type='application/json', data=json.dumps([raw_event])) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=SparkPostInboundWebhookView, + event=ANY, esp_name='SparkPost') + # AnymailInboundEvent + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertIsNone(event.timestamp) + self.assertIsNone(event.event_id) + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_event) + + # AnymailInboundMessage - convenience properties + message = event.message + + self.assertEqual(message.from_email.display_name, 'Displayed From') + self.assertEqual(message.from_email.addr_spec, 'from+test@example.org') + self.assertEqual([str(e) for e in message.to], + ['Test Inbound ', 'other@example.com']) + self.assertEqual([str(e) for e in message.cc], + ['cc@example.com']) + self.assertEqual(message.subject, 'Test subject') + self.assertEqual(message.date.isoformat(" "), "2017-10-11 18:31:04-07:00") + self.assertEqual(message.text, u"It's a body\N{HORIZONTAL ELLIPSIS}\n") + self.assertEqual(message.html, u"""
It's a body\N{HORIZONTAL ELLIPSIS}
\n""") + + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'test@inbound.example.com') + self.assertIsNone(message.stripped_text) + self.assertIsNone(message.stripped_html) + self.assertIsNone(message.spam_detected) + self.assertIsNone(message.spam_score) + + # AnymailInboundMessage - other headers + self.assertEqual(message['Message-ID'], "") + self.assertEqual(message.get_all('Received'), [ + "from mail.example.org by c.mta1vsmtp.cc.prd.sparkpost ...", + "by mail.example.org for ...", + "by 10.10.1.71 with HTTP; Wed, 11 Oct 2017 18:31:04 -0700 (PDT)", + ]) + + def test_attachments(self): + image_content = sample_image_content() + email_content = sample_email_content() + raw_mime = dedent("""\ + MIME-Version: 1.0 + From: from@example.org + Subject: Attachments + To: test@inbound.example.com + Content-Type: multipart/mixed; boundary="boundary0" + + --boundary0 + Content-Type: multipart/related; boundary="boundary1" + + --boundary1 + Content-Type: text/html; charset="UTF-8" + +
This is the HTML body. It has an inline image: .
+ + --boundary1 + Content-Type: image/png + Content-Disposition: inline; filename="image.png" + Content-ID: + Content-Transfer-Encoding: base64 + + {image_content_base64} + --boundary1-- + --boundary0 + Content-Type: text/plain; charset="UTF-8" + Content-Disposition: attachment; filename="test.txt" + + test attachment + --boundary0 + Content-Type: message/rfc822; charset="US-ASCII" + Content-Disposition: attachment + X-Comment: (the only valid transfer encodings for message/* are 7bit, 8bit, and binary) + + {email_content} + --boundary0-- + """).format(image_content_base64=b64encode(image_content).decode('ascii'), + email_content=email_content.decode('ascii')) + + raw_event = {'msys': {'relay_message': { + 'protocol': "smtp", + 'content': { + 'email_rfc822_is_base64': True, + 'email_rfc822': b64encode(raw_mime.encode('utf-8')).decode('ascii'), + }, + }}} + + response = self.client.post('/anymail/sparkpost/inbound/', + content_type='application/json', data=json.dumps([raw_event])) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=SparkPostInboundWebhookView, + event=ANY, esp_name='SparkPost') + event = kwargs['event'] + message = event.message + attachments = message.attachments # AnymailInboundMessage convenience accessor + self.assertEqual(len(attachments), 2) + self.assertEqual(attachments[0].get_filename(), 'test.txt') + self.assertEqual(attachments[0].get_content_type(), 'text/plain') + self.assertEqual(attachments[0].get_content_text(), u'test attachment') + self.assertEqual(attachments[1].get_content_type(), 'message/rfc822') + self.assertEqualIgnoringHeaderFolding(attachments[1].get_content_bytes(), email_content) + + # the message attachment (its payload) is fully parsed + # (see the original in test_files/sample_email.txt) + att_message = attachments[1].get_payload(0) + self.assertEqual(att_message.get_content_type(), "multipart/alternative") + self.assertEqual(att_message['Subject'], "Test email") + self.assertEqual(att_message.text, "Hi Bob, This is a message. Thanks!\n") + + inlines = message.inline_attachments + self.assertEqual(len(inlines), 1) + inline = inlines['abc123'] + self.assertEqual(inline.get_filename(), 'image.png') + self.assertEqual(inline.get_content_type(), 'image/png') + self.assertEqual(inline.get_content_bytes(), image_content) diff --git a/tests/utils.py b/tests/utils.py index 903a790..479a4e0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,7 @@ import warnings from base64 import b64decode from contextlib import contextmanager +import six from django.test import Client @@ -35,6 +36,12 @@ def decode_att(att): return b64decode(att.encode('ascii')) +def rfc822_unfold(text): + # "Unfolding is accomplished by simply removing any CRLF that is immediately followed by WSP" + # (WSP is space or tab, and per email.parser semantics, we allow CRLF, CR, or LF endings) + return re.sub(r'(\r\n|\r|\n)(?=[ \t])', "", text) + + # # Sample files for testing (in ./test_files subdir) # @@ -133,11 +140,18 @@ class AnymailTestMixin: except TypeError: return self.assertRegexpMatches(*args, **kwargs) # Python 2 - def assertEqualIgnoringWhitespace(self, first, second, msg=None): - # Useful for message/rfc822 attachment tests - self.assertEqual(first.replace(b'\n', b'').replace(b' ', b''), - second.replace(b'\n', b'').replace(b' ', b''), - msg) + def assertEqualIgnoringHeaderFolding(self, first, second, msg=None): + # Unfold (per RFC-8222) all text first and second, then compare result. + # Useful for message/rfc822 attachment tests, where various Python email + # versions handled folding slightly differently. + # (Technically, this is unfolding both headers and (incorrectly) bodies, + # but that doesn't really affect the tests.) + if isinstance(first, six.binary_type) and isinstance(second, six.binary_type): + first = first.decode('utf-8') + second = second.decode('utf-8') + first = rfc822_unfold(first) + second = rfc822_unfold(second) + self.assertEqual(first, second, msg) # Backported from python 3.5 diff --git a/tests/webhook_cases.py b/tests/webhook_cases.py index 26fd569..249bf19 100644 --- a/tests/webhook_cases.py +++ b/tests/webhook_cases.py @@ -64,6 +64,12 @@ class WebhookTestCase(AnymailTestMixin, SimpleTestCase): self.assertEqual(actual_kwargs[key], expected_value) return actual_kwargs + def get_kwargs(self, mockfn): + """Return the kwargs passed to the most recent call to mockfn""" + self.assertIsNotNone(mockfn.call_args) # mockfn hasn't been called yet + actual_args, actual_kwargs = mockfn.call_args + return actual_kwargs + # noinspection PyUnresolvedReferences class WebhookBasicAuthTestsMixin(object):