diff --git a/README.rst b/README.rst index d580810..b001c8f 100644 --- a/README.rst +++ b/README.rst @@ -23,7 +23,7 @@ Anymail integrates several transactional email service providers (ESPs) into Dja with a consistent API that lets you use ESP-added features without locking your code to a particular ESP. -It currently fully supports **Mailgun, Mailjet, Postmark, SendinBlue, SendGrid,** +It currently fully supports **Amazon SES, Mailgun, Mailjet, Postmark, SendinBlue, SendGrid,** and **SparkPost,** and has limited support for **Mandrill.** Anymail normalizes ESP functionality so it "just works" with Django's diff --git a/anymail/backends/amazon_ses.py b/anymail/backends/amazon_ses.py new file mode 100644 index 0000000..e63a34d --- /dev/null +++ b/anymail/backends/amazon_ses.py @@ -0,0 +1,387 @@ +from email.header import Header +from email.mime.base import MIMEBase + +from django.core.mail import BadHeaderError + +from .base import AnymailBaseBackend, BasePayload +from .._version import __version__ +from ..exceptions import AnymailAPIError, AnymailImproperlyInstalled +from ..message import AnymailRecipientStatus +from ..utils import get_anymail_setting, UNSET + +try: + import boto3 + from botocore.client import Config + from botocore.exceptions import BotoCoreError, ClientError, ConnectionError +except ImportError: + raise AnymailImproperlyInstalled(missing_package='boto3', backend='amazon_ses') + + +# boto3 has several root exception classes; this is meant to cover all of them +BOTO_BASE_ERRORS = (BotoCoreError, ClientError, ConnectionError) + + +# Work around Python 2 bug in email.message.Message.to_string, where long headers +# containing commas or semicolons get an extra space inserted after every ',' or ';' +# not already followed by a space. https://bugs.python.org/issue25257 +if Header("test,Python2,header,comma,bug", maxlinelen=20).encode() == "test,Python2,header,comma,bug": + # no workaround needed + HeaderBugWorkaround = None + + def add_header(message, name, val): + message[name] = val + +else: + # workaround: custom Header subclass that won't consider ',' and ';' as folding candidates + + class HeaderBugWorkaround(Header): + def encode(self, splitchars=' ', **kwargs): # only split on spaces, rather than splitchars=';, ' + return Header.encode(self, splitchars, **kwargs) + + def add_header(message, name, val): + # Must bypass Django's SafeMIMEMessage.__set_item__, because its call to + # forbid_multi_line_headers converts the val back to a str, undoing this + # workaround. That makes this code responsible for sanitizing val: + if '\n' in val or '\r' in val: + raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name)) + val = HeaderBugWorkaround(val, header_name=name) + assert isinstance(message, MIMEBase) + MIMEBase.__setitem__(message, name, val) + + +class EmailBackend(AnymailBaseBackend): + """ + Amazon SES Email Backend (using boto3) + """ + + esp_name = "Amazon SES" + + def __init__(self, **kwargs): + """Init options from Django settings""" + super(EmailBackend, self).__init__(**kwargs) + # AMAZON_SES_CLIENT_PARAMS is optional - boto3 can find credentials several other ways + self.session_params, self.client_params = _get_anymail_boto3_params(kwargs=kwargs) + self.configuration_set_name = get_anymail_setting("configuration_set_name", esp_name=self.esp_name, + kwargs=kwargs, allow_bare=False, default=None) + self.message_tag_name = get_anymail_setting("message_tag_name", esp_name=self.esp_name, + kwargs=kwargs, allow_bare=False, default=None) + self.client = None + + def open(self): + if self.client: + return False # already exists + try: + self.client = boto3.session.Session(**self.session_params).client("ses", **self.client_params) + except BOTO_BASE_ERRORS: + if not self.fail_silently: + raise + + def close(self): + if self.client is None: + return + # self.client.close() # boto3 doesn't currently seem to support (or require) this + self.client = None + + def build_message_payload(self, message, defaults): + # The SES SendRawEmail and SendBulkTemplatedEmail calls have + # very different signatures, so use a custom payload for each + if getattr(message, "template_id", UNSET) is not UNSET: + return AmazonSESSendBulkTemplatedEmailPayload(message, defaults, self) + else: + return AmazonSESSendRawEmailPayload(message, defaults, self) + + def post_to_esp(self, payload, message): + try: + response = payload.call_send_api(self.client) + except BOTO_BASE_ERRORS as err: + # ClientError has a response attr with parsed json error response (other errors don't) + raise AnymailAPIError(str(err), backend=self, email_message=message, payload=payload, + response=getattr(err, 'response', None), raised_from=err) + return response + + def parse_recipient_status(self, response, payload, message): + return payload.parse_recipient_status(response) + + +class AmazonSESBasePayload(BasePayload): + def init_payload(self): + self.params = {} + if self.backend.configuration_set_name is not None: + self.params["ConfigurationSetName"] = self.backend.configuration_set_name + + def call_send_api(self, ses_client): + raise NotImplementedError() + + def parse_recipient_status(self, response): + # response is the parsed (dict) JSON returned from the API call + raise NotImplementedError() + + def set_esp_extra(self, extra): + # e.g., ConfigurationSetName, FromArn, SourceArn, ReturnPathArn + self.params.update(extra) + + +class AmazonSESSendRawEmailPayload(AmazonSESBasePayload): + def init_payload(self): + super(AmazonSESSendRawEmailPayload, self).init_payload() + self.all_recipients = [] + self.mime_message = self.message.message() + if HeaderBugWorkaround and "Subject" in self.mime_message: + # (message.message() will have already checked subject for BadHeaderError) + self.mime_message.replace_header("Subject", HeaderBugWorkaround(self.message.subject)) + + def call_send_api(self, ses_client): + self.params["RawMessage"] = { + # Note: "Destinations" is determined from message headers if not provided + # "Destinations": [email.addr_spec for email in self.all_recipients], + "Data": self.mime_message.as_bytes() + } + return ses_client.send_raw_email(**self.params) + + def parse_recipient_status(self, response): + try: + message_id = response["MessageId"] + except (KeyError, TypeError) as err: + raise AnymailAPIError( + "%s parsing Amazon SES send result %r" % (str(err), response), + backend=self.backend, email_message=self.message, payload=self) + + recipient_status = AnymailRecipientStatus(message_id=message_id, status="queued") + return {recipient.addr_spec: recipient_status for recipient in self.all_recipients} + + # Standard EmailMessage attrs... + # These all get rolled into the RFC-5322 raw mime directly via EmailMessage.message() + + def _no_send_defaults(self, attr): + # Anymail global send defaults don't work for standard attrs, because the + # merged/computed value isn't forced back into the EmailMessage. + if attr in self.defaults: + self.unsupported_feature("Anymail send defaults for '%s' with Amazon SES" % attr) + + def set_from_email_list(self, emails): + # Although Amazon SES will send messages with any From header, it can only parse Source + # if the From header is a single email. Explicit Source avoids an "Illegal address" error: + if len(emails) > 1: + self.params["Source"] = emails[0].addr_spec + # (else SES will look at the (single) address in the From header) + + def set_recipients(self, recipient_type, emails): + self.all_recipients += emails + # included in mime_message + assert recipient_type in ("to", "cc", "bcc") + self._no_send_defaults(recipient_type) + + def set_subject(self, subject): + # included in mime_message + self._no_send_defaults("subject") + + def set_reply_to(self, emails): + # included in mime_message + self._no_send_defaults("reply_to") + + def set_extra_headers(self, headers): + # included in mime_message + self._no_send_defaults("extra_headers") + + def set_text_body(self, body): + # included in mime_message + self._no_send_defaults("body") + + def set_html_body(self, body): + # included in mime_message + self._no_send_defaults("body") + + def set_alternatives(self, alternatives): + # included in mime_message + self._no_send_defaults("alternatives") + + def set_attachments(self, attachments): + # included in mime_message + self._no_send_defaults("attachments") + + # Anymail-specific payload construction + def set_envelope_sender(self, email): + self.params["Source"] = email.addr_spec + + def set_spoofed_to_header(self, header_to): + # django.core.mail.EmailMessage.message() has already set + # self.mime_message["To"] = header_to + # and performed any necessary header sanitization + self.params["Destinations"] = [email.addr_spec for email in self.all_recipients] + + def set_metadata(self, metadata): + # Amazon SES has two mechanisms for adding custom data to a message: + # * Custom message headers are available to webhooks (SNS notifications), + # but not in CloudWatch metrics/dashboards or Kinesis Firehose streams. + # Custom headers can be sent only with SendRawEmail. + # * "Message Tags" are available to CloudWatch and Firehose, and to SNS + # notifications for SES *events* but not SES *notifications*. (Got that?) + # Message Tags also allow *very* limited characters in both name and value. + # Message Tags can be sent with any SES send call. + # (See "How do message tags work?" in https://aws.amazon.com/blogs/ses/introducing-sending-metrics/ + # and https://forums.aws.amazon.com/thread.jspa?messageID=782922.) + # To support reliable retrieval in webhooks, just use custom headers for metadata. + add_header(self.mime_message, "X-Metadata", self.serialize_json(metadata)) + + def set_tags(self, tags): + # See note about Amazon SES Message Tags and custom headers in set_metadata above. + # To support reliable retrieval in webhooks, use custom headers for tags. + # (There are no restrictions on number or content for custom header tags.) + for tag in tags: + add_header(self.mime_message, "X-Tag", tag) # creates multiple X-Tag headers, one per tag + + # Also *optionally* pass a single Message Tag if the AMAZON_SES_MESSAGE_TAG_NAME + # Anymail setting is set (default no). The AWS API restricts tag content in this case. + # (This is useful for dashboard segmentation; use esp_extra["Tags"] for anything more complex.) + if tags and self.backend.message_tag_name is not None: + if len(tags) > 1: + self.unsupported_feature("multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting") + self.params.setdefault("Tags", []).append( + {"Name": self.backend.message_tag_name, "Value": tags[0]}) + + def set_template_id(self, template_id): + raise NotImplementedError("AmazonSESSendRawEmailPayload should not have been used with template_id") + + def set_merge_data(self, merge_data): + self.unsupported_feature("merge_data without template_id") + + def set_merge_global_data(self, merge_global_data): + self.unsupported_feature("global_merge_data without template_id") + + +class AmazonSESSendBulkTemplatedEmailPayload(AmazonSESBasePayload): + def init_payload(self): + super(AmazonSESSendBulkTemplatedEmailPayload, self).init_payload() + # late-bind recipients and merge_data in call_send_api + self.recipients = {"to": [], "cc": [], "bcc": []} + self.merge_data = {} + + def call_send_api(self, ses_client): + # include any 'cc' or 'bcc' in every destination + cc_and_bcc_addresses = {} + if self.recipients["cc"]: + cc_and_bcc_addresses["CcAddresses"] = [cc.address for cc in self.recipients["cc"]] + if self.recipients["bcc"]: + cc_and_bcc_addresses["BccAddresses"] = [bcc.address for bcc in self.recipients["bcc"]] + + # set up destination and data for each 'to' + self.params["Destinations"] = [{ + "Destination": dict(ToAddresses=[to.address], **cc_and_bcc_addresses), + "ReplacementTemplateData": self.serialize_json(self.merge_data.get(to.addr_spec, {})) + } for to in self.recipients["to"]] + + return ses_client.send_bulk_templated_email(**self.params) + + def parse_recipient_status(self, response): + try: + # response["Status"] should be a list in Destinations (to) order + anymail_statuses = [ + AnymailRecipientStatus( + message_id=status.get("MessageId", None), + status='queued' if status.get("Status") == "Success" else 'failed') + for status in response["Status"] + ] + except (KeyError, TypeError) as err: + raise AnymailAPIError( + "%s parsing Amazon SES send result %r" % (str(err), response), + backend=self.backend, email_message=self.message, payload=self) + + to_addrs = [to.addr_spec for to in self.recipients["to"]] + if len(anymail_statuses) != len(to_addrs): + raise AnymailAPIError( + "Sent to %d destinations, but only %d statuses in Amazon SES send result %r" + % (len(to_addrs), len(anymail_statuses), response), + backend=self.backend, email_message=self.message, payload=self) + + return dict(zip(to_addrs, anymail_statuses)) + + def set_from_email(self, email): + self.params["Source"] = email.address # this will RFC2047-encode display_name if needed + + def set_recipients(self, recipient_type, emails): + # late-bound in call_send_api + assert recipient_type in ("to", "cc", "bcc") + self.recipients[recipient_type] = emails + + def set_subject(self, subject): + # (subject can only come from template; you can use substitution vars in that) + if subject: + self.unsupported_feature("overriding template subject") + + def set_reply_to(self, emails): + if emails: + self.params["ReplyToAddresses"] = [email.address for email in emails] + + def set_extra_headers(self, headers): + self.unsupported_feature("extra_headers with template") + + def set_text_body(self, body): + if body: + self.unsupported_feature("overriding template body content") + + def set_html_body(self, body): + if body: + self.unsupported_feature("overriding template body content") + + def set_attachments(self, attachments): + if attachments: + self.unsupported_feature("attachments with template") + + # Anymail-specific payload construction + def set_envelope_sender(self, email): + self.params["ReturnPath"] = email.addr_spec + + def set_metadata(self, metadata): + # no custom headers with SendBulkTemplatedEmail + self.unsupported_feature("metadata with template") + + def set_tags(self, tags): + # no custom headers with SendBulkTemplatedEmail, but support AMAZON_SES_MESSAGE_TAG_NAME if used + # (see tags/metadata in AmazonSESSendRawEmailPayload for more info) + if tags: + if self.backend.message_tag_name is not None: + if len(tags) > 1: + self.unsupported_feature("multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting") + self.params["DefaultTags"] = [{"Name": self.backend.message_tag_name, "Value": tags[0]}] + else: + self.unsupported_feature( + "tags with template (unless using the AMAZON_SES_MESSAGE_TAG_NAME setting)") + + def set_template_id(self, template_id): + self.params["Template"] = template_id + + def set_merge_data(self, merge_data): + # late-bound in call_send_api + self.merge_data = merge_data + + def set_merge_global_data(self, merge_global_data): + self.params["DefaultTemplateData"] = self.serialize_json(merge_global_data) + + +def _get_anymail_boto3_params(esp_name=EmailBackend.esp_name, kwargs=None): + """Returns 2 dicts of params for boto3.session.Session() and .client() + + Incorporates ANYMAIL["AMAZON_SES_SESSION_PARAMS"] and + ANYMAIL["AMAZON_SES_CLIENT_PARAMS"] settings. + + Converts config dict to botocore.client.Config if needed + + May remove keys from kwargs, but won't modify original settings + """ + # (shared with ..webhooks.amazon_ses) + session_params = get_anymail_setting("session_params", esp_name=esp_name, kwargs=kwargs, default={}) + client_params = get_anymail_setting("client_params", esp_name=esp_name, kwargs=kwargs, default={}) + + # Add Anymail user-agent, and convert config dict to botocore.client.Config + client_params = client_params.copy() # don't modify source + config = Config(user_agent_extra="django-anymail/{version}-{esp}".format( + esp=esp_name.lower().replace(" ", "-"), version=__version__)) + if "config" in client_params: + # convert config dict to botocore.client.Config if needed + client_params_config = client_params["config"] + if not isinstance(client_params_config, Config): + client_params_config = Config(**client_params_config) + config = config.merge(client_params_config) + client_params["config"] = config + + return session_params, client_params diff --git a/anymail/urls.py b/anymail/urls.py index 9c8ae9f..75eb99f 100644 --- a/anymail/urls.py +++ b/anymail/urls.py @@ -1,5 +1,6 @@ from django.conf.urls import url +from .webhooks.amazon_ses import AmazonSESInboundWebhookView, AmazonSESTrackingWebhookView from .webhooks.mailgun import MailgunInboundWebhookView, MailgunTrackingWebhookView from .webhooks.mailjet import MailjetInboundWebhookView, MailjetTrackingWebhookView from .webhooks.mandrill import MandrillCombinedWebhookView @@ -11,12 +12,14 @@ from .webhooks.sparkpost import SparkPostInboundWebhookView, SparkPostTrackingWe app_name = 'anymail' urlpatterns = [ + url(r'^amazon_ses/inbound/$', AmazonSESInboundWebhookView.as_view(), name='amazon_ses_inbound_webhook'), url(r'^mailgun/inbound(_mime)?/$', MailgunInboundWebhookView.as_view(), name='mailgun_inbound_webhook'), url(r'^mailjet/inbound/$', MailjetInboundWebhookView.as_view(), name='mailjet_inbound_webhook'), url(r'^postmark/inbound/$', PostmarkInboundWebhookView.as_view(), name='postmark_inbound_webhook'), url(r'^sendgrid/inbound/$', SendGridInboundWebhookView.as_view(), name='sendgrid_inbound_webhook'), url(r'^sparkpost/inbound/$', SparkPostInboundWebhookView.as_view(), name='sparkpost_inbound_webhook'), + url(r'^amazon_ses/tracking/$', AmazonSESTrackingWebhookView.as_view(), name='amazon_ses_tracking_webhook'), url(r'^mailgun/tracking/$', MailgunTrackingWebhookView.as_view(), name='mailgun_tracking_webhook'), url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'), url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'), diff --git a/anymail/utils.py b/anymail/utils.py index ba0e23b..24f6b40 100644 --- a/anymail/utils.py +++ b/anymail/utils.py @@ -349,7 +349,7 @@ def get_anymail_setting(name, default=UNSET, esp_name=None, kwargs=None, allow_b pass if esp_name is not None: - setting = "{}_{}".format(esp_name.upper(), name.upper()) + setting = "{}_{}".format(esp_name.upper().replace(" ", "_"), name.upper()) else: setting = name.upper() anymail_setting = "ANYMAIL_%s" % setting diff --git a/anymail/webhooks/amazon_ses.py b/anymail/webhooks/amazon_ses.py new file mode 100644 index 0000000..62bdc65 --- /dev/null +++ b/anymail/webhooks/amazon_ses.py @@ -0,0 +1,348 @@ +import io +import json +from base64 import b64decode + +from django.http import HttpResponse +from django.utils.dateparse import parse_datetime + +from .base import AnymailBaseWebhookView +from ..backends.amazon_ses import _get_anymail_boto3_params +from ..exceptions import ( + AnymailAPIError, AnymailConfigurationError, AnymailImproperlyInstalled, AnymailWebhookValidationFailure) +from ..inbound import AnymailInboundMessage +from ..signals import AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason, inbound, tracking +from ..utils import combine, get_anymail_setting, getfirst + +try: + import boto3 + import botocore.exceptions +except ImportError: + raise AnymailImproperlyInstalled(missing_package='boto3', backend='amazon_ses') + + +class AmazonSESBaseWebhookView(AnymailBaseWebhookView): + """Base view class for Amazon SES webhooks (SNS Notifications)""" + + esp_name = "Amazon SES" + + def __init__(self, **kwargs): + # whether to automatically respond to SNS SubscriptionConfirmation requests; default True + # (Future: could also take a TopicArn or list to auto-confirm) + self.auto_confirm_enabled = get_anymail_setting( + "auto_confirm_sns_subscriptions", esp_name=self.esp_name, kwargs=kwargs, default=True) + # boto3 params for connecting to S3 (inbound downloads) and SNS (auto-confirm subscriptions): + self.session_params, self.client_params = _get_anymail_boto3_params(kwargs=kwargs) + super(AmazonSESBaseWebhookView, self).__init__(**kwargs) + + @staticmethod + def _parse_sns_message(request): + # cache so we don't have to parse the json multiple times + if not hasattr(request, '_sns_message'): + try: + body = request.body.decode(request.encoding or 'utf-8') + request._sns_message = json.loads(body) + except (TypeError, ValueError, UnicodeDecodeError) as err: + raise AnymailAPIError("Malformed SNS message body %r" % request.body, raised_from=err) + return request._sns_message + + def validate_request(self, request): + # Block random posts that don't even have matching SNS headers + sns_message = self._parse_sns_message(request) + header_type = request.META.get("HTTP_X_AMZ_SNS_MESSAGE_TYPE", "<>") + body_type = sns_message.get("Type", "<>") + if header_type != body_type: + raise AnymailWebhookValidationFailure( + 'SNS header "x-amz-sns-message-type: %s" doesn\'t match body "Type": "%s"' + % (header_type, body_type)) + + if header_type not in ["Notification", "SubscriptionConfirmation", "UnsubscribeConfirmation"]: + raise AnymailAPIError("Unknown SNS message type '%s'" % header_type) + + header_id = request.META.get("HTTP_X_AMZ_SNS_MESSAGE_ID", "<>") + body_id = sns_message.get("MessageId", "<>") + if header_id != body_id: + raise AnymailWebhookValidationFailure( + 'SNS header "x-amz-sns-message-id: %s" doesn\'t match body "MessageId": "%s"' + % (header_id, body_id)) + + # Future: Verify SNS message signature + # https://docs.aws.amazon.com/sns/latest/dg/SendMessageToHttp.verify.signature.html + + def post(self, request, *args, **kwargs): + # request has *not* yet been validated at this point + if self.basic_auth and not request.META.get("HTTP_AUTHORIZATION"): + # Amazon SNS requires a proper 401 response before it will attempt to send basic auth + response = HttpResponse(status=401) + response["WWW-Authenticate"] = 'Basic realm="Anymail WEBHOOK_SECRET"' + return response + return super(AmazonSESBaseWebhookView, self).post(request, *args, **kwargs) + + def parse_events(self, request): + # request *has* been validated by now + events = [] + sns_message = self._parse_sns_message(request) + sns_type = sns_message.get("Type") + if sns_type == "Notification": + message_string = sns_message.get("Message") + try: + ses_event = json.loads(message_string) + except (TypeError, ValueError): + if message_string == "Successfully validated SNS topic for Amazon SES event publishing.": + pass # this Notification is generated after SubscriptionConfirmation + else: + raise AnymailAPIError("Unparsable SNS Message %r" % message_string) + else: + events = self.esp_to_anymail_events(ses_event, sns_message) + elif sns_type == "SubscriptionConfirmation": + self.auto_confirm_sns_subscription(sns_message) + # else: just ignore other SNS messages (e.g., "UnsubscribeConfirmation") + return events + + def esp_to_anymail_events(self, ses_event, sns_message): + raise NotImplementedError() + + def auto_confirm_sns_subscription(self, sns_message): + """Automatically accept a subscription to Amazon SNS topics, if the request is expected. + + If an SNS SubscriptionConfirmation arrives with HTTP basic auth proving it is meant for us, + automatically load the SubscribeURL to confirm the subscription. + """ + if not self.auto_confirm_enabled: + return + + if not self.basic_auth: + # Note: basic_auth (shared secret) confirms the notification was meant for us. + # If WEBHOOK_SECRET isn't set, Anymail logs a warning but allows the request. + # (Also, verifying the SNS message signature would be insufficient here: + # if someone else tried to point their own SNS topic at our webhook url, + # SNS would send a SubscriptionConfirmation with a valid Amazon signature.) + raise AnymailWebhookValidationFailure( + "Anymail received an unexpected SubscriptionConfirmation request for Amazon SNS topic " + "'{topic_arn!s}'. (Anymail can automatically confirm SNS subscriptions if you set a " + "WEBHOOK_SECRET and use that in your SNS notification url. Or you can manually confirm " + "this subscription in the SNS dashboard with token '{token!s}'.)" + "".format(topic_arn=sns_message.get('TopicArn'), token=sns_message.get('Token'))) + + # WEBHOOK_SECRET *is* set, so the request's basic auth has been verified by now (in run_validators). + # We're good to confirm... + sns_client = boto3.session.Session(**self.session_params).client('sns', **self.client_params) + sns_client.confirm_subscription( + TopicArn=sns_message["TopicArn"], Token=sns_message["Token"], AuthenticateOnUnsubscribe='true') + + +class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView): + """Handler for Amazon SES tracking notifications""" + + signal = tracking + + def esp_to_anymail_events(self, ses_event, sns_message): + # Amazon SES has two notification formats, which are almost exactly the same: + # - https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html + # - https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html + # This code should handle either. + ses_event_type = getfirst(ses_event, ["eventType", "notificationType"], "<>") + if ses_event_type == "Received": + # This is an inbound event + raise AnymailConfigurationError( + "You seem to have set an Amazon SES *inbound* receipt rule to publish " + "to an SNS Topic that posts to Anymail's *tracking* webhook URL. " + "(SNS TopicArn %s)" % sns_message.get("TopicArn")) + + event_id = sns_message.get("MessageId") # unique to the SNS notification + try: + timestamp = parse_datetime(sns_message["Timestamp"]) + except (KeyError, ValueError): + timestamp = None + + mail_object = ses_event.get("mail", {}) + message_id = mail_object.get("messageId") # same as MessageId in SendRawEmail response + all_recipients = mail_object.get("destination", []) + + # Recover tags and metadata from custom headers + metadata = {} + tags = [] + for header in mail_object.get("headers", []): + name = header["name"].lower() + if name == "x-tag": + tags.append(header["value"]) + elif name == "x-metadata": + try: + metadata = json.loads(header["value"]) + except (ValueError, TypeError, KeyError): + pass + + common_props = dict( # AnymailTrackingEvent props for all recipients + esp_event=ses_event, + event_id=event_id, + message_id=message_id, + metadata=metadata, + tags=tags, + timestamp=timestamp, + ) + per_recipient_props = [ # generate individual events for each of these + dict(recipient=email_address) + for email_address in all_recipients + ] + + event_object = ses_event.get(ses_event_type.lower(), {}) # e.g., ses_event["bounce"] + + if ses_event_type == "Bounce": + common_props.update( + event_type=EventType.BOUNCED, + description="{bounceType}: {bounceSubType}".format(**event_object), + reject_reason=RejectReason.BOUNCED, + ) + per_recipient_props = [dict( + recipient=recipient["emailAddress"], + mta_response=recipient.get("diagnosticCode"), + ) for recipient in event_object["bouncedRecipients"]] + elif ses_event_type == "Complaint": + common_props.update( + event_type=EventType.COMPLAINED, + description=event_object.get("complaintFeedbackType"), + reject_reason=RejectReason.SPAM, + user_agent=event_object.get("userAgent"), + ) + per_recipient_props = [dict( + recipient=recipient["emailAddress"], + ) for recipient in event_object["complainedRecipients"]] + elif ses_event_type == "Delivery": + common_props.update( + event_type=EventType.DELIVERED, + mta_response=event_object.get("smtpResponse"), + ) + per_recipient_props = [dict( + recipient=recipient, + ) for recipient in event_object["recipients"]] + elif ses_event_type == "Send": + common_props.update( + event_type=EventType.SENT, + ) + elif ses_event_type == "Reject": + common_props.update( + event_type=EventType.REJECTED, + description=event_object["reason"], + reject_reason=RejectReason.BLOCKED, + ) + elif ses_event_type == "Open": + # SES doesn't report which recipient opened the message (it doesn't + # track them separately), so just report it for all_recipients + common_props.update( + event_type=EventType.OPENED, + user_agent=event_object.get("userAgent"), + ) + elif ses_event_type == "Click": + # SES doesn't report which recipient clicked the message (it doesn't + # track them separately), so just report it for all_recipients + common_props.update( + event_type=EventType.CLICKED, + user_agent=event_object.get("userAgent"), + click_url=event_object.get("link"), + ) + elif ses_event_type == "Rendering Failure": + event_object = ses_event["failure"] # rather than ses_event["rendering failure"] + common_props.update( + event_type=EventType.FAILED, + description=event_object["errorMessage"], + ) + else: + # Umm... new event type? + common_props.update( + event_type=EventType.UNKNOWN, + description="Unknown SES eventType '%s'" % ses_event_type, + ) + + return [ + # AnymailTrackingEvent(**common_props, **recipient_props) # Python 3.5+ (PEP-448 syntax) + AnymailTrackingEvent(**combine(common_props, recipient_props)) + for recipient_props in per_recipient_props + ] + + +class AmazonSESInboundWebhookView(AmazonSESBaseWebhookView): + """Handler for Amazon SES inbound notifications""" + + signal = inbound + + def esp_to_anymail_events(self, ses_event, sns_message): + ses_event_type = ses_event.get("notificationType") + if ses_event_type != "Received": + # This is not an inbound event + raise AnymailConfigurationError( + "You seem to have set an Amazon SES *sending* event or notification " + "to publish to an SNS Topic that posts to Anymail's *inbound* webhook URL. " + "(SNS TopicArn %s)" % sns_message.get("TopicArn")) + + receipt_object = ses_event.get("receipt", {}) + action_object = receipt_object.get("action", {}) + mail_object = ses_event.get("mail", {}) + + action_type = action_object.get("type") + if action_type == "SNS": + content = ses_event.get("content") + if action_object.get("encoding") == "BASE64": + content = b64decode(content.encode("ascii")) + message = AnymailInboundMessage.parse_raw_mime_bytes(content) + else: + message = AnymailInboundMessage.parse_raw_mime(content) + elif action_type == "S3": + # download message from s3 into memory, then parse + # (SNS has 15s limit for an http response; hope download doesn't take that long) + bucket_name = action_object["bucketName"] + object_key = action_object["objectKey"] + s3 = boto3.session.Session(**self.session_params).client("s3", **self.client_params) + content = io.BytesIO() + try: + s3.download_fileobj(bucket_name, object_key, content) + content.seek(0) + message = AnymailInboundMessage.parse_raw_mime_file(content) + except botocore.exceptions.ClientError as err: + # improve the botocore error message + raise AnymailBotoClientAPIError( + "Anymail AmazonSESInboundWebhookView couldn't download S3 object '{bucket_name}:{object_key}'" + "".format(bucket_name=bucket_name, object_key=object_key), + raised_from=err) + finally: + content.close() + else: + raise AnymailConfigurationError( + "Anymail's Amazon SES inbound webhook works only with 'SNS' or 'S3' receipt rule actions, " + "not SNS notifications for {action_type!s} actions. (SNS TopicArn {topic_arn!s})" + "".format(action_type=action_type, topic_arn=sns_message.get("TopicArn"))) + + message.envelope_sender = mail_object.get("source") # "the envelope MAIL FROM address" + try: + # "recipients that were matched by the active receipt rule" + message.envelope_recipient = receipt_object["recipients"][0] + except (KeyError, TypeError, IndexError): + pass + spam_status = receipt_object.get("spamVerdict", {}).get("status", "").upper() + message.spam_detected = {"PASS": False, "FAIL": True}.get(spam_status) # else None if unsure + + event_id = mail_object.get("messageId") # "unique ID assigned to the email by Amazon SES" + try: + timestamp = parse_datetime(mail_object["timestamp"]) # "time at which the email was received" + except (KeyError, ValueError): + timestamp = None + + return [AnymailInboundEvent( + event_type=EventType.INBOUND, + event_id=event_id, + message=message, + timestamp=timestamp, + esp_event=ses_event, + )] + + +class AnymailBotoClientAPIError(AnymailAPIError, botocore.exceptions.ClientError): + """An AnymailAPIError that is also a Boto ClientError""" + def __init__(self, *args, **kwargs): + raised_from = kwargs.pop('raised_from') + assert isinstance(raised_from, botocore.exceptions.ClientError) + assert len(kwargs) == 0 # can't support other kwargs + # init self as boto ClientError (which doesn't cooperatively subclass): + super(AnymailBotoClientAPIError, self).__init__( + error_response=raised_from.response, operation_name=raised_from.operation_name) + # emulate AnymailError init: + self.args = args + self.raised_from = raised_from diff --git a/docs/contributing.rst b/docs/contributing.rst index 60f8b75..5c3944c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -91,7 +91,7 @@ Or: .. code-block:: console - $ pip install mock sparkpost # install test dependencies + $ pip install mock boto3 sparkpost # install test dependencies $ python runtests.py ## this command can also run just a few test cases, e.g.: diff --git a/docs/esps/amazon_ses.rst b/docs/esps/amazon_ses.rst new file mode 100644 index 0000000..2b54dc1 --- /dev/null +++ b/docs/esps/amazon_ses.rst @@ -0,0 +1,710 @@ +.. _amazon-ses-backend: + +Amazon SES +========== + +Anymail integrates with `Amazon Simple Email Service`_ (SES) using the `Boto 3`_ +AWS SDK for Python, and includes sending, tracking, and inbound receiving capabilities. + +.. sidebar:: Alternatives + + At least two other packages offer Django integration with + Amazon SES: :pypi:`django-amazon-ses` and :pypi:`django-ses`. + Depending on your needs, one of them may be more appropriate than Anymail. + + +.. versionadded:: 2.1 + +.. _Amazon Simple Email Service: https://aws.amazon.com/ses/ +.. _Boto 3: https://boto3.readthedocs.io/en/stable/ + + +Installation +------------ + +You must ensure the :pypi:`boto3` package is installed to use Anymail's Amazon SES +backend. Either include the "amazon_ses" option when you install Anymail: + + .. code-block:: console + + $ pip install django-anymail[amazon_ses] + +or separately run `pip install boto3`. + +To send mail with Anymail's Amazon SES backend, set: + + .. code-block:: python + + EMAIL_BACKEND = "anymail.backends.amazon_ses.EmailBackend" + +in your settings.py. + +In addition, you must make sure boto3 is configured with AWS credentials having the +necessary :ref:`amazon-ses-iam-permissions`. +There are several ways to do this; see `Credentials`_ in the Boto docs for options. +Usually, an IAM role for EC2 instances, standard Boto environment variables, +or a shared AWS credentials file will be appropriate. For more complex cases, +use Anymail's :setting:`AMAZON_SES_CLIENT_PARAMS ` +setting to customize the Boto session. + + +.. _Credentials: https://boto3.readthedocs.io/en/stable/guide/configuration.html#configuring-credentials + + +.. _amazon-ses-quirks: + +Limitations and quirks +---------------------- + +**Hard throttling** + Like most ESPs, Amazon SES `throttles sending`_ for new customers. But unlike + most ESPs, SES does not queue and slowly release throttled messages. Instead, it + hard-fails the send API call. A strategy for :ref:`retrying errors ` + is required with any ESP; you're likely to run into it right away with Amazon SES. + +**Tags limitations** + Amazon SES's handling for tags is a bit different from other ESPs. + Anymail tries to provide a useful, portable default behavior for its + :attr:`~anymail.message.AnymailMessage.tags` feature. See :ref:`amazon-ses-tags` + below for more information and additional options. + +**Open and click tracking overrides** + Anymail's :attr:`~anymail.message.AnymailMessage.track_opens` and + :attr:`~anymail.message.AnymailMessage.track_clicks` are not supported. + Although Amazon SES *does* support open and click tracking, it doesn't offer + a simple mechanism to override the settings for individual messages. If you + need this feature, provide a custom ConfigurationSetName in Anymail's + :ref:`esp_extra `. + +**No delayed sending** + Amazon SES does not support :attr:`~anymail.message.AnymailMessage.send_at`. + +**No global send defaults for non-Anymail options** + With the Amazon SES backend, Anymail's :ref:`global send defaults ` + are only supported for Anymail's added message options (like + :attr:`~anymail.message.AnymailMessage.metadata` and + :attr:`~anymail.message.AnymailMessage.esp_extra`), not for standard EmailMessage + attributes like `bcc` or `from_email`. + +**Arbitrary alternative parts allowed** + Amazon SES is one of the few ESPs that *does* support sending arbitrary alternative + message parts (beyond just a single text/plain and text/html part). + +**Spoofed To header and multiple From emails allowed** + Amazon SES is one of the few ESPs that supports spoofing the :mailheader:`To` header + (see :ref:`message-headers`) and supplying multiple addresses in a message's `from_email`. + (Most ISPs consider these to be very strong spam signals, and using either them will almost + certainly prevent delivery of your mail.) + +**Template limitations** + Messages sent with templates have a number of additional limitations, such as not + supporting attachments. See :ref:`amazon-ses-templates` below. + + +.. _throttles sending: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/manage-sending-limits.html + +.. _amazon-ses-tags: + +Tags and metadata +----------------- + +Amazon SES provides two mechanisms for associating additional data with sent messages, +which Anymail uses to implement its :attr:`~anymail.message.AnymailMessage.tags` +and :attr:`~anymail.message.AnymailMessage.metadata` features: + +* **SES Message Tags** can be used for filtering or segmenting CloudWatch metrics and + dashboards, and are available to Kinesis Firehose streams. (See "How do message + tags work?" in the Amazon blog post `Introducing Sending Metrics`_.) + + By default, Anymail does *not* use SES Message Tags. They have strict limitations + on characters allowed, and are not consistently available to tracking webhooks. + (They may be included in `SES Event Publishing`_ but not `SES Notifications`_.) + +* **Custom Email Headers** are available to all SNS notifications (webhooks), but + not to CloudWatch or Kinesis. + + These are ordinary extension headers included in the sent message (and visible to + recipients who view the full headers). There are no restrictions on characters allowed. + +By default, Anymail uses only custom email headers. A message's +:attr:`~anymail.message.AnymailMessage.metadata` is sent JSON-encoded in a custom +:mailheader:`X-Metadata` header, and a message's :attr:`~anymail.message.AnymailMessage.tags` +are sent in custom :mailheader:`X-Tag` headers. Both are available in Anymail's +:ref:`tracking webhooks `. + +Because Anymail :attr:`~anymail.message.AnymailMessage.tags` are often used for +segmenting reports, Anymail has an option to easily send an Anymail tag +as an SES Message Tag that can be used in CloudWatch. Set the Anymail setting +:setting:`AMAZON_SES_MESSAGE_TAG_NAME ` +to the name of an SES Message Tag whose value will be the *single* Anymail tag +on the message. For example, with this setting: + + .. code-block:: python + + ANYMAIL = { + ... + "AMAZON_SES_MESSAGE_TAG_NAME": "Type", + } + +this send will appear in CloudWatch with the SES Message Tag `"Type": "Marketing"`: + + .. code-block:: python + + message = EmailMessage(...) + message.tags = ["Marketing"] + message.send() + +Anymail's :setting:`AMAZON_SES_MESSAGE_TAG_NAME ` +setting is disabled by default. If you use it, then only a single tag is supported, +and both the tag and the name must be limited to alphanumeric, hyphen, and underscore +characters. + +For more complex use cases, set the SES `Tags` parameter directly in Anymail's +:ref:`esp_extra `. See the example below. (Because custom headers do not +work with SES's SendBulkTemplatedEmail call, esp_extra Tags is the only way to attach +data to SES messages also using Anymail's :attr:`~anymail.message.AnymailMessage.template_id` +and :attr:`~anymail.message.AnymailMessage.merge_data` features.) + + +.. _Introducing Sending Metrics: + https://aws.amazon.com/blogs/ses/introducing-sending-metrics/ +.. _SES Event Publishing: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/monitor-using-event-publishing.html +.. _SES Notifications: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/monitor-sending-using-notifications.html + + +.. _amazon-ses-esp-extra: + +esp_extra support +----------------- + +To use Amazon SES features not directly supported by Anymail, you can +set a message's :attr:`~anymail.message.AnymailMessage.esp_extra` to +a `dict` that will be merged into the params for the `SendRawEmail`_ +or `SendBulkTemplatedEmail`_ SES API call. + +Example: + + .. code-block:: python + + message.esp_extra = { + # Override AMAZON_SES_CONFIGURATION_SET_NAME for this message + 'ConfigurationSetName': 'NoOpenOrClickTrackingConfigSet', + # Authorize a custom sender + 'SourceArn': 'arn:aws:ses:us-east-1:123456789012:identity/example.com', + # Set Amazon SES Message Tags + 'Tags': [ + # (Names and values must be A-Z a-z 0-9 - and _ only) + {'Name': 'UserID', 'Value': str(user_id)}, + {'Name': 'TestVariation', 'Value': 'Subject-Emoji-Trial-A'}, + ], + } + + +(You can also set `"esp_extra"` in Anymail's :ref:`global send defaults ` +to apply it to all messages.) + +.. _SendRawEmail: + https://docs.aws.amazon.com/ses/latest/APIReference/API_SendRawEmail.html + +.. _SendBulkTemplatedEmail: + https://docs.aws.amazon.com/ses/latest/APIReference/API_SendBulkTemplatedEmail.html + + +.. _amazon-ses-templates: + +Batch sending/merge and ESP templates +------------------------------------- + +Amazon SES offers :ref:`ESP stored templates ` +and :ref:`batch sending ` with per-recipient merge data. +See Amazon's `Sending personalized email`_ guide for more information. + +When you set a message's :attr:`~anymail.message.AnymailMessage.template_id` +to the name of one of your SES templates, Anymail will use the SES +`SendBulkTemplatedEmail`_ call to send template messages personalized with data +from Anymail's normalized :attr:`~anymail.message.AnymailMessage.merge_data` +and :attr:`~anymail.message.AnymailMessage.merge_global_data` +message attributes. + + .. code-block:: python + + message = EmailMessage( + from_email="shipping@example.com", + # you must omit subject and body (or set to None) with Amazon SES templates + to=["alice@example.com", "Bob "] + ) + message.template_id = "MyTemplateName" # Amazon SES TemplateName + message.merge_data = { + 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, + 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, + } + message.merge_global_data = { + 'ship_date': "May 15", + } + +Amazon's templated email APIs don't support several features available for regular email. +When :attr:`~anymail.message.AnymailMessage.template_id` is used: + +* Attachments are not supported +* Extra headers are not supported +* Overriding the template's subject or body is not supported +* Anymail's :attr:`~anymail.message.AnymailMessage.metadata` is not supported +* Anymail's :attr:`~anymail.message.AnymailMessage.tags` are only supported + with the :setting:`AMAZON_SES_MESSAGE_TAG_NAME ` + setting; only a single tag is allowed, and the tag is not directly available + to webhooks. (See :ref:`amazon-ses-tags` above.) + +.. _Sending personalized email: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-personalized-email-api.html + + +.. _amazon-ses-webhooks: + +Status tracking webhooks +------------------------ + +Anymail can provide normalized :ref:`status tracking ` notifications +for messages sent through Amazon SES. SES offers two (confusingly) similar kinds of +tracking, and Anymail supports both: + +* `SES Notifications`_ include delivered, bounced, and complained (spam) Anymail + :attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s. (Enabling these + notifications may allow you to disable SES "email feedback forwarding.") + +* `SES Event Publishing`_ also includes delivered, bounced and complained events, + as well as sent, rejected, opened, clicked, and (template rendering) failed. + +Both types of tracking events are delivered to Anymail's webhook URL through +Amazon Simple Notification Service (SNS) subscriptions. + +Amazon's naming here can be really confusing. We'll try to be clear about "SES Notifications" +vs. "SES Event Publishing" as the two different kinds of SES tracking events. +And then distinguish all of that from "SNS"---the publish/subscribe service +used to notify Anymail's tracking webhooks about *both* kinds of SES tracking event. + +To use Anymail's status tracking webhooks with Amazon SES: + +1. First, :ref:`configure Anymail webhooks ` and deploy your + Django project. (Deploying allows Anymail to confirm the SNS subscription for you + in step 3.) + +Then in Amazon's **Simple Notification Service** console: + +2. `Create an SNS Topic`_ to receive Amazon SES tracking events. + The exact topic name is up to you; choose something meaningful like *SES_Tracking_Events*. + +3. Subscribe Anymail's tracking webhook to the SNS Topic you just created. In the SNS + console, click into the topic from step 2, then click the "Create subscription" button. + For protocol choose HTTPS. For endpoint enter: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/amazon_ses/tracking/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret + * *yoursite.example.com* is your Django site + + Anymail will automatically confirm the SNS subscription. (For other options, see + :ref:`amazon-ses-confirm-sns-subscriptions` below.) + +Finally, switch to Amazon's **Simple Email Service** console: + +4. **If you want to use SES Notifications:** Follow Amazon's guide to + `configure SES notifications through SNS`_, using the SNS Topic you created above. + Choose any event types you want to receive. Be sure to choose "Include original headers" + if you need access to Anymail's :attr:`~anymail.message.AnymailMessage.metadata` or + :attr:`~anymail.message.AnymailMessage.tags` in your webhook handlers. + +5. **If you want to use SES Event Publishing:** + + a. Follow Amazon's guide to `create an SES "Configuration Set"`_. Name it something meaningful, + like *TrackingConfigSet.* + + b. Follow Amazon's guide to `add an SNS event destination for SES event publishing`_, using the + SNS Topic you created above. Choose any event types you want to receive. + + c. Update your Anymail settings to send using this Configuration Set by default: + + .. code-block:: python + + ANYMAIL = { + ... + "AMAZON_SES_CONFIGURATION_SET_NAME": "TrackingConfigSet", + } + +.. caution:: + + The delivery, bounce, and complaint event types are available in both SES Notifications + *and* SES Event Publishing. If you're using both, don't enable the same events in both + places, or you'll receive duplicate notifications with *different* + :attr:`~anymail.signals.AnymailTrackingEvent.event_id`\s. + + +Note that Amazon SES's open and click tracking does not distinguish individual recipients. +If you send a single message to multiple recipients, Anymail will call your tracking handler +with the "opened" or "clicked" event for *every* original recipient of the message, including +all to, cc and bcc addresses. (Amazon recommends avoiding multiple recipients with SES.) + +In your tracking signal receiver, the normalized AnymailTrackingEvent's +:attr:`~anymail.signals.AnymailTrackingEvent.esp_event` will be set to the +the parsed, top-level JSON event object from SES: either `SES Notification contents`_ +or `SES Event Publishing contents`_. (The two formats are nearly identical.) +You can use this to obtain SES Message Tags (see :ref:`amazon-ses-tags`) from +SES Event Publishing notifications: + +.. code-block:: python + + from anymail.signals import tracking + from django.dispatch import receiver + + @receiver(tracking) # add weak=False if inside some other function/class + def handle_tracking(sender, event, esp_name, **kwargs): + if esp_name == "Amazon SES": + try: + message_tags = { + name: values[0] + for name, values in event.esp_event["mail"]["tags"].items()} + except KeyError: + message_tags = None # SES Notification (not Event Publishing) event + print("Message %s to %s event %s: Message Tags %r" % ( + event.message_id, event.recipient, event.event_type, message_tags)) + + +Anymail does *not* currently check `SNS signature verification`_, because Amazon has not +released a standard way to do that in Python. Instead, Anymail relies on your +:setting:`WEBHOOK_SECRET ` to verify SNS notifications are from an +authorized source. + +.. _amazon-ses-sns-retry-policy: + +.. note:: + + Amazon SNS's default policy for handling HTTPS notification failures is to retry + three times, 20 seconds apart, and then drop the notification. That means + **if your webhook is ever offline for more than one minute, you may miss events.** + + For most uses, it probably makes sense to `configure an SNS retry policy`_ with more + attempts over a longer period. E.g., 20 retries ranging from 5 seconds minimum + to 600 seconds (5 minutes) maximum delay between attempts, with geometric backoff. + + Also, SNS does *not* guarantee notifications will be delivered to HTTPS subscribers + like Anymail webhooks. The longest SNS will ever keep retrying is one hour total. If you need + retries longer than that, or guaranteed delivery, you may need to implement your own queuing + mechanism with something like Celery or directly on Amazon Simple Queue Service (SQS). + + +.. _Create an SNS Topic: + https://docs.aws.amazon.com/sns/latest/dg/CreateTopic.html +.. _configure SES notifications through SNS: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/configure-sns-notifications.html +.. _create an SES "Configuration Set": + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-create-configuration-set.html +.. _add an SNS event destination for SES event publishing: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-add-event-destination-sns.html +.. _SES Notification contents: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html +.. _SES Event Publishing contents: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html +.. _SNS signature verification: + https://docs.aws.amazon.com/sns/latest/dg/SendMessageToHttp.verify.signature.html +.. _configure an SNS retry policy: + https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html + + +.. _amazon-ses-inbound: + +Inbound webhook +--------------- + +You can receive email through Amazon SES with Anymail's normalized :ref:`inbound ` +handling. See `Receiving email with Amazon SES`_ for background. + +Configuring Anymail's inbound webhook for Amazon SES is similar to installing the +:ref:`tracking webhook `. You must use a different SNS Topic +for inbound. + +To use Anymail's inbound webhook with Amazon SES: + +1. First, if you haven't already, :ref:`configure Anymail webhooks ` + and deploy your Django project. (Deploying allows Anymail to confirm the SNS subscription + for you in step 3.) + +2. `Create an SNS Topic`_ to receive Amazon SES inbound events. + The exact topic name is up to you; choose something meaningful like *SES_Inbound_Events*. + (If you are also using Anymail's tracking events, this must be a *different* SNS Topic.) + +3. Subscribe Anymail's inbound webhook to the SNS Topic you just created. In the SNS + console, click into the topic from step 2, then click the "Create subscription" button. + For protocol choose HTTPS. For endpoint enter: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/amazon_ses/inbound/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret + * *yoursite.example.com* is your Django site + + Anymail will automatically confirm the SNS subscription. (For other options, see + :ref:`amazon-ses-confirm-sns-subscriptions` below.) + +4. Next, follow Amazon's guide to `Setting up Amazon SES email receiving`_. + There are several steps. Come back here when you get to "Action Options" + in the last step, "Creating Receipt Rules." + +5. Anymail supports two SES receipt actions: S3 and SNS. (Both actually use SNS.) + You can choose either one: the SNS action is easier to set up, but the S3 action + allows you to receive larger messages and can be more robust. + (You can change at any time, but don't use both simultaneously.) + + * **For the SNS action:** choose the SNS Topic you created in step 2. Anymail will handle + either Base64 or UTF-8 encoding; use Base64 if you're not sure. + + * **For the S3 action:** choose or create any S3 bucket that Boto will be able to read. + (See :ref:`amazon-ses-iam-permissions`; *don't* use a world-readable bucket!) + "Object key prefix" is optional. Anymail does *not* currently support the + "Encrypt message" option. Finally, choose the SNS Topic you created in step 2. + +Amazon SES will likely deliver a test message to your Anymail inbound handler immediately +after you complete the last step. + +If you are using the S3 receipt action, note that Anymail does not delete the S3 object. +You can delete it from your code after successful processing, or set up S3 bucket policies +to automatically delete older messages. In your inbound handler, you can retrieve the S3 +object key by prepending the "object key prefix" (if any) from your receipt rule to Anymail's +:attr:`event.event_id `. + +Amazon SNS imposes a 15 second limit on all notifications. This includes time to download +the message (if you are using the S3 receipt action) and any processing in your +signal receiver. If the total takes longer, SNS will consider the notification failed +and will make several repeat attempts. To avoid problems, it's essential any lengthy +operations are offloaded to a background task. + +Amazon SNS's default retry policy times out after one minute of failed notifications. +If your webhook is ever unreachable for more than a minute, **you may miss inbound mail.** +You'll probably want to adjust your SNS topic settings to reduce the chances of that. +See the note about :ref:`retry policies ` in the tracking +webhooks discussion above. + +In your inbound signal receiver, the normalized AnymailTrackingEvent's +:attr:`~anymail.signals.AnymailTrackingEvent.esp_event` will be set to the +the parsed, top-level JSON object described in `SES Email Receiving contents`_. + +.. _Receiving email with Amazon SES: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/receiving-email.html +.. _Setting up Amazon SES email receiving: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/receiving-email-setting-up.html +.. _SES Email Receiving contents: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/receiving-email-notifications-contents.html + + +.. _amazon-ses-confirm-sns-subscriptions: + +Confirming SNS subscriptions +---------------------------- + +Amazon SNS requires HTTPS endpoints (webhooks) to confirm they actually want to subscribe +to an SNS Topic. See `Sending SNS messages to HTTPS endpoints`_ in the Amazon SNS docs +for more information. + +(This has nothing to do with verifying email identities in Amazon *SES*, +and is not related to email recipients confirming subscriptions to your content.) + +Anymail will automatically handle SNS endpoint confirmation for you, for both tracking and inbound +webhooks, if both: + +1. You have deployed your Django project with :ref:`Anymail webhooks enabled ` + and an Anymail :setting:`WEBHOOK_SECRET ` set, before subscribing the SNS Topic + to the webhook URL. + + (If you subscribed the SNS topic too early, you can re-send the confirmation request later + from the Subscriptions section of the Amazon SNS dashboard.) + +2. The SNS endpoint URL includes the correct Anymail :setting:`WEBHOOK_SECRET ` + as HTTP basic authentication. (Amazon SNS only allows this with https urls, not plain http.) + + Anymail requires a valid secret to ensure the subscription request is coming from you, not some other + AWS user. + +If you do not want Anymail to automatically confirm SNS subscriptions for its webhook URLs, set +:setting:`AMAZON_SES_AUTO_CONFIRM_SNS_SUBSCRIPTIONS ` +to `False` in your ANYMAIL settings. + +When auto-confirmation is disabled (or if Anymail receives an unexpected confirmation request), +it will raise an :exc:`AnymailWebhookValidationFailure`, which should show up in your Django error +logging. The error message will include the Token you can use to manually confirm the subscription +in the Amazon SNS dashboard or through the SNS API. + + +.. _Sending SNS messages to HTTPS endpoints: + https://docs.aws.amazon.com/sns/latest/dg/SendMessageToHttp.html + + +.. _amazon-ses-settings: + +Settings +-------- + +Additional Anymail settings for use with Amazon SES: + +.. setting:: ANYMAIL_AMAZON_SES_CLIENT_PARAMS + +.. rubric:: AMAZON_SES_CLIENT_PARAMS + +Optional. Additional `client parameters`_ Anymail should use to create the boto3 session client. Example: + + .. code-block:: python + + ANYMAIL = { + ... + "AMAZON_SES_CLIENT_PARAMS": { + # example: override normal Boto credentials specifically for Anymail + "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_FOR_ANYMAIL_SES"), + "aws_secret_access_key": os.getenv("AWS_SECRET_KEY_FOR_ANYMAIL_SES"), + "region_name": "us-west-2", + # override other default options + "config": { + "connect_timeout": 30, + "read_timeout": 30, + } + }, + } + +In most cases, it's better to let Boto obtain its own credentials through one of its other +mechanisms: an IAM role for EC2 instances, standard AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +and AWS_SESSION_TOKEN environment variables, or a shared AWS credentials file. + +.. _client parameters: + https://boto3.readthedocs.io/en/stable/reference/core/session.html#boto3.session.Session.client + + +.. setting:: ANYMAIL_AMAZON_SES_SESSION_PARAMS + +.. rubric:: AMAZON_SES_SESSION_PARAMS + +Optional. Additional `session parameters`_ Anymail should use to create the boto3 Session. Example: + + .. code-block:: python + + ANYMAIL = { + ... + "AMAZON_SES_SESSION_PARAMS": { + "profile_name": "anymail-testing", + }, + } + +.. _session parameters: + https://boto3.readthedocs.io/en/stable/reference/core/session.html#boto3.session.Session + + +.. setting:: ANYMAIL_AMAZON_SES_CONFIGURATION_SET_NAME + +.. rubric:: AMAZON_SES_CONFIGURATION_SET_NAME + +Optional. The name of an Amazon SES `Configuration Set`_ Anymail should use when sending messages. +The default is to send without any Configuration Set. Note that a Configuration Set is +required to receive SES Event Publishing tracking events. See :ref:`amazon-ses-webhooks` above. + +You can override this for individual messages with :ref:`esp_extra `. + +.. _Configuration Set: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/using-configuration-sets.html + + +.. setting:: ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME + +.. rubric:: AMAZON_SES_MESSAGE_TAG_NAME + +Optional, default `None`. The name of an Amazon SES "Message Tag" whose value is set +from a message's Anymail :attr:`~anymail.message.AnymailMessage.tags`. +See :ref:`amazon-ses-tags` above. + + +.. setting:: ANYMAIL_AMAZON_SES_AUTO_CONFIRM_SNS_SUBSCRIPTIONS + +.. rubric:: AMAZON_SES_AUTO_CONFIRM_SNS_SUBSCRIPTIONS + +Optional boolean, default `True`. Set to `False` to prevent Anymail webhooks from automatically +accepting Amazon SNS subscription confirmation requests. +See :ref:`amazon-ses-confirm-sns-subscriptions` above. + + +.. _amazon-ses-iam-permissions: + +IAM permissions +--------------- + +Anymail requires IAM permissions that will allow it to use these actions: + +* To send mail: + + * Ordinary (non-templated) sends: ``ses:SendRawEmail`` + * Template/merge sends: ``ses:SendBulkTemplatedEmail`` + +* To :ref:`automatically confirm ` + webhook SNS subscriptions: ``sns:ConfirmSubscription`` + +* For status tracking webhooks: no special permissions + +* To receive inbound mail: + + * With an "SNS action" receipt rule: no special permissions + * With an "S3 action" receipt rule: ``s3:GetObject`` on the S3 bucket + and prefix used (or S3 Access Control List read access for inbound + messages in that bucket) + + +This IAM policy covers all of those: + + .. code-block:: json + + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["ses:SendRawEmail", "ses:SendBulkTemplatedEmail"], + "Resource": "*" + }, { + "Effect": "Allow", + "Action": ["sns:ConfirmSubscription"], + "Resource": ["arn:aws:sns:*:*:*"] + }, { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::MY-PRIVATE-BUCKET-NAME/MY-INBOUND-PREFIX/*"] + }] + } + +Following the principle of `least privilege`_, you should omit permissions +for any features you aren't using, and you may want to add additional restrictions: + +* For Amazon SES sending, you can add conditions to restrict senders, recipients, times, + or other properties. See Amazon's `Controlling access to Amazon SES`_ guide. + +* For auto-confirming webhooks, you might limit the resource to SNS topics owned + by your AWS account, and/or specific topic names or patterns. E.g., + ``"arn:aws:sns:*:0000000000000000:SES_*_Events"`` (replacing the zeroes with + your numeric AWS account id). See Amazon's guide to `Amazon SNS ARNs`_. + +* For inbound S3 delivery, there are multiple ways to control S3 access and data + retention. See Amazon's `Managing access permissions to your Amazon S3 resources`_. + (And obviously, you should *never store incoming emails to a public bucket!*) + + Also, you may need to grant Amazon SES (but *not* Anymail) permission to *write* + to your inbound bucket. See Amazon's `Giving permissions to Amazon SES for email receiving`_. + +* For all operations, you can limit source IP, allowable times, user agent, and more. + (Requests from Anymail will include "django-anymail/*version*" along with Boto's user-agent.) + See Amazon's guide to `IAM condition context keys`_. + + +.. _least privilege: + https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#grant-least-privilege +.. _Controlling access to Amazon SES: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/control-user-access.html +.. _Amazon SNS ARNs: + https://docs.aws.amazon.com/sns/latest/dg/UsingIAMwithSNS.html#SNS_ARN_Format +.. _Managing access permissions to your Amazon S3 resources: + https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html +.. _Giving permissions to Amazon SES for email receiving: + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/receiving-email-permissions.html +.. _IAM condition context keys: + https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html diff --git a/docs/esps/index.rst b/docs/esps/index.rst index abb908a..fd5bda1 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -12,6 +12,7 @@ and notes about any quirks or limitations: .. toctree:: :maxdepth: 1 + amazon_ses mailgun mailjet mandrill @@ -30,33 +31,33 @@ The table below summarizes the Anymail features supported for each ESP. .. rst-class:: sticky-left -============================================ =========== ========== =========== ========== ========== ============ =========== -Email Service Provider |Mailgun| |Mailjet| |Mandrill| |Postmark| |SendGrid| |SendinBlue| |SparkPost| -============================================ =========== ========== =========== ========== ========== ============ =========== +============================================ ============ =========== ========== =========== ========== ========== ============ =========== +Email Service Provider |Amazon SES| |Mailgun| |Mailjet| |Mandrill| |Postmark| |SendGrid| |SendinBlue| |SparkPost| +============================================ ============ =========== ========== =========== ========== ========== ============ =========== .. rubric:: :ref:`Anymail send options ` -------------------------------------------------------------------------------------------------------------------------------------- -:attr:`~AnymailMessage.envelope_sender` Domain only Yes Domain only No No No Yes -:attr:`~AnymailMessage.metadata` Yes Yes Yes No Yes Yes Yes -:attr:`~AnymailMessage.send_at` Yes No Yes No Yes No Yes -:attr:`~AnymailMessage.tags` Yes Max 1 tag Yes Max 1 tag Yes Max 1 tag Max 1 tag -:attr:`~AnymailMessage.track_clicks` Yes Yes Yes Yes Yes No Yes -:attr:`~AnymailMessage.track_opens` Yes Yes Yes Yes Yes No Yes +--------------------------------------------------------------------------------------------------------------------------------------------------- +:attr:`~AnymailMessage.envelope_sender` Yes Domain only Yes Domain only No No No Yes +:attr:`~AnymailMessage.metadata` Yes Yes Yes Yes No Yes Yes Yes +:attr:`~AnymailMessage.send_at` No Yes No Yes No Yes No Yes +:attr:`~AnymailMessage.tags` Yes Yes Max 1 tag Yes Max 1 tag Yes Max 1 tag Max 1 tag +:attr:`~AnymailMessage.track_clicks` No Yes Yes Yes Yes Yes No Yes +:attr:`~AnymailMessage.track_opens` No Yes Yes Yes Yes Yes No Yes .. rubric:: :ref:`templates-and-merge` -------------------------------------------------------------------------------------------------------------------------------------- -:attr:`~AnymailMessage.template_id` No Yes Yes Yes Yes Yes Yes -:attr:`~AnymailMessage.merge_data` Yes Yes Yes No Yes No Yes -:attr:`~AnymailMessage.merge_global_data` (emulated) Yes Yes Yes Yes Yes Yes +--------------------------------------------------------------------------------------------------------------------------------------------------- +:attr:`~AnymailMessage.template_id` Yes No Yes Yes Yes Yes Yes Yes +:attr:`~AnymailMessage.merge_data` Yes Yes Yes Yes No Yes No Yes +:attr:`~AnymailMessage.merge_global_data` Yes (emulated) Yes Yes Yes Yes Yes Yes .. rubric:: :ref:`Status ` and :ref:`event tracking ` -------------------------------------------------------------------------------------------------------------------------------------- -:attr:`~AnymailMessage.anymail_status` Yes Yes Yes Yes Yes Yes Yes -|AnymailTrackingEvent| from webhooks Yes Yes Yes Yes Yes Yes Yes +--------------------------------------------------------------------------------------------------------------------------------------------------- +:attr:`~AnymailMessage.anymail_status` Yes Yes Yes Yes Yes Yes Yes Yes +|AnymailTrackingEvent| from webhooks Yes Yes Yes Yes Yes Yes Yes Yes .. rubric:: :ref:`Inbound handling ` -------------------------------------------------------------------------------------------------------------------------------------- -|AnymailInboundEvent| from webhooks Yes Yes Yes Yes Yes No Yes -============================================ =========== ========== =========== ========== ========== ============ =========== +--------------------------------------------------------------------------------------------------------------------------------------------------- +|AnymailInboundEvent| from webhooks Yes Yes Yes Yes Yes Yes No Yes +============================================ ============ =========== ========== =========== ========== ========== ============ =========== Trying to choose an ESP? Please **don't** start with this table. It's far more @@ -64,6 +65,7 @@ important to consider things like an ESP's deliverability stats, latency, uptime and support for developers. The *number* of extra features an ESP offers is almost meaningless. (And even specific features don't matter if you don't plan to use them.) +.. |Amazon SES| replace:: :ref:`amazon-ses-backend` .. |Mailgun| replace:: :ref:`mailgun-backend` .. |Mailjet| replace:: :ref:`mailjet-backend` .. |Mandrill| replace:: :ref:`mandrill-backend` diff --git a/docs/sending/django_email.rst b/docs/sending/django_email.rst index af1c665..2273e34 100644 --- a/docs/sending/django_email.rst +++ b/docs/sending/django_email.rst @@ -130,11 +130,11 @@ has special handling for certain headers. Anymail replicates its behavior for co the :mailheader:`Return-Path` at the recipient end. (Only if your ESP supports altering envelope sender, otherwise you'll get an :ref:`unsupported feature ` error.) -* If you supply a "To" header, you'll get an :ref:`unsupported feature ` error. +* If you supply a "To" header, you'll usually get an :ref:`unsupported feature ` error. With Django's SMTP EmailBackend, this can be used to show the recipient a :mailheader:`To` address that's different from the actual envelope recipients in the message's :class:`to ` list. Spoofing the :mailheader:`To` header like this - is popular with spammers, and none of Anymail's supported ESPs allow it. + is popular with spammers, and almost none of Anymail's supported ESPs allow it. .. versionchanged:: 2.0 diff --git a/setup.py b/setup.py index 2d09088..f95247f 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ setup( description='Django email integration for Mailgun, Mailjet, Postmark, SendGrid, SendinBlue, SparkPost ' 'and other transactional ESPs', keywords="Django, email, email backend, ESP, transactional mail, " - "Mailgun, Mailjet, Mandrill, Postmark, SendinBlue, SendGrid, SparkPost", + "Amazon SES, Mailgun, Mailjet, Mandrill, Postmark, SendinBlue, SendGrid, SparkPost", author="Mike Edmunds and Anymail contributors", author_email="medmunds@gmail.com", url="https://github.com/anymail/django-anymail", @@ -45,9 +45,9 @@ setup( zip_safe=False, install_requires=["django>=1.8", "requests>=2.4.3", "six"], extras_require={ - # This can be used if particular backends have unique dependencies - # (e.g., AWS-SES would want boto). + # This can be used if particular backends have unique dependencies. # For simplicity, requests is included in the base requirements. + "amazon_ses": ["boto3"], "mailgun": [], "mailjet": [], "mandrill": [], @@ -58,7 +58,7 @@ setup( }, include_package_data=True, test_suite="runtests.runtests", - tests_require=["mock", "sparkpost"], + tests_require=["mock", "boto3", "sparkpost"], classifiers=[ "Development Status :: 5 - Production/Stable", "Programming Language :: Python", diff --git a/tests/test_amazon_ses_backend.py b/tests/test_amazon_ses_backend.py new file mode 100644 index 0000000..1de828c --- /dev/null +++ b/tests/test_amazon_ses_backend.py @@ -0,0 +1,664 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import json +from datetime import datetime +from email.mime.application import MIMEApplication +from unittest import skipIf + +import botocore.config +import botocore.exceptions +import six +from django.core import mail +from django.core.mail import BadHeaderError +from django.test import SimpleTestCase +from django.test.utils import override_settings +from mock import ANY, patch + +from anymail.exceptions import AnymailAPIError, AnymailUnsupportedFeature +from anymail.inbound import AnymailInboundMessage +from anymail.message import attach_inline_image_file, AnymailMessage +from .utils import ( + AnymailTestMixin, SAMPLE_IMAGE_FILENAME, python_has_broken_mime_param_handling, + sample_image_content, sample_image_path) + + +@override_settings(EMAIL_BACKEND='anymail.backends.amazon_ses.EmailBackend') +class AmazonSESBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin): + """TestCase that uses the Amazon SES EmailBackend with a mocked boto3 client""" + + def setUp(self): + super(AmazonSESBackendMockAPITestCase, self).setUp() + + # Mock boto3.session.Session().client('ses').send_raw_email (and any other client operations) + # (We could also use botocore.stub.Stubber, but mock works well with our test structure) + self.patch_boto3_session = patch('anymail.backends.amazon_ses.boto3.session.Session', autospec=True) + self.mock_session = self.patch_boto3_session.start() # boto3.session.Session + self.addCleanup(self.patch_boto3_session.stop) + self.mock_client = self.mock_session.return_value.client # boto3.session.Session().client + self.mock_client_instance = self.mock_client.return_value # boto3.session.Session().client('ses', ...) + self.set_mock_response() + + # Simple message useful for many tests + self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', + 'from@example.com', ['to@example.com']) + + DEFAULT_SEND_RESPONSE = { + 'MessageId': '1111111111111111-bbbbbbbb-3333-7777-aaaa-eeeeeeeeeeee-000000', + 'ResponseMetadata': { + 'RequestId': 'aaaaaaaa-2222-1111-8888-bbbb3333bbbb', + 'HTTPStatusCode': 200, + 'HTTPHeaders': { + 'x-amzn-requestid': 'aaaaaaaa-2222-1111-8888-bbbb3333bbbb', + 'content-type': 'text/xml', + 'content-length': '338', + 'date': 'Sat, 17 Mar 2018 03:33:33 GMT' + }, + 'RetryAttempts': 0 + } + } + + def set_mock_response(self, response=None, operation_name="send_raw_email"): + mock_operation = getattr(self.mock_client_instance, operation_name) + mock_operation.return_value = response or self.DEFAULT_SEND_RESPONSE + return mock_operation.return_value + + def set_mock_failure(self, response, operation_name="send_raw_email"): + mock_operation = getattr(self.mock_client_instance, operation_name) + mock_operation.side_effect = botocore.exceptions.ClientError(response, operation_name=operation_name) + + def get_session_params(self): + if self.mock_session.call_args is None: + raise AssertionError("boto3 Session was not created") + (args, kwargs) = self.mock_session.call_args + if args: + raise AssertionError("boto3 Session created with unexpected positional args %r" % args) + return kwargs + + def get_client_params(self, service="ses"): + """Returns kwargs params passed to mock boto3 client constructor + + Fails test if boto3 client wasn't constructed with named service + """ + if self.mock_client.call_args is None: + raise AssertionError("boto3 client was not created") + (args, kwargs) = self.mock_client.call_args + if len(args) != 1: + raise AssertionError("boto3 client created with unexpected positional args %r" % args) + if args[0] != service: + raise AssertionError("boto3 client created with service %r, not %r" % (args[0], service)) + return kwargs + + def get_send_params(self, operation_name="send_raw_email"): + """Returns kwargs params passed to the mock send API. + + Fails test if API wasn't called. + """ + self.mock_client.assert_called_with("ses", config=ANY) + mock_operation = getattr(self.mock_client_instance, operation_name) + if mock_operation.call_args is None: + raise AssertionError("API was not called") + (args, kwargs) = mock_operation.call_args + return kwargs + + def get_sent_message(self): + """Returns a parsed version of the send_raw_email RawMessage.Data param""" + params = self.get_send_params(operation_name="send_raw_email") # (other operations don't have raw mime param) + raw_mime = params['RawMessage']['Data'] + parsed = AnymailInboundMessage.parse_raw_mime_bytes(raw_mime) + return parsed + + def assert_esp_not_called(self, msg=None, operation_name="send_raw_email"): + mock_operation = getattr(self.mock_client_instance, operation_name) + if mock_operation.called: + raise AssertionError(msg or "ESP API was called and shouldn't have been") + + +class AmazonSESBackendStandardEmailTests(AmazonSESBackendMockAPITestCase): + """Test backend support for Django standard email features""" + + def test_send_mail(self): + """Test basic API for simple send""" + mail.send_mail('Subject here', 'Here is the message.', + 'from@example.com', ['to@example.com'], fail_silently=False) + params = self.get_send_params() + # send_raw_email takes a fully-formatted MIME message. + # This is a simple (if inexact) way to check for expected headers and body: + raw_mime = params['RawMessage']['Data'] + self.assertIsInstance(raw_mime, six.binary_type) # SendRawEmail expects Data as bytes + self.assertIn(b"\nFrom: from@example.com\n", raw_mime) + self.assertIn(b"\nTo: to@example.com\n", raw_mime) + self.assertIn(b"\nSubject: Subject here\n", raw_mime) + self.assertIn(b"\n\nHere is the message", raw_mime) + + # Since the SES backend generates the MIME message using Django's + # EmailMessage.message().to_string(), there's not really a need + # to exhaustively test all the various standard email features. + # (EmailMessage.message() is well tested in the Django codebase.) + # Instead, just spot-check a few things... + + def test_non_ascii_headers(self): + self.message.subject = "Thử tin nhắn" # utf-8 in subject header + self.message.to = ['"Người nhận" '] # utf-8 in display name + self.message.cc = ["cc@thư.example.com"] # utf-8 in domain + self.message.send() + params = self.get_send_params() + raw_mime = params['RawMessage']['Data'] + # Non-ASCII headers must use MIME encoded-word syntax: + self.assertIn(b"\nSubject: =?utf-8?b?VGjhu60gdGluIG5o4bqvbg==?=\n", raw_mime) + # Non-ASCII display names as well: + self.assertIn(b"\nTo: =?utf-8?b?TmfGsOG7nWkgbmjhuq1u?= \n", raw_mime) + # Non-ASCII address domains must use Punycode: + self.assertIn(b"\nCc: cc@xn--th-e0a.example.com\n", raw_mime) + # SES doesn't support non-ASCII in the username@ part (RFC 6531 "SMTPUTF8" extension) + + @skipIf(python_has_broken_mime_param_handling(), + "This Python has a buggy email package that crashes on non-ASCII " + "characters in RFC2231-encoded MIME header parameters") + def test_attachments(self): + text_content = "• Item one\n• Item two\n• Item three" # those are \u2022 bullets ("\N{BULLET}") + self.message.attach(filename="Une pièce jointe.txt", # utf-8 chars in filename + content=text_content, mimetype="text/plain") + + # Should guess mimetype if not provided... + png_content = b"PNG\xb4 pretend this is the contents of a png file" + self.message.attach(filename="test.png", content=png_content) + + # Should work with a MIMEBase object (also tests no filename)... + pdf_content = b"PDF\xb4 pretend this is valid pdf params" + mimeattachment = MIMEApplication(pdf_content, 'pdf') # application/pdf + mimeattachment["Content-Disposition"] = "attachment" + self.message.attach(mimeattachment) + + self.message.send() + sent_message = self.get_sent_message() + attachments = sent_message.attachments + self.assertEqual(len(attachments), 3) + + self.assertEqual(attachments[0].get_content_type(), "text/plain") + self.assertEqual(attachments[0].get_filename(), "Une pièce jointe.txt") + self.assertEqual(attachments[0].get_param("charset"), "utf-8") + self.assertEqual(attachments[0].get_content_text(), text_content) + + self.assertEqual(attachments[1].get_content_type(), "image/png") + self.assertEqual(attachments[1].get_content_disposition(), "attachment") # not inline + self.assertEqual(attachments[1].get_filename(), "test.png") + self.assertEqual(attachments[1].get_content_bytes(), png_content) + + self.assertEqual(attachments[2].get_content_type(), "application/pdf") + self.assertIsNone(attachments[2].get_filename()) # no filename specified + self.assertEqual(attachments[2].get_content_bytes(), pdf_content) + + def test_embedded_images(self): + image_filename = SAMPLE_IMAGE_FILENAME + image_path = sample_image_path(image_filename) + image_data = sample_image_content(image_filename) + + cid = attach_inline_image_file(self.message, image_path, domain="example.com") + html_content = '

This has an inline image.

' % cid + self.message.attach_alternative(html_content, "text/html") + + self.message.send() + sent_message = self.get_sent_message() + + self.assertEqual(sent_message.html, html_content) + + inlines = sent_message.inline_attachments + self.assertEqual(len(inlines), 1) + self.assertEqual(inlines[cid].get_content_type(), "image/png") + self.assertEqual(inlines[cid].get_filename(), image_filename) + self.assertEqual(inlines[cid].get_content_bytes(), image_data) + + # Make sure neither the html nor the inline image is treated as an attachment: + params = self.get_send_params() + raw_mime = params['RawMessage']['Data'] + self.assertNotIn(b'\nContent-Disposition: attachment', raw_mime) + + def test_multiple_html_alternatives(self): + # Multiple alternatives *are* allowed + self.message.attach_alternative("

First html is OK

", "text/html") + self.message.attach_alternative("

And so is second

", "text/html") + self.message.send() + params = self.get_send_params() + raw_mime = params['RawMessage']['Data'] + # just check the alternative smade it into the message (assume that Django knows how to format them properly) + self.assertIn(b'\n\n

First html is OK

\n', raw_mime) + self.assertIn(b'\n\n

And so is second

\n', raw_mime) + + def test_alternative(self): + # Non-HTML alternatives *are* allowed + self.message.attach_alternative('{"is": "allowed"}', "application/json") + self.message.send() + params = self.get_send_params() + raw_mime = params['RawMessage']['Data'] + # just check the alternative made it into the message (assume that Django knows how to format it properly) + self.assertIn(b"\nContent-Type: application/json\n", raw_mime) + + def test_multiple_from(self): + # Amazon allows multiple addresses in the From header, but must specify which is Source + self.message.from_email = "from1@example.com, from2@example.com" + self.message.send() + params = self.get_send_params() + raw_mime = params['RawMessage']['Data'] + self.assertIn(b"\nFrom: from1@example.com, from2@example.com\n", raw_mime) + self.assertEqual(params['Source'], "from1@example.com") + + def test_commas_in_subject(self): + """Anymail works around a Python 2 email header bug that adds unwanted spaces after commas in long subjects""" + self.message.subject = "100,000,000 isn't a number you'd really want to break up in this email subject, right?" + self.message.send() + sent_message = self.get_sent_message() + self.assertEqual(sent_message["Subject"], self.message.subject) + + def test_api_failure(self): + error_response = { + 'Error': { + 'Type': 'Sender', + 'Code': 'MessageRejected', + 'Message': 'Email address is not verified. The following identities failed ' + 'the check in region US-EAST-1: to@example.com' + }, + 'ResponseMetadata': { + 'RequestId': 'aaaaaaaa-2222-1111-8888-bbbb3333bbbb', + 'HTTPStatusCode': 400, + 'HTTPHeaders': { + 'x-amzn-requestid': 'aaaaaaaa-2222-1111-8888-bbbb3333bbbb', + 'content-type': 'text/xml', + 'content-length': '277', + 'date': 'Sat, 17 Mar 2018 04:44:44 GMT' + }, + 'RetryAttempts': 0 + } + } + + self.set_mock_failure(error_response) + with self.assertRaises(AnymailAPIError) as cm: + self.message.send() + err = cm.exception + # AWS error is included in Anymail message: + self.assertIn('Email address is not verified. The following identities failed ' + 'the check in region US-EAST-1: to@example.com', + str(err)) + # Raw AWS response is available on the exception: + self.assertEqual(err.response, error_response) + + def test_api_failure_fail_silently(self): + # Make sure fail_silently is respected + self.set_mock_failure({ + 'Error': {'Type': 'Sender', 'Code': 'InvalidParameterValue', 'Message': 'That is not allowed'}}) + sent = self.message.send(fail_silently=True) + self.assertEqual(sent, 0) + + def test_prevents_header_injection(self): + # Since we build the raw MIME message, we're responsible for preventing header injection. + # django.core.mail.EmailMessage.message() implements most of that (for the SMTP backend); + # spot check some likely cases just to be sure... + with self.assertRaises(BadHeaderError): + mail.send_mail('Subject\r\ninjected', 'Body', 'from@example.com', ['to@example.com']) + with self.assertRaises(BadHeaderError): + mail.send_mail('Subject', 'Body', '"Display-Name\nInjected" ', ['to@example.com']) + with self.assertRaises(BadHeaderError): + mail.send_mail('Subject', 'Body', 'from@example.com', ['"Display-Name\rInjected" ']) + with self.assertRaises(BadHeaderError): + mail.EmailMessage('Subject', 'Body', 'from@example.com', ['to@example.com'], + headers={"X-Header": "custom header value\r\ninjected"}).send() + + +class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase): + """Test backend support for Anymail added features""" + + def test_envelope_sender(self): + self.message.envelope_sender = "bounce-handler@bounces.example.com" + self.message.send() + params = self.get_send_params() + self.assertEqual(params['Source'], "bounce-handler@bounces.example.com") + + def test_spoofed_to(self): + # Amazon SES is one of the few ESPs that actually permits the To header + # to differ from the envelope recipient... + self.message.to = ["Envelope "] + self.message.extra_headers["To"] = "Spoofed " + self.message.send() + params = self.get_send_params() + raw_mime = params['RawMessage']['Data'] + self.assertEqual(params['Destinations'], ["envelope-to@example.com"]) + self.assertIn(b"\nTo: Spoofed \n", raw_mime) + self.assertNotIn(b"envelope-to@example.com", raw_mime) + + def test_metadata(self): + # (that \n is a header-injection test) + self.message.metadata = { + 'User ID': 12345, 'items': 'Correct horse,Battery,\nStaple', 'Cart-Total': '22.70'} + self.message.send() + + # Metadata is passed as JSON in a message header field: + sent_message = self.get_sent_message() + self.assertJSONEqual( + sent_message["X-Metadata"], + '{"User ID": 12345, "items": "Correct horse,Battery,\\nStaple", "Cart-Total": "22.70"}') + + def test_send_at(self): + # Amazon SES does not support delayed sending + self.message.send_at = datetime(2016, 3, 4, 5, 6, 7) + with self.assertRaisesMessage(AnymailUnsupportedFeature, "send_at"): + self.message.send() + + def test_tags(self): + self.message.tags = ["Transactional", "Cohort 12/2017"] + self.message.send() + + # Tags are added as multiple X-Tag message headers: + sent_message = self.get_sent_message() + self.assertCountEqual(sent_message.get_all("X-Tag"), + ["Transactional", "Cohort 12/2017"]) + + # Tags are *not* by default used as Amazon SES "Message Tags": + params = self.get_send_params() + self.assertNotIn("Tags", params) + + @override_settings(ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign") + def test_amazon_message_tags(self): + """The Anymail AMAZON_SES_MESSAGE_TAG_NAME setting enables a single Message Tag""" + self.message.tags = ["Welcome"] + self.message.send() + params = self.get_send_params() + self.assertEqual(params['Tags'], [{"Name": "Campaign", "Value": "Welcome"}]) + + # Multiple Anymail tags are not supported when using this feature + self.message.tags = ["Welcome", "Variation_A"] + with self.assertRaisesMessage( + AnymailUnsupportedFeature, + "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting" + ): + self.message.send() + + def test_tracking(self): + # Amazon SES doesn't support overriding click/open-tracking settings + # on individual messages through any standard API params. + # (You _can_ use a ConfigurationSet to control this; see esp_extra below.) + self.message.track_clicks = True + with self.assertRaisesMessage(AnymailUnsupportedFeature, "track_clicks"): + self.message.send() + delattr(self.message, 'track_clicks') + + self.message.track_opens = True + with self.assertRaisesMessage(AnymailUnsupportedFeature, "track_opens"): + self.message.send() + + def test_merge_data(self): + # Amazon SES only supports merging when using templates (see below) + self.message.merge_data = {} + with self.assertRaisesMessage(AnymailUnsupportedFeature, "merge_data without template_id"): + self.message.send() + delattr(self.message, 'merge_data') + + self.message.merge_global_data = {'group': "Users", 'site': "ExampleCo"} + with self.assertRaisesMessage(AnymailUnsupportedFeature, "global_merge_data without template_id"): + self.message.send() + + @override_settings(ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign") # only way to use tags with template_id + def test_template(self): + """With template_id, Anymail switches to SES SendBulkTemplatedEmail""" + # SendBulkTemplatedEmail uses a completely different API call and payload structure, + # so this re-tests a bunch of Anymail features that were handled differently above. + # (See test_amazon_ses_integration for a more realistic template example.) + raw_response = { + "Status": [ + {"Status": "Success", "MessageId": "1111111111111111-bbbbbbbb-3333-7777"}, + {"Status": "AccountThrottled"}, + ], + "ResponseMetadata": self.DEFAULT_SEND_RESPONSE["ResponseMetadata"] + } + self.set_mock_response(raw_response, operation_name="send_bulk_templated_email") + message = AnymailMessage( + template_id="welcome_template", + from_email='"Example, Inc." ', + to=['alice@example.com', '罗伯特 '], + cc=['cc@example.com'], + reply_to=['reply1@example.com', 'Reply 2 '], + merge_data={ + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, # and leave group undefined + 'nobody@example.com': {'name': "Not a recipient for this message"}, + }, + merge_global_data={'group': "Users", 'site': "ExampleCo"}, + tags=["WelcomeVariantA"], # (only with AMAZON_SES_MESSAGE_TAG_NAME when using template) + envelope_sender="bounces@example.com", + esp_extra={'SourceArn': "arn:aws:ses:us-east-1:123456789012:identity/example.com"}, + ) + message.send() + + self.assert_esp_not_called(operation_name="send_raw_email") # templates use a different API call... + params = self.get_send_params(operation_name="send_bulk_templated_email") + self.assertEqual(params['Template'], "welcome_template") + self.assertEqual(params['Source'], '"Example, Inc." ') + destinations = params['Destinations'] + self.assertEqual(len(destinations), 2) + self.assertEqual(destinations[0]['Destination'], + {"ToAddresses": ['alice@example.com'], + "CcAddresses": ['cc@example.com']}) + self.assertEqual(json.loads(destinations[0]['ReplacementTemplateData']), + {'name': "Alice", 'group': "Developers"}) + self.assertEqual(destinations[1]['Destination'], + {"ToAddresses": ['=?utf-8?b?572X5Lyv54m5?= '], # SES requires RFC2047 + "CcAddresses": ['cc@example.com']}) + self.assertEqual(json.loads(destinations[1]['ReplacementTemplateData']), + {'name': "Bob"}) + self.assertEqual(json.loads(params['DefaultTemplateData']), + {'group': "Users", 'site': "ExampleCo"}) + self.assertEqual(params['ReplyToAddresses'], + ['reply1@example.com', 'Reply 2 ']) + self.assertEqual(params['DefaultTags'], [{"Name": "Campaign", "Value": "WelcomeVariantA"}]) + self.assertEqual(params['ReturnPath'], "bounces@example.com") + self.assertEqual(params['SourceArn'], "arn:aws:ses:us-east-1:123456789012:identity/example.com") # esp_extra + + self.assertEqual(message.anymail_status.status, {"queued", "failed"}) + self.assertEqual(message.anymail_status.message_id, + {"1111111111111111-bbbbbbbb-3333-7777", None}) # different for each recipient + self.assertEqual(message.anymail_status.recipients["alice@example.com"].status, "queued") + self.assertEqual(message.anymail_status.recipients["bob@example.com"].status, "failed") + self.assertEqual(message.anymail_status.recipients["alice@example.com"].message_id, + "1111111111111111-bbbbbbbb-3333-7777") + self.assertIsNone(message.anymail_status.recipients["bob@example.com"].message_id) + self.assertEqual(message.anymail_status.esp_response, raw_response) + + def test_template_unsupported(self): + """A lot of options are not compatible with SendBulkTemplatedEmail""" + message = AnymailMessage(template_id="welcome_template", to=['to@example.com']) + + message.subject = "nope, can't change template subject" + with self.assertRaisesMessage(AnymailUnsupportedFeature, "overriding template subject"): + message.send() + message.subject = None + + message.body = "nope, can't change text body" + with self.assertRaisesMessage(AnymailUnsupportedFeature, "overriding template body content"): + message.send() + message.content_subtype = "html" + with self.assertRaisesMessage(AnymailUnsupportedFeature, "overriding template body content"): + message.send() + message.body = None + + message.attach("attachment.txt", "this is an attachment", "text/plain") + with self.assertRaisesMessage(AnymailUnsupportedFeature, "attachments with template"): + message.send() + message.attachments = [] + + message.extra_headers = {"X-Custom": "header"} + with self.assertRaisesMessage(AnymailUnsupportedFeature, "extra_headers with template"): + message.send() + message.extra_headers = {} + + message.metadata = {"meta": "data"} + with self.assertRaisesMessage(AnymailUnsupportedFeature, "metadata with template"): + message.send() + message.metadata = None + + message.tags = ["tag 1", "tag 2"] + with self.assertRaisesMessage(AnymailUnsupportedFeature, "tags with template"): + message.send() + message.tags = None + + def test_send_anymail_message_without_template(self): + # Make sure SendRawEmail is used for non-template_id messages + message = AnymailMessage(from_email="from@example.com", to=["to@example.com"], subject="subject") + message.send() + self.assert_esp_not_called(operation_name="send_bulk_templated_email") + self.get_send_params(operation_name="send_raw_email") # fails if send_raw_email not called + + def test_default_omits_options(self): + """Make sure by default we don't send any ESP-specific options. + + Options not specified by the caller should be omitted entirely from + the API call (*not* sent as False or empty). This ensures + that your ESP account settings apply by default. + """ + self.message.send() + params = self.get_send_params() + self.assertNotIn('ConfigurationSetName', params) + self.assertNotIn('DefaultTags', params) + self.assertNotIn('DefaultTemplateData', params) + self.assertNotIn('Destinations', params) + self.assertNotIn('FromArn', params) + self.assertNotIn('Message', params) + self.assertNotIn('ReplyToAddresses', params) + self.assertNotIn('ReturnPath', params) + self.assertNotIn('ReturnPathArn', params) + self.assertNotIn('Source', params) + self.assertNotIn('SourceArn', params) + self.assertNotIn('Tags', params) + self.assertNotIn('Template', params) + self.assertNotIn('TemplateArn', params) + self.assertNotIn('TemplateData', params) + + sent_message = self.get_sent_message() + self.assertNotIn("X-Metadata", sent_message) # custom headers not added if not needed + self.assertNotIn("X-Tag", sent_message) + + def test_esp_extra(self): + # Values in esp_extra are merged into the Amazon SES SendRawEmail parameters + self.message.esp_extra = { + # E.g., if you've set up a configuration set that disables open/click tracking: + 'ConfigurationSetName': 'NoTrackingConfigurationSet', + } + self.message.send() + params = self.get_send_params() + self.assertEqual(params['ConfigurationSetName'], 'NoTrackingConfigurationSet') + + def test_send_attaches_anymail_status(self): + """The anymail_status should be attached to the message when it is sent """ + msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['to1@example.com'],) + sent = msg.send() + self.assertEqual(sent, 1) + self.assertEqual(msg.anymail_status.status, {'queued'}) + self.assertEqual(msg.anymail_status.message_id, + '1111111111111111-bbbbbbbb-3333-7777-aaaa-eeeeeeeeeeee-000000') + self.assertEqual(msg.anymail_status.recipients['to1@example.com'].status, 'queued') + self.assertEqual(msg.anymail_status.recipients['to1@example.com'].message_id, + '1111111111111111-bbbbbbbb-3333-7777-aaaa-eeeeeeeeeeee-000000') + self.assertEqual(msg.anymail_status.esp_response, self.DEFAULT_SEND_RESPONSE) + + # Amazon SES doesn't report rejected addresses at send time in a form that can be + # distinguished from other API errors. If SES rejects *any* recipient you'll get + # an AnymailAPIError, and the message won't be sent to *all* recipients. + + # noinspection PyUnresolvedReferences + def test_send_unparsable_response(self): + """If the send succeeds, but result is unexpected format, should raise an API exception""" + response_content = {'wrong': 'format'} + self.set_mock_response(response_content) + with self.assertRaisesMessage(AnymailAPIError, "parsing Amazon SES send result"): + self.message.send() + self.assertIsNone(self.message.anymail_status.status) + self.assertIsNone(self.message.anymail_status.message_id) + self.assertEqual(self.message.anymail_status.recipients, {}) + self.assertEqual(self.message.anymail_status.esp_response, response_content) + + +class AmazonSESBackendConfigurationTests(AmazonSESBackendMockAPITestCase): + """Test configuration options""" + + def test_boto_default_config(self): + """By default, boto3 gets credentials from the environment or its config files + + See http://boto3.readthedocs.io/en/stable/guide/configuration.html + """ + self.message.send() + + session_params = self.get_session_params() + self.assertEqual(session_params, {}) # no additional params passed to boto3.session.Session() + + client_params = self.get_client_params() + config = client_params.pop("config") # Anymail adds a default config, which doesn't support == + self.assertEqual(client_params, {}) # no additional params passed to session.client('ses') + self.assertRegex(config.user_agent_extra, r'django-anymail/\d(\.\w+){1,}-amazon-ses') + + @override_settings(ANYMAIL={ + "AMAZON_SES_CLIENT_PARAMS": { + # Example for testing; it's not a good idea to hardcode credentials in your code + "aws_access_key_id": "test-access-key-id", # safer: `os.getenv("MY_SPECIAL_AWS_KEY_ID")` + "aws_secret_access_key": "test-secret-access-key", + "region_name": "ap-northeast-1", + # config can be given as dict of botocore.config.Config params + "config": { + "read_timeout": 30, + "retries": {"max_attempts": 2}, + }, + } + }) + def test_client_params_in_setting(self): + """The Anymail AMAZON_SES_CLIENT_PARAMS setting specifies boto3 session.client() params for Anymail""" + self.message.send() + client_params = self.get_client_params() + config = client_params.pop("config") # botocore.config.Config doesn't support == + self.assertEqual(client_params, { + "aws_access_key_id": "test-access-key-id", + "aws_secret_access_key": "test-secret-access-key", + "region_name": "ap-northeast-1", + }) + self.assertEqual(config.read_timeout, 30) + self.assertEqual(config.retries, {"max_attempts": 2}) + + def test_client_params_in_connection_init(self): + """You can also supply credentials specifically for a particular EmailBackend connection instance""" + boto_config = botocore.config.Config(connect_timeout=30) + conn = mail.get_connection( + 'anymail.backends.amazon_ses.EmailBackend', + client_params={"aws_session_token": "test-session-token", "config": boto_config}) + conn.send_messages([self.message]) + + client_params = self.get_client_params() + config = client_params.pop("config") # botocore.config.Config doesn't support == + self.assertEqual(client_params, {"aws_session_token": "test-session-token"}) + self.assertEqual(config.connect_timeout, 30) + + @override_settings(ANYMAIL={ + "AMAZON_SES_SESSION_PARAMS": { + "profile_name": "anymail-testing" + } + }) + def test_session_params_in_setting(self): + """The Anymail AMAZON_SES_SESSION_PARAMS setting specifies boto3.session.Session() params for Anymail""" + self.message.send() + + session_params = self.get_session_params() + self.assertEqual(session_params, {"profile_name": "anymail-testing"}) + + client_params = self.get_client_params() + client_params.pop("config") # Anymail adds a default config, which doesn't support == + self.assertEqual(client_params, {}) # no additional params passed to session.client('ses') + + @override_settings(ANYMAIL={ + "AMAZON_SES_CONFIGURATION_SET_NAME": "MyConfigurationSet" + }) + def test_config_set_setting(self): + """You can supply a default ConfigurationSetName""" + self.message.send() + params = self.get_send_params() + self.assertEqual(params["ConfigurationSetName"], "MyConfigurationSet") + + # override on individual message using esp_extra + self.message.esp_extra = {"ConfigurationSetName": "CustomConfigurationSet"} + self.message.send() + params = self.get_send_params() + self.assertEqual(params["ConfigurationSetName"], "CustomConfigurationSet") diff --git a/tests/test_amazon_ses_inbound.py b/tests/test_amazon_ses_inbound.py new file mode 100644 index 0000000..cff6283 --- /dev/null +++ b/tests/test_amazon_ses_inbound.py @@ -0,0 +1,311 @@ +from __future__ import unicode_literals + +import json +from base64 import b64encode +from datetime import datetime +from textwrap import dedent + +import botocore.exceptions +from django.utils.timezone import utc +from mock import ANY, patch + +from anymail.exceptions import AnymailAPIError, AnymailConfigurationError +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.amazon_ses import AmazonSESInboundWebhookView + +from .test_amazon_ses_webhooks import AmazonSESWebhookTestsMixin +from .webhook_cases import WebhookTestCase + + +class AmazonSESInboundTests(WebhookTestCase, AmazonSESWebhookTestsMixin): + + def setUp(self): + super(AmazonSESInboundTests, self).setUp() + # Mock boto3.session.Session().client('s3').download_fileobj + # (We could also use botocore.stub.Stubber, but mock works well with our test structure) + self.patch_boto3_session = patch('anymail.webhooks.amazon_ses.boto3.session.Session', autospec=True) + self.mock_session = self.patch_boto3_session.start() # boto3.session.Session + self.addCleanup(self.patch_boto3_session.stop) + + def mock_download_fileobj(bucket, key, fileobj): + fileobj.write(self.mock_s3_downloadables[bucket][key]) + + self.mock_s3_downloadables = {} # bucket: key: bytes + self.mock_client = self.mock_session.return_value.client # boto3.session.Session().client + self.mock_s3 = self.mock_client.return_value # boto3.session.Session().client('s3', ...) + self.mock_s3.download_fileobj.side_effect = mock_download_fileobj + + TEST_MIME_MESSAGE = dedent("""\ + Return-Path: + Received: from mail.example.org by inbound-smtp.us-east-1.amazonaws.com... + MIME-Version: 1.0 + Received: by 10.1.1.1 with HTTP; Fri, 30 Mar 2018 10:21:49 -0700 (PDT) + From: "Sender, Inc." + Date: Fri, 30 Mar 2018 10:21:50 -0700 + Message-ID: + Subject: Test inbound message + To: Recipient , someone-else@example.org + Content-Type: multipart/alternative; boundary="94eb2c05e174adb140055b6339c5" + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/plain; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + + It's a body=E2=80=A6 + + --94eb2c05e174adb140055b6339c5 + Content-Type: text/html; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + +
It's a body=E2=80=A6
+ + --94eb2c05e174adb140055b6339c5-- + """).replace("\n", "\r\n") + + def test_inbound_sns_utf8(self): + raw_ses_event = { + "notificationType": "Received", + "mail": { + "timestamp": "2018-03-30T17:21:51.636Z", + "source": "envelope-from@example.org", + "messageId": "jili9m351il3gkburn7o2f0u6788stij94c8ld01", # assigned by Amazon SES + "destination": ["inbound@example.com", "someone-else@example.org"], + "headersTruncated": False, + "headers": [ + # (omitting a few headers that Amazon SES adds on receipt) + {"name": "Return-Path", "value": ""}, + {"name": "Received", "value": "from mail.example.org by inbound-smtp.us-east-1.amazonaws.com..."}, + {"name": "MIME-Version", "value": "1.0"}, + {"name": "Received", "value": "by 10.1.1.1 with HTTP; Fri, 30 Mar 2018 10:21:49 -0700 (PDT)"}, + {"name": "From", "value": '"Sender, Inc." '}, + {"name": "Date", "value": "Fri, 30 Mar 2018 10:21:50 -0700"}, + {"name": "Message-ID", "value": ""}, + {"name": "Subject", "value": "Test inbound message"}, + {"name": "To", "value": "Recipient , someone-else@example.org"}, + {"name": "Content-Type", "value": 'multipart/alternative; boundary="94eb2c05e174adb140055b6339c5"'}, + ], + "commonHeaders": { + "returnPath": "bounce-handler@mail.example.org", + "from": ['"Sender, Inc." '], + "date": "Fri, 30 Mar 2018 10:21:50 -0700", + "to": ["Recipient ", "someone-else@example.org"], + "messageId": "", + "subject": "Test inbound message", + }, + }, + "receipt": { + "timestamp": "2018-03-30T17:21:51.636Z", + "processingTimeMillis": 357, + "recipients": ["inbound@example.com"], + "spamVerdict": {"status": "PASS"}, + "virusVerdict": {"status": "PASS"}, + "spfVerdict": {"status": "PASS"}, + "dkimVerdict": {"status": "PASS"}, + "dmarcVerdict": {"status": "PASS"}, + "action": { + "type": "SNS", + "topicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "encoding": "UTF8", + }, + }, + "content": self.TEST_MIME_MESSAGE, + } + + raw_sns_message = { + "Type": "Notification", + "MessageId": "8f6dee70-c885-558a-be7d-bd48bbf5335e", + "TopicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "Subject": "Amazon SES Email Receipt Notification", + "Message": json.dumps(raw_ses_event), + "Timestamp": "2018-03-30T17:17:36.516Z", + "SignatureVersion": "1", + "Signature": "EXAMPLE_SIGNATURE==", + "SigningCertURL": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-12345abcde.pem", + "UnsubscribeURL": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn...", + } + + response = self.post_from_sns('/anymail/amazon_ses/inbound/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=AmazonSESInboundWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertEqual(event.timestamp, datetime(2018, 3, 30, 17, 21, 51, microsecond=636000, tzinfo=utc)) + self.assertEqual(event.event_id, "jili9m351il3gkburn7o2f0u6788stij94c8ld01") + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_ses_event) + + message = event.message + self.assertIsInstance(message, AnymailInboundMessage) + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'inbound@example.com') + self.assertEqual(str(message.from_email), '"Sender, Inc." ') + self.assertEqual([str(to) for to in message.to], + ['Recipient ', 'someone-else@example.org']) + self.assertEqual(message.subject, 'Test inbound message') + self.assertEqual(message.text, "It's a body\N{HORIZONTAL ELLIPSIS}\r\n") + self.assertEqual(message.html, """
It's a body\N{HORIZONTAL ELLIPSIS}
\r\n""") + self.assertIs(message.spam_detected, False) + + def test_inbound_sns_base64(self): + """Should handle 'Base 64' content option on received email SNS action""" + raw_ses_event = { + # (omitting some fields that aren't used by Anymail) + "notificationType": "Received", + "mail": { + "source": "envelope-from@example.org", + "timestamp": "2018-03-30T17:21:51.636Z", + "messageId": "jili9m351il3gkburn7o2f0u6788stij94c8ld01", # assigned by Amazon SES + "destination": ["inbound@example.com", "someone-else@example.org"], + }, + "receipt": { + "recipients": ["inbound@example.com"], + "action": { + "type": "SNS", + "topicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "encoding": "BASE64", + }, + "spamVerdict": {"status": "FAIL"}, + }, + "content": b64encode(self.TEST_MIME_MESSAGE.encode('ascii')).decode('ascii'), + } + + raw_sns_message = { + "Type": "Notification", + "MessageId": "8f6dee70-c885-558a-be7d-bd48bbf5335e", + "TopicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "Message": json.dumps(raw_ses_event), + } + + response = self.post_from_sns('/anymail/amazon_ses/inbound/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=AmazonSESInboundWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertEqual(event.timestamp, datetime(2018, 3, 30, 17, 21, 51, microsecond=636000, tzinfo=utc)) + self.assertEqual(event.event_id, "jili9m351il3gkburn7o2f0u6788stij94c8ld01") + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_ses_event) + + message = event.message + self.assertIsInstance(message, AnymailInboundMessage) + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'inbound@example.com') + self.assertEqual(str(message.from_email), '"Sender, Inc." ') + self.assertEqual([str(to) for to in message.to], + ['Recipient ', 'someone-else@example.org']) + self.assertEqual(message.subject, 'Test inbound message') + self.assertEqual(message.text, "It's a body\N{HORIZONTAL ELLIPSIS}\r\n") + self.assertEqual(message.html, """
It's a body\N{HORIZONTAL ELLIPSIS}
\r\n""") + self.assertIs(message.spam_detected, True) + + def test_inbound_s3(self): + """Should handle 'S3' receipt action""" + + self.mock_s3_downloadables["InboundEmailBucket-KeepPrivate"] = { + "inbound/fqef5sop459utgdf4o9lqbsv7jeo73pejig34301": self.TEST_MIME_MESSAGE.encode('ascii') + } + + raw_ses_event = { + # (omitting some fields that aren't used by Anymail) + "notificationType": "Received", + "mail": { + "source": "envelope-from@example.org", + "timestamp": "2018-03-30T17:21:51.636Z", + "messageId": "fqef5sop459utgdf4o9lqbsv7jeo73pejig34301", # assigned by Amazon SES + "destination": ["inbound@example.com", "someone-else@example.org"], + }, + "receipt": { + "recipients": ["inbound@example.com"], + "action": { + "type": "S3", + "topicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "bucketName": "InboundEmailBucket-KeepPrivate", + "objectKeyPrefix": "inbound", + "objectKey": "inbound/fqef5sop459utgdf4o9lqbsv7jeo73pejig34301" + }, + "spamVerdict": {"status": "GRAY"}, + }, + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "8f6dee70-c885-558a-be7d-bd48bbf5335e", + "TopicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "Message": json.dumps(raw_ses_event), + } + response = self.post_from_sns('/anymail/amazon_ses/inbound/', raw_sns_message) + self.assertEqual(response.status_code, 200) + + self.mock_client.assert_called_once_with('s3', config=ANY) + self.mock_s3.download_fileobj.assert_called_once_with( + "InboundEmailBucket-KeepPrivate", "inbound/fqef5sop459utgdf4o9lqbsv7jeo73pejig34301", ANY) + + kwargs = self.assert_handler_called_once_with(self.inbound_handler, sender=AmazonSESInboundWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, 'inbound') + self.assertEqual(event.timestamp, datetime(2018, 3, 30, 17, 21, 51, microsecond=636000, tzinfo=utc)) + self.assertEqual(event.event_id, "fqef5sop459utgdf4o9lqbsv7jeo73pejig34301") + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_ses_event) + + message = event.message + self.assertIsInstance(message, AnymailInboundMessage) + self.assertEqual(message.envelope_sender, 'envelope-from@example.org') + self.assertEqual(message.envelope_recipient, 'inbound@example.com') + self.assertEqual(str(message.from_email), '"Sender, Inc." ') + self.assertEqual([str(to) for to in message.to], + ['Recipient ', 'someone-else@example.org']) + self.assertEqual(message.subject, 'Test inbound message') + # rstrip below because the Python 3 EmailBytesParser converts \r\n to \n, but the Python 2 version doesn't + self.assertEqual(message.text.rstrip(), "It's a body\N{HORIZONTAL ELLIPSIS}") + self.assertEqual(message.html.rstrip(), """
It's a body\N{HORIZONTAL ELLIPSIS}
""") + self.assertIsNone(message.spam_detected) + + def test_inbound_s3_failure_message(self): + """Issue a helpful error when S3 download fails""" + # Boto's error: "An error occurred (403) when calling the HeadObject operation: Forbidden") + self.mock_s3.download_fileobj.side_effect = botocore.exceptions.ClientError( + {'Error': {'Code': 403, 'Message': 'Forbidden'}}, operation_name='HeadObject') + + raw_ses_event = { + "notificationType": "Received", + "receipt": { + "action": {"type": "S3", "bucketName": "YourBucket", "objectKey": "inbound/the_object_key"} + }, + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "8f6dee70-c885-558a-be7d-bd48bbf5335e", + "TopicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "Message": json.dumps(raw_ses_event), + } + with self.assertRaisesMessage( + AnymailAPIError, + "Anymail AmazonSESInboundWebhookView couldn't download S3 object 'YourBucket:inbound/the_object_key'" + ) as cm: + self.post_from_sns('/anymail/amazon_ses/inbound/', raw_sns_message) + self.assertIsInstance(cm.exception, botocore.exceptions.ClientError) # both Boto and Anymail exception class + self.assertIn("ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden", + str(cm.exception)) # original Boto message included + + def test_incorrect_tracking_event(self): + """The inbound webhook should warn if it receives tracking events""" + raw_sns_message = { + "Type": "Notification", + "MessageId": "8f6dee70-c885-558a-be7d-bd48bbf5335e", + "TopicArn": "arn:...:111111111111:SES_Tracking", + "Message": '{"notificationType": "Delivery"}', + } + + with self.assertRaisesMessage( + AnymailConfigurationError, + "You seem to have set an Amazon SES *sending* event or notification to publish to an SNS Topic " + "that posts to Anymail's *inbound* webhook URL. (SNS TopicArn arn:...:111111111111:SES_Tracking)" + ): + self.post_from_sns('/anymail/amazon_ses/inbound/', raw_sns_message) diff --git a/tests/test_amazon_ses_integration.py b/tests/test_amazon_ses_integration.py new file mode 100644 index 0000000..f198523 --- /dev/null +++ b/tests/test_amazon_ses_integration.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import unittest +import warnings + +from django.test import SimpleTestCase +from django.test.utils import override_settings + +from anymail.exceptions import AnymailAPIError +from anymail.message import AnymailMessage + +from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS + +try: + ResourceWarning +except NameError: + ResourceWarning = Warning # Python 2 + + +AMAZON_SES_TEST_ACCESS_KEY_ID = os.getenv("AMAZON_SES_TEST_ACCESS_KEY_ID") +AMAZON_SES_TEST_SECRET_ACCESS_KEY = os.getenv("AMAZON_SES_TEST_SECRET_ACCESS_KEY") +AMAZON_SES_TEST_REGION_NAME = os.getenv("AMAZON_SES_TEST_REGION_NAME", "us-east-1") + + +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") +@unittest.skipUnless(AMAZON_SES_TEST_ACCESS_KEY_ID and AMAZON_SES_TEST_SECRET_ACCESS_KEY, + "Set AMAZON_SES_TEST_ACCESS_KEY_ID and AMAZON_SES_TEST_SECRET_ACCESS_KEY " + "environment variables to run Amazon SES integration tests") +@override_settings( + EMAIL_BACKEND="anymail.backends.amazon_ses.EmailBackend", + ANYMAIL={ + "AMAZON_SES_CLIENT_PARAMS": { + # This setting provides Anymail-specific AWS credentials to boto3.client(), + # overriding any credentials in the environment or boto config. It's often + # *not* the best approach -- see the Anymail and boto3 docs for other options. + "aws_access_key_id": AMAZON_SES_TEST_ACCESS_KEY_ID, + "aws_secret_access_key": AMAZON_SES_TEST_SECRET_ACCESS_KEY, + "region_name": AMAZON_SES_TEST_REGION_NAME, + # Can supply any other boto3.client params, including botocore.config.Config as dict + "config": {"retries": {"max_attempts": 2}}, + }, + "AMAZON_SES_CONFIGURATION_SET_NAME": "TestConfigurationSet", # actual config set in Anymail test account + }) +class AmazonSESBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): + """Amazon SES API integration tests + + These tests run against the **live** Amazon SES API, using the environment + variables `AMAZON_SES_TEST_ACCESS_KEY_ID` and `AMAZON_SES_TEST_SECRET_ACCESS_KEY` + as AWS credentials. If those variables are not set, these tests won't run. + (You can also set the environment variable `AMAZON_SES_TEST_REGION_NAME` + to test SES using a region other than the default "us-east-1".) + + Amazon SES doesn't offer a test mode -- it tries to send everything you ask. + To avoid stacking up a pile of undeliverable @example.com + emails, the tests use Amazon's @simulator.amazonses.com addresses. + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/mailbox-simulator.html + + Amazon SES also doesn't support arbitrary senders (so no from@example.com). + We've set up @test-ses.anymail.info as a validated sending domain for these tests. + You may need to change the from_email to your own address when testing. + + """ + + def setUp(self): + super(AmazonSESBackendIntegrationTests, self).setUp() + self.message = AnymailMessage('Anymail Amazon SES integration test', 'Text content', + 'test@test-ses.anymail.info', ['success@simulator.amazonses.com']) + self.message.attach_alternative('

HTML content

', "text/html") + + # boto3 relies on GC to close connections. Python 3 warns about unclosed ssl.SSLSocket during cleanup. + # We don't care. (It might not be a real problem worth warning, but in any case it's not our problem.) + # https://www.google.com/search?q=unittest+boto3+ResourceWarning+unclosed+ssl.SSLSocket + # Filter in TestCase.setUp because unittest resets the warning filters for each test. + # https://stackoverflow.com/a/26620811/647002 + warnings.filterwarnings("ignore", message=r"unclosed "], + cc=["success+cc1@simulator.amazonses.com", "Copy 2 "], + bcc=["success+bcc1@simulator.amazonses.com", "Blind Copy 2 "], + reply_to=["reply1@example.com", "Reply 2 "], + headers={"X-Anymail-Test": "value"}, + metadata={"meta1": "simple_string", "meta2": 2}, + tags=["Re-engagement", "Cohort 12/2017"], + ) + message.attach("attachment1.txt", "Here is some\ntext for you", "text/plain") + message.attach("attachment2.csv", "ID,Name\n1,Amy Lina", "text/csv") + cid = message.attach_inline_image_file(sample_image_path()) + message.attach_alternative( + "

HTML: with link" + "and image: " % cid, + "text/html") + + message.attach_alternative( + "Amazon SES SendRawEmail actually supports multiple alternative parts", + "text/x-note-for-email-geeks") + + message.send() + self.assertEqual(message.anymail_status.status, {'queued'}) + + def test_stored_template(self): + # Using a template created like this: + # boto3.client('ses').create_template(Template={ + # "TemplateName": "TestTemplate", + # "SubjectPart": "Your order {{order}} shipped", + # "HtmlPart": "

Dear {{name}}:

Your order {{order}} shipped {{ship_date}}.

", + # "TextPart": "Dear {{name}}:\r\nYour order {{order}} shipped {{ship_date}}." + # }) + message = AnymailMessage( + template_id='TestTemplate', + from_email='"Test From" ', + to=["First Recipient ", + "success+to2@simulator.amazonses.com"], + merge_data={ + 'success+to1@simulator.amazonses.com': {'order': 12345, 'name': "Test Recipient"}, + 'success+to2@simulator.amazonses.com': {'order': 6789}, + }, + merge_global_data={ + 'name': "Customer", # default + 'ship_date': "today" + }, + ) + message.send() + recipient_status = message.anymail_status.recipients + self.assertEqual(recipient_status['success+to1@simulator.amazonses.com'].status, 'queued') + self.assertRegex(recipient_status['success+to1@simulator.amazonses.com'].message_id, r'[0-9a-f-]+') + self.assertEqual(recipient_status['success+to2@simulator.amazonses.com'].status, 'queued') + self.assertRegex(recipient_status['success+to2@simulator.amazonses.com'].message_id, r'[0-9a-f-]+') + + @override_settings(ANYMAIL={ + "AMAZON_SES_CLIENT_PARAMS": { + "aws_access_key_id": "test-invalid-access-key-id", + "aws_secret_access_key": "test-invalid-secret-access-key", + "region_name": AMAZON_SES_TEST_REGION_NAME, + } + }) + def test_invalid_aws_credentials(self): + with self.assertRaises(AnymailAPIError) as cm: + self.message.send() + err = cm.exception + # Make sure the exception message includes AWS's response: + self.assertIn("The security token included in the request is invalid", str(err)) diff --git a/tests/test_amazon_ses_webhooks.py b/tests/test_amazon_ses_webhooks.py new file mode 100644 index 0000000..6b8faa9 --- /dev/null +++ b/tests/test_amazon_ses_webhooks.py @@ -0,0 +1,538 @@ +import json +import warnings +from datetime import datetime + +import botocore.exceptions +from django.test import override_settings +from django.utils.timezone import utc +from mock import ANY, patch + +from anymail.exceptions import AnymailConfigurationError, AnymailInsecureWebhookWarning +from anymail.signals import AnymailTrackingEvent +from anymail.webhooks.amazon_ses import AmazonSESTrackingWebhookView + +from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase + + +class AmazonSESWebhookTestsMixin(object): + def post_from_sns(self, path, raw_sns_message, **kwargs): + # noinspection PyUnresolvedReferences + return self.client.post( + path, + content_type='text/plain; charset=UTF-8', # SNS posts JSON as text/plain + data=json.dumps(raw_sns_message), + HTTP_X_AMZ_SNS_MESSAGE_ID=raw_sns_message["MessageId"], + HTTP_X_AMZ_SNS_MESSAGE_TYPE=raw_sns_message["Type"], + # Anymail doesn't use other x-amz-sns-* headers + **kwargs) + + +class AmazonSESWebhookSecurityTests(WebhookTestCase, AmazonSESWebhookTestsMixin, WebhookBasicAuthTestsMixin): + def call_webhook(self): + return self.post_from_sns('/anymail/amazon_ses/tracking/', + {"Type": "Notification", "MessageId": "123", "Message": "{}"}) + + # Most actual tests are in WebhookBasicAuthTestsMixin + + def test_verifies_missing_auth(self): + # Must handle missing auth header slightly differently from Anymail default 400 SuspiciousOperation: + # SNS will only send basic auth after missing auth responds 401 WWW-Authenticate: Basic realm="..." + self.clear_basic_auth() + response = self.call_webhook() + self.assertEqual(response.status_code, 401) + self.assertEqual(response["WWW-Authenticate"], 'Basic realm="Anymail WEBHOOK_SECRET"') + + +class AmazonSESNotificationsTests(WebhookTestCase, AmazonSESWebhookTestsMixin): + def test_bounce_event(self): + # This test includes a complete Amazon SES example event. (Later tests omit some payload for brevity.) + # https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-examples.html#notification-examples-bounce + raw_ses_event = { + "notificationType": "Bounce", + "bounce": { + "bounceType": "Permanent", + "reportingMTA": "dns; email.example.com", + "bouncedRecipients": [{ + "emailAddress": "jane@example.com", + "status": "5.1.1", + "action": "failed", + "diagnosticCode": "smtp; 550 5.1.1 ... User unknown", + }], + "bounceSubType": "General", + "timestamp": "2016-01-27T14:59:44.101Z", # when bounce sent (by receiving ISP) + "feedbackId": "00000138111222aa-44455566-cccc-cccc-cccc-ddddaaaa068a-000000", # unique id for bounce + "remoteMtaIp": "127.0.2.0", + }, + "mail": { + "timestamp": "2016-01-27T14:59:38.237Z", # when message sent + "source": "john@example.com", + "sourceArn": "arn:aws:ses:us-west-2:888888888888:identity/example.com", + "sourceIp": "127.0.3.0", + "sendingAccountId": "123456789012", + "messageId": "00000138111222aa-33322211-cccc-cccc-cccc-ddddaaaa0680-000000", + "destination": ["jane@example.com", "mary@example.com", "richard@example.com"], + "headersTruncated": False, + "headers": [ + {"name": "From", "value": '"John Doe" '}, + {"name": "To", "value": '"Jane Doe" , "Mary Doe" ,' + ' "Richard Doe" '}, + {"name": "Message-ID", "value": "custom-message-ID"}, + {"name": "Subject", "value": "Hello"}, + {"name": "Content-Type", "value": 'text/plain; charset="UTF-8"'}, + {"name": "Content-Transfer-Encoding", "value": "base64"}, + {"name": "Date", "value": "Wed, 27 Jan 2016 14:05:45 +0000"}, + {"name": "X-Tag", "value": "tag 1"}, + {"name": "X-Tag", "value": "tag 2"}, + {"name": "X-Metadata", "value": '{"meta1":"string","meta2":2}'}, + ], + "commonHeaders": { + "from": ["John Doe "], + "date": "Wed, 27 Jan 2016 14:05:45 +0000", + "to": ["Jane Doe , Mary Doe ," + " Richard Doe "], + "messageId": "custom-message-ID", + "subject": "Hello", + }, + }, + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", # unique id for SNS event + "TopicArn": "arn:aws:sns:us-east-1:1234567890:SES_Events", + "Subject": "Amazon SES Email Event Notification", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + "SignatureVersion": "1", + "Signature": "EXAMPLE-SIGNATURE==", + "SigningCertURL": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-12345abcde.pem", + "UnsubscribeURL": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn...", + } + + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.esp_event, raw_ses_event) + self.assertEqual(event.timestamp, datetime(2018, 3, 26, 17, 58, 59, microsecond=675000, tzinfo=utc)) # SNS + self.assertEqual(event.message_id, "00000138111222aa-33322211-cccc-cccc-cccc-ddddaaaa0680-000000") + self.assertEqual(event.event_id, "19ba9823-d7f2-53c1-860e-cb10e0d13dfc") + self.assertEqual(event.recipient, "jane@example.com") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.description, "Permanent: General") + self.assertEqual(event.mta_response, "smtp; 550 5.1.1 ... User unknown") + self.assertEqual(event.tags, ["tag 1", "tag 2"]) + self.assertEqual(event.metadata, {"meta1": "string", "meta2": 2}) + + # For brevity, remaining tests omit some event fields that aren't used by Anymail + + def test_multiple_bounce_event(self): + """Amazon SES notification can cover multiple recipients""" + raw_ses_event = { + "notificationType": "Bounce", + "bounce": { + "bounceType": "Permanent", + "bounceSubType": "General", + "bouncedRecipients": [ + {"emailAddress": "jane@example.com"}, + {"emailAddress": "richard@example.com"} + ], + }, + "mail": { + "messageId": "00000137860315fd-34208509-5b74-41f3-95c5-22c1edc3c924-000000", + "destination": ["jane@example.com", "mary@example.com", "richard@example.com"], + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + + # tracking handler should be called twice -- once for each bounced recipient + # (but not for the third, non-bounced recipient) + self.assertEqual(self.tracking_handler.call_count, 2) + + _, kwargs = self.tracking_handler.call_args_list[0] + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.recipient, "jane@example.com") + self.assertEqual(event.description, "Permanent: General") + self.assertIsNone(event.mta_response) + + _, kwargs = self.tracking_handler.call_args_list[1] + event = kwargs['event'] + self.assertEqual(event.esp_event, raw_ses_event) + self.assertEqual(event.recipient, "richard@example.com") + + def test_complaint_event(self): + raw_ses_event = { + "notificationType": "Complaint", + "complaint": { + "userAgent": "AnyCompany Feedback Loop (V0.01)", + "complainedRecipients": [{"emailAddress": "richard@example.com"}], + "complaintFeedbackType": "abuse", + }, + "mail": { + "messageId": "000001378603177f-7a5433e7-8edb-42ae-af10-f0181f34d6ee-000000", + "destination": ["jane@example.com", "mary@example.com", "richard@example.com"], + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertEqual(event.event_type, "complained") + self.assertEqual(event.recipient, "richard@example.com") + self.assertEqual(event.reject_reason, "spam") + self.assertEqual(event.description, "abuse") + self.assertEqual(event.user_agent, "AnyCompany Feedback Loop (V0.01)") + + def test_delivery_event(self): + raw_ses_event = { + "notificationType": "Delivery", + "mail": { + "timestamp": "2016-01-27T14:59:38.237Z", + "messageId": "0000014644fe5ef6-9a483358-9170-4cb4-a269-f5dcdf415321-000000", + "destination": ["jane@example.com", "mary@example.com", "richard@example.com"], + }, + "delivery": { + "timestamp": "2016-01-27T14:59:38.237Z", + "recipients": ["jane@example.com"], + "processingTimeMillis": 546, + "reportingMTA": "a8-70.smtp-out.amazonses.com", + "smtpResponse": "250 ok: Message 64111812 accepted", + "remoteMtaIp": "127.0.2.0" + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertEqual(event.event_type, "delivered") + self.assertEqual(event.recipient, "jane@example.com") + self.assertEqual(event.mta_response, "250 ok: Message 64111812 accepted") + + def test_send_event(self): + raw_ses_event = { + "eventType": "Send", + "mail": { + "timestamp": "2016-10-14T05:02:16.645Z", + "messageId": "7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000", + "destination": ["recipient@example.com"], + "tags": { + "ses:configuration-set": ["ConfigSet"], + "ses:source-ip": ["192.0.2.0"], + "ses:from-domain": ["example.com"], + "ses:caller-identity": ["ses_user"], + "myCustomTag1": ["myCustomTagValue1"], + "myCustomTag2": ["myCustomTagValue2"] + } + }, + "send": {} + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "sent") + self.assertEqual(event.esp_event, raw_ses_event) + self.assertEqual(event.timestamp, datetime(2018, 3, 26, 17, 58, 59, microsecond=675000, tzinfo=utc)) # SNS + self.assertEqual(event.message_id, "7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000") + self.assertEqual(event.event_id, "19ba9823-d7f2-53c1-860e-cb10e0d13dfc") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.tags, []) # Anymail doesn't load Amazon SES "Message Tags" + self.assertEqual(event.metadata, {}) + + def test_reject_event(self): + raw_ses_event = { + "eventType": "Reject", + "mail": { + "timestamp": "2016-10-14T17:38:15.211Z", + "messageId": "7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000", + "destination": ["recipient@example.com"], + }, + "reject": { + "reason": "Bad content" + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertEqual(event.event_type, "rejected") + self.assertEqual(event.reject_reason, "blocked") + self.assertEqual(event.description, "Bad content") + self.assertEqual(event.recipient, "recipient@example.com") + + def test_open_event(self): + raw_ses_event = { + "eventType": "Open", + "mail": { + "destination": ["recipient@example.com"], + "messageId": "7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000", + }, + "open": { + "ipAddress": "192.0.2.1", + "timestamp": "2017-08-09T22:00:19.652Z", + "userAgent": "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X)..." + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertEqual(event.event_type, "opened") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.user_agent, "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X)...") + + def test_click_event(self): + raw_ses_event = { + "eventType": "Click", + "click": { + "ipAddress": "192.0.2.1", + "link": "https://docs.aws.amazon.com/ses/latest/DeveloperGuide/", + "linkTags": { + "samplekey0": ["samplevalue0"], + "samplekey1": ["samplevalue1"], + }, + "timestamp": "2017-08-09T23:51:25.570Z", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36..." + }, + "mail": { + "destination": ["recipient@example.com"], + "messageId": "7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000", + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertEqual(event.event_type, "clicked") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.user_agent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...") + self.assertEqual(event.click_url, "https://docs.aws.amazon.com/ses/latest/DeveloperGuide/") + + def test_rendering_failure_event(self): + raw_ses_event = { + "eventType": "Rendering Failure", + "mail": { + "messageId": "c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000", + "destination": ["recipient@example.com"], + }, + "failure": { + "errorMessage": "Attribute 'attributeName' is not present in the rendering data.", + "templateName": "MyTemplate" + } + } + raw_sns_message = { + "Type": "Notification", + "MessageId": "19ba9823-d7f2-53c1-860e-cb10e0d13dfc", + "Message": json.dumps(raw_ses_event) + "\n", + "Timestamp": "2018-03-26T17:58:59.675Z", + } + response = self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=AmazonSESTrackingWebhookView, + event=ANY, esp_name='Amazon SES') + event = kwargs['event'] + self.assertEqual(event.event_type, "failed") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.description, "Attribute 'attributeName' is not present in the rendering data.") + + def test_incorrect_received_event(self): + """The tracking webhook should warn if it receives inbound events""" + raw_sns_message = { + "Type": "Notification", + "MessageId": "8f6dee70-c885-558a-be7d-bd48bbf5335e", + "TopicArn": "arn:aws:sns:us-east-1:111111111111:SES_Inbound", + "Message": '{"notificationType": "Received"}', + } + with self.assertRaisesMessage( + AnymailConfigurationError, + "You seem to have set an Amazon SES *inbound* receipt rule to publish to an SNS Topic that posts " + "to Anymail's *tracking* webhook URL. (SNS TopicArn arn:aws:sns:us-east-1:111111111111:SES_Inbound)" + ): + self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message) + + +class AmazonSESSubscriptionManagementTests(WebhookTestCase, AmazonSESWebhookTestsMixin): + # Anymail will automatically respond to SNS subscription notifications + # if Anymail is configured to require basic auth via WEBHOOK_SECRET. + # (Note that WebhookTestCase sets up ANYMAIL WEBHOOK_SECRET.) + + def setUp(self): + super(AmazonSESSubscriptionManagementTests, self).setUp() + # Mock boto3.session.Session().client('sns').confirm_subscription (and any other client operations) + # (We could also use botocore.stub.Stubber, but mock works well with our test structure) + self.patch_boto3_session = patch('anymail.webhooks.amazon_ses.boto3.session.Session', autospec=True) + self.mock_session = self.patch_boto3_session.start() # boto3.session.Session + self.addCleanup(self.patch_boto3_session.stop) + self.mock_client = self.mock_session.return_value.client # boto3.session.Session().client + self.mock_client_instance = self.mock_client.return_value # boto3.session.Session().client('sns', ...) + self.mock_client_instance.confirm_subscription.return_value = { + 'SubscriptionArn': 'arn:aws:sns:us-west-2:123456789012:SES_Notifications:aaaaaaa-...' + } + + SNS_SUBSCRIPTION_CONFIRMATION = { + "Type": "SubscriptionConfirmation", + "MessageId": "165545c9-2a5c-472c-8df2-7ff2be2b3b1b", + "Token": "EXAMPLE_TOKEN", + "TopicArn": "arn:aws:sns:us-west-2:123456789012:SES_Notifications", + "Message": "You have chosen to subscribe ...\nTo confirm..., visit the SubscribeURL included in this message.", + "SubscribeURL": "https://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=...", + "Timestamp": "2012-04-26T20:45:04.751Z", + "SignatureVersion": "1", + "Signature": "EXAMPLE-SIGNATURE==", + "SigningCertURL": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-12345abcde.pem" + } + + def test_sns_subscription_auto_confirmation(self): + """Anymail webhook will auto-confirm SNS topic subscriptions""" + response = self.post_from_sns('/anymail/amazon_ses/tracking/', self.SNS_SUBSCRIPTION_CONFIRMATION) + self.assertEqual(response.status_code, 200) + # auto-confirmed: + self.mock_client.assert_called_once_with('sns', config=ANY) + self.mock_client_instance.confirm_subscription.assert_called_once_with( + TopicArn="arn:aws:sns:us-west-2:123456789012:SES_Notifications", + Token="EXAMPLE_TOKEN", AuthenticateOnUnsubscribe="true") + # didn't notify receivers: + self.assertEqual(self.tracking_handler.call_count, 0) + self.assertEqual(self.inbound_handler.call_count, 0) + + def test_sns_subscription_confirmation_failure(self): + """Auto-confirmation allows error through if confirm call fails""" + self.mock_client_instance.confirm_subscription.side_effect = botocore.exceptions.ClientError({ + 'Error': { + 'Type': 'Sender', + 'Code': 'InternalError', + 'Message': 'Gremlins!', + }, + 'ResponseMetadata': { + 'RequestId': 'aaaaaaaa-2222-1111-8888-bbbb3333bbbb', + 'HTTPStatusCode': 500, + } + }, operation_name="confirm_subscription") + with self.assertRaisesMessage(botocore.exceptions.ClientError, "Gremlins!"): + self.post_from_sns('/anymail/amazon_ses/tracking/', self.SNS_SUBSCRIPTION_CONFIRMATION) + # didn't notify receivers: + self.assertEqual(self.tracking_handler.call_count, 0) + self.assertEqual(self.inbound_handler.call_count, 0) + + @override_settings(ANYMAIL={}) # clear WEBHOOK_SECRET setting from base WebhookTestCase + def test_sns_subscription_confirmation_auth_disabled(self): + """Anymail *won't* auto-confirm SNS subscriptions if WEBHOOK_SECRET isn't in use""" + warnings.simplefilter("ignore", AnymailInsecureWebhookWarning) # (this gets tested elsewhere) + with self.assertLogs('django.security.AnymailWebhookValidationFailure') as cm: + response = self.post_from_sns('/anymail/amazon_ses/tracking/', self.SNS_SUBSCRIPTION_CONFIRMATION) + self.assertEqual(response.status_code, 400) # bad request + self.assertEqual( + ["Anymail received an unexpected SubscriptionConfirmation request for Amazon SNS topic " + "'arn:aws:sns:us-west-2:123456789012:SES_Notifications'. (Anymail can automatically confirm " + "SNS subscriptions if you set a WEBHOOK_SECRET and use that in your SNS notification url. Or " + "you can manually confirm this subscription in the SNS dashboard with token 'EXAMPLE_TOKEN'.)"], + [record.getMessage() for record in cm.records]) + # *didn't* try to confirm the subscription: + self.assertEqual(self.mock_client_instance.confirm_subscription.call_count, 0) + # didn't notify receivers: + self.assertEqual(self.tracking_handler.call_count, 0) + self.assertEqual(self.inbound_handler.call_count, 0) + + def test_sns_confirmation_success_notification(self): + """Anymail ignores the 'Successfully validated' notification after confirming an SNS subscription""" + response = self.post_from_sns('/anymail/amazon_ses/tracking/', { + "Type": "Notification", + "MessageId": "7fbca0d9-eeab-5285-ae27-f3f57f2e84b0", + "TopicArn": "arn:aws:sns:us-west-2:123456789012:SES_Notifications", + "Message": "Successfully validated SNS topic for Amazon SES event publishing.", + "Timestamp": "2018-03-21T16:58:45.077Z", + "SignatureVersion": "1", + "Signature": "EXAMPLE_SIGNATURE==", + "SigningCertURL": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-12345abcde.pem", + "UnsubscribeURL": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe...", + }) + self.assertEqual(response.status_code, 200) + # didn't notify receivers: + self.assertEqual(self.tracking_handler.call_count, 0) + self.assertEqual(self.inbound_handler.call_count, 0) + + def test_sns_unsubscribe_confirmation(self): + """Anymail ignores the UnsubscribeConfirmation SNS message after deleting a subscription""" + response = self.post_from_sns('/anymail/amazon_ses/tracking/', { + "Type": "UnsubscribeConfirmation", + "MessageId": "47138184-6831-46b8-8f7c-afc488602d7d", + "Token": "EXAMPLE_TOKEN", + "TopicArn": "arn:aws:sns:us-west-2:123456789012:SES_Notifications", + "Message": "You have chosen to deactivate subscription ...\nTo cancel ... visit the SubscribeURL...", + "SubscribeURL": "https://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=...", + "Timestamp": "2012-04-26T20:06:41.581Z", + "SignatureVersion": "1", + "Signature": "EXAMPLE_SIGNATURE==", + "SigningCertURL": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-12345abcde.pem", + }) + self.assertEqual(response.status_code, 200) + # *didn't* try to use the Token to re-enable the subscription: + self.assertEqual(self.mock_client_instance.confirm_subscription.call_count, 0) + # didn't notify receivers: + self.assertEqual(self.tracking_handler.call_count, 0) + self.assertEqual(self.inbound_handler.call_count, 0) + + @override_settings(ANYMAIL_AMAZON_SES_AUTO_CONFIRM_SNS_SUBSCRIPTIONS=False) + def test_disable_auto_confirmation(self): + """The ANYMAIL setting AMAZON_SES_AUTO_CONFIRM_SNS_SUBSCRIPTIONS will disable this feature""" + response = self.post_from_sns('/anymail/amazon_ses/tracking/', self.SNS_SUBSCRIPTION_CONFIRMATION) + self.assertEqual(response.status_code, 200) + # *didn't* try to subscribe: + self.assertEqual(self.mock_session.call_count, 0) + self.assertEqual(self.mock_client.call_count, 0) + # didn't notify receivers: + self.assertEqual(self.tracking_handler.call_count, 0) + self.assertEqual(self.inbound_handler.call_count, 0) diff --git a/tox.ini b/tox.ini index 93ae11a..721e260 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ deps = djangoMaster: https://github.com/django/django/tarball/master # testing dependencies (duplicates setup.py tests_require): mock + boto3 sparkpost ignore_outcome = django21: True @@ -39,6 +40,7 @@ commands = passenv = RUN_LIVE_TESTS CONTINUOUS_INTEGRATION + AMAZON_SES_TEST_* MAILGUN_TEST_* MAILJET_TEST_* MANDRILL_TEST_*