From ea446a94d3a34f93ea4cfab56451071e39364866 Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Sun, 26 Feb 2023 14:16:46 -0800 Subject: [PATCH] Amazon SES: implement SES v2 API backend * Add `anymail.backends.amazon_sesv2.EmailBackend` using SES v2 API * Declare current SES v1 backend as "deprecated" Closes #274 --- CHANGELOG.rst | 8 + anymail/backends/amazon_sesv2.py | 520 ++++++++++++++ docs/esps/amazon_ses.rst | 184 ++++- tests/test_amazon_sesv2_backend.py | 959 +++++++++++++++++++++++++ tests/test_amazon_sesv2_integration.py | 199 +++++ 5 files changed, 1841 insertions(+), 29 deletions(-) create mode 100644 anymail/backends/amazon_sesv2.py create mode 100644 tests/test_amazon_sesv2_backend.py create mode 100644 tests/test_amazon_sesv2_integration.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 12e1b5a..88096b4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,6 +30,14 @@ vNext *Unreleased changes* +Deprecations +~~~~~~~~~~~~ +* **Amazon SES:** Anymail is switching to the Amazon SES v2 API for sending mail. + Support for the original SES v1 API is now deprecated, and will be dropped in a + future Anymail release (likely in late 2023). Many projects will not + require code changes, but you will need to update your IAM permissions. See + `Migrating to the SES v2 API `__. + Other ~~~~~ * Test against Django 4.2 prerelease, Python 3.11 (with Django 4.2), diff --git a/anymail/backends/amazon_sesv2.py b/anymail/backends/amazon_sesv2.py new file mode 100644 index 0000000..553a218 --- /dev/null +++ b/anymail/backends/amazon_sesv2.py @@ -0,0 +1,520 @@ +import email.charset +import email.encoders +import email.policy + +from .._version import __version__ +from ..exceptions import AnymailAPIError, AnymailImproperlyInstalled +from ..message import AnymailRecipientStatus +from ..utils import UNSET, get_anymail_setting +from .base import AnymailBaseBackend, BasePayload + +try: + import boto3 + from botocore.client import Config + from botocore.exceptions import BotoCoreError, ClientError, ConnectionError +except ImportError as err: + raise AnymailImproperlyInstalled( + missing_package="boto3", backend="amazon_sesv2" + ) from err + + +# boto3 has several root exception classes; this is meant to cover all of them +BOTO_BASE_ERRORS = (BotoCoreError, ClientError, ConnectionError) + + +class EmailBackend(AnymailBaseBackend): + """ + Amazon SES v2 Email Backend (using boto3) + """ + + # Deliberately no "v2" in the esp_name: we don't want to force client changes + # when the v1 backend is retired and this one replace it as just "amazon_ses". + esp_name = "Amazon SES" + + def __init__(self, **kwargs): + """Init options from Django settings""" + super().__init__(**kwargs) + # AMAZON_SES_CLIENT_PARAMS is optional + # (boto3 can find credentials several other ways) + self.session_params, self.client_params = _get_anymail_boto3_params( + esp_name=self.esp_name, kwargs=kwargs + ) + self.configuration_set_name = get_anymail_setting( + "configuration_set_name", + esp_name=self.esp_name, + kwargs=kwargs, + allow_bare=False, + default=None, + ) + self.message_tag_name = get_anymail_setting( + "message_tag_name", + esp_name=self.esp_name, + kwargs=kwargs, + allow_bare=False, + default=None, + ) + + self.client = None + + def open(self): + if self.client: + return False # already exists + try: + self.client = boto3.session.Session(**self.session_params).client( + "sesv2", **self.client_params + ) + except BOTO_BASE_ERRORS: + if not self.fail_silently: + raise + else: + return True # created client + + def close(self): + if self.client is None: + return + self.client.close() + self.client = None + + def build_message_payload(self, message, defaults): + if getattr(message, "template_id", UNSET) is not UNSET: + # For simplicity, use SESv2 SendBulkEmail for all templated messages + # (even though SESv2 SendEmail has a template option). + return AmazonSESV2SendBulkEmailPayload(message, defaults, self) + else: + return AmazonSESV2SendEmailPayload(message, defaults, self) + + def post_to_esp(self, payload, message): + payload.finalize_payload() + try: + client_send_api = getattr(self.client, payload.api_name) + except AttributeError: + raise NotImplementedError( + f"boto3 sesv2 client does not have method {payload.api_name!r}." + f" Check {payload.__class__.__name__}.api_name." + ) from None + try: + response = client_send_api(**payload.params) + except BOTO_BASE_ERRORS as err: + # ClientError has a response attr with parsed json error response + # (other errors don't) + raise AnymailAPIError( + str(err), + backend=self, + email_message=message, + payload=payload, + response=getattr(err, "response", None), + ) from err + return response + + def parse_recipient_status(self, response, payload, message): + return payload.parse_recipient_status(response) + + +class AmazonSESBasePayload(BasePayload): + #: Name of the boto3 SES/SESv2 client method to call + api_name = "SUBCLASS_MUST_OVERRIDE" + + def init_payload(self): + self.params = {} + if self.backend.configuration_set_name is not None: + self.params["ConfigurationSetName"] = self.backend.configuration_set_name + + def finalize_payload(self): + pass + + def parse_recipient_status(self, response): + # response is the parsed (dict) JSON returned from the API call + raise NotImplementedError() + + def set_esp_extra(self, extra): + # e.g., ConfigurationSetName, FromEmailAddressIdentityArn, + # FeedbackForwardingEmailAddress, ListManagementOptions + self.params.update(extra) + + +class AmazonSESV2SendEmailPayload(AmazonSESBasePayload): + api_name = "send_email" + + def init_payload(self): + super().init_payload() + self.all_recipients = [] # for parse_recipient_status + self.mime_message = self.message.message() + + def finalize_payload(self): + # (The boto3 SES client handles base64 encoding raw_message.) + raw_message = self.generate_raw_message() + self.params["Content"] = {"Raw": {"Data": raw_message}} + + def generate_raw_message(self): + """ + Serialize self.mime_message as an RFC-5322/-2045 MIME message, + encoded as 7bit-clean, us-ascii byte data. + """ + # Amazon SES does not support `Content-Transfer-Encoding: 8bit`. And using 8bit + # with SES open or click tracking results in mis-encoded characters. To avoid + # this, convert any 8bit parts to 7bit quoted printable or base64. (We own + # self.mime_message, so destructively modifying it should be OK.) + # (You might think cte_type="7bit" in the email.policy below would cover this, + # but it seems that cte_type is only examined as the MIME parts are constructed, + # not when an email.generator serializes them.) + for part in self.mime_message.walk(): + if part["Content-Transfer-Encoding"] == "8bit": + del part["Content-Transfer-Encoding"] + if part.get_content_maintype() == "text": + # (Avoid base64 for text parts, which can trigger spam filters) + email.encoders.encode_quopri(part) + else: + email.encoders.encode_base64(part) + + self.mime_message.policy = email.policy.default.clone(cte_type="7bit") + return self.mime_message.as_bytes() + + def parse_recipient_status(self, response): + try: + message_id = response["MessageId"] + except (KeyError, TypeError) as err: + raise AnymailAPIError( + f"{err!s} parsing Amazon SES send result {response!r}", + backend=self.backend, + email_message=self.message, + payload=self, + ) from None + + recipient_status = AnymailRecipientStatus( + message_id=message_id, status="queued" + ) + return { + recipient.addr_spec: recipient_status for recipient in self.all_recipients + } + + # Standard EmailMessage attrs... + # These all get rolled into the RFC-5322 raw mime directly via + # EmailMessage.message() + + def _no_send_defaults(self, attr): + # Anymail global send defaults don't work for standard attrs, because the + # merged/computed value isn't forced back into the EmailMessage. + if attr in self.defaults: + self.unsupported_feature( + f"Anymail send defaults for '{attr}' with Amazon SES" + ) + + def set_from_email(self, email): + # If params["FromEmailAddress"] is not provided, SES will parse it from the raw + # mime_message headers. (And setting it replaces any From header. Note that + # v2 SendEmail doesn't have an equivalent to v1 SendRawEmail's Sender param.) + self._no_send_defaults("from_email") + + def set_recipients(self, recipient_type, emails): + # Although Amazon SES can parse the 'to' and 'cc' recipients from the raw + # mime_message headers, providing them in the Destination param makes it + # explicit (and is required for 'bcc' and for spoofed 'to'). + self.all_recipients += emails # save for parse_recipient_status + self._no_send_defaults(recipient_type) + + if emails: + # params["Destination"] = {"ToAddresses": [...], "CcAddresses": etc.} + # (Unlike most SendEmail params, these _don't_ replace the corresponding + # raw mime_message headers.) + assert recipient_type in ("to", "cc", "bcc") + destination_key = f"{recipient_type.capitalize()}Addresses" + self.params.setdefault("Destination", {})[destination_key] = [ + email.address for email in emails + ] + + def set_subject(self, subject): + # included in mime_message + self._no_send_defaults("subject") + + def set_reply_to(self, emails): + # included in mime_message + # (and setting params["ReplyToAddresses"] replaces any Reply-To header) + self._no_send_defaults("reply_to") + + def set_extra_headers(self, headers): + # included in mime_message + self._no_send_defaults("extra_headers") + + def set_text_body(self, body): + # included in mime_message + self._no_send_defaults("body") + + def set_html_body(self, body): + # included in mime_message + self._no_send_defaults("body") + + def set_alternatives(self, alternatives): + # included in mime_message + self._no_send_defaults("alternatives") + + def set_attachments(self, attachments): + # included in mime_message + self._no_send_defaults("attachments") + + # Anymail-specific payload construction + + def set_envelope_sender(self, email): + # Amazon SES will generate a unique mailfrom, and then forward any delivery + # problem reports that address receives to the address specified here: + self.params["FeedbackForwardingEmailAddress"] = email.addr_spec + + def set_spoofed_to_header(self, header_to): + # django.core.mail.EmailMessage.message() has already set + # self.mime_message["To"] = header_to + # and performed any necessary header sanitization. + # + # The actual "to" is already in params["Destination"]["ToAddresses"]. + # + # So, nothing to do here, except prevent the default + # "unsupported feature" error. + pass + + def set_metadata(self, metadata): + # Amazon SES has two mechanisms for adding custom data to a message: + # * Custom message headers are available to webhooks (SNS notifications), + # but not in CloudWatch metrics/dashboards or Kinesis Firehose streams. + # Custom headers can be sent only with SendRawEmail. + # * "Message Tags" are available to CloudWatch and Firehose, and to SNS + # notifications for SES *events* but not SES *notifications*. (Got that?) + # Message Tags also allow *very* limited characters in both name and value. + # Message Tags can be sent with any SES send call. + # (See "How do message tags work?" in + # https://aws.amazon.com/blogs/ses/introducing-sending-metrics/ + # and https://forums.aws.amazon.com/thread.jspa?messageID=782922.) + # To support reliable retrieval in webhooks, just use custom headers for + # metadata. + self.mime_message["X-Metadata"] = self.serialize_json(metadata) + + def set_tags(self, tags): + # See note about Amazon SES Message Tags and custom headers in set_metadata + # above. To support reliable retrieval in webhooks, use custom headers for tags. + # (There are no restrictions on number or content for custom header tags.) + for tag in tags: + # creates multiple X-Tag headers, one per tag: + self.mime_message.add_header("X-Tag", tag) + + # Also *optionally* pass a single Message Tag if the AMAZON_SES_MESSAGE_TAG_NAME + # Anymail setting is set (default no). The AWS API restricts tag content in this + # case. (This is useful for dashboard segmentation; use esp_extra["Tags"] for + # anything more complex.) + if tags and self.backend.message_tag_name is not None: + if len(tags) > 1: + self.unsupported_feature( + "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting" + ) + self.params.setdefault("EmailTags", []).append( + {"Name": self.backend.message_tag_name, "Value": tags[0]} + ) + + def set_template_id(self, template_id): + raise NotImplementedError( + f"{self.__class__.__name__} should not have been used with template_id" + ) + + def set_merge_data(self, merge_data): + self.unsupported_feature("merge_data without template_id") + + def set_merge_global_data(self, merge_global_data): + self.unsupported_feature("global_merge_data without template_id") + + +class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload): + api_name = "send_bulk_email" + + def init_payload(self): + super().init_payload() + # late-bind recipients and merge_data in finalize_payload + self.recipients = {"to": [], "cc": [], "bcc": []} + self.merge_data = {} + + def finalize_payload(self): + # Build BulkEmailEntries from recipients and merge_data. + + # Any cc and bcc recipients should be included in every entry: + cc_and_bcc_addresses = {} + if self.recipients["cc"]: + cc_and_bcc_addresses["CcAddresses"] = [ + cc.address for cc in self.recipients["cc"] + ] + if self.recipients["bcc"]: + cc_and_bcc_addresses["BccAddresses"] = [ + bcc.address for bcc in self.recipients["bcc"] + ] + + # Construct an entry with merge data for each "to" recipient: + self.params["BulkEmailEntries"] = [ + { + "Destination": dict(ToAddresses=[to.address], **cc_and_bcc_addresses), + "ReplacementEmailContent": { + "ReplacementTemplate": { + "ReplacementTemplateData": self.serialize_json( + self.merge_data.get(to.addr_spec, {}) + ), + } + }, + } + for to in self.recipients["to"] + ] + + def parse_recipient_status(self, response): + try: + results = response["BulkEmailEntryResults"] + ses_status_set = set(result["Status"] for result in results) + anymail_statuses = [ + AnymailRecipientStatus( + message_id=result.get("MessageId", None), + status="queued" if result["Status"] == "SUCCESS" else "failed", + ) + for result in results + ] + except (KeyError, TypeError) as err: + raise AnymailAPIError( + f"{err!s} parsing Amazon SES send result {response!r}", + backend=self.backend, + email_message=self.message, + payload=self, + ) from None + + # If all BulkEmailEntryResults[].Status are the same non-success status, + # raise an APIError to expose the error message/reason (matching behavior + # of non-template SendEmail call). + if len(ses_status_set) == 1 and ses_status_set != {"SUCCESS"}: + raise AnymailAPIError( + # use Error text if available, else the Status enum, from first result + results[0].get("Error", results[0]["Status"]), + backend=self.backend, + email_message=self.message, + payload=self, + response=response, + ) + + # Otherwise, return per-recipient status (just "queued" or "failed") for + # all-success, mixed success/error, or all-error mixed-reason cases. + # The BulkEmailEntryResults are in the same order as the Destination param + # (which is in the same order as recipients["to"]). + to_addrs = [to.addr_spec for to in self.recipients["to"]] + if len(anymail_statuses) != len(to_addrs): + raise AnymailAPIError( + f"Sent to {len(to_addrs)} destinations," + f" but only {len(anymail_statuses)} statuses" + f" in Amazon SES send result {response!r}", + backend=self.backend, + email_message=self.message, + payload=self, + ) + return dict(zip(to_addrs, anymail_statuses)) + + def set_from_email(self, email): + # this will RFC2047-encode display_name if needed: + self.params["FromEmailAddress"] = email.address + + def set_recipients(self, recipient_type, emails): + # late-bound in finalize_payload + assert recipient_type in ("to", "cc", "bcc") + self.recipients[recipient_type] = emails + + def set_subject(self, subject): + # (subject can only come from template; you can use substitution vars in that) + if subject: + self.unsupported_feature("overriding template subject") + + def set_reply_to(self, emails): + if emails: + self.params["ReplyToAddresses"] = [email.address for email in emails] + + def set_extra_headers(self, headers): + self.unsupported_feature("extra_headers with template") + + def set_text_body(self, body): + if body: + self.unsupported_feature("overriding template body content") + + def set_html_body(self, body): + if body: + self.unsupported_feature("overriding template body content") + + def set_attachments(self, attachments): + if attachments: + self.unsupported_feature("attachments with template") + + # Anymail-specific payload construction + + def set_envelope_sender(self, email): + # Amazon SES will generate a unique mailfrom, and then forward any delivery + # problem reports that address receives to the address specified here: + self.params["FeedbackForwardingEmailAddress"] = email.addr_spec + + def set_metadata(self, metadata): + # no custom headers with SendBulkEmail + self.unsupported_feature("metadata with template") + + def set_tags(self, tags): + # no custom headers with SendBulkEmail, but support + # AMAZON_SES_MESSAGE_TAG_NAME if used (see tags/metadata in + # AmazonSESV2SendEmailPayload for more info) + if tags: + if self.backend.message_tag_name is not None: + if len(tags) > 1: + self.unsupported_feature( + "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting" + ) + self.params["DefaultEmailTags"] = [ + {"Name": self.backend.message_tag_name, "Value": tags[0]} + ] + else: + self.unsupported_feature( + "tags with template (unless using the" + " AMAZON_SES_MESSAGE_TAG_NAME setting)" + ) + + def set_template_id(self, template_id): + # DefaultContent.Template.TemplateName + self.params.setdefault("DefaultContent", {}).setdefault("Template", {})[ + "TemplateName" + ] = template_id + + def set_merge_data(self, merge_data): + # late-bound in finalize_payload + self.merge_data = merge_data + + def set_merge_global_data(self, merge_global_data): + # DefaultContent.Template.TemplateData + self.params.setdefault("DefaultContent", {}).setdefault("Template", {})[ + "TemplateData" + ] = self.serialize_json(merge_global_data) + + +def _get_anymail_boto3_params(esp_name=EmailBackend.esp_name, kwargs=None): + """Returns 2 dicts of params for boto3.session.Session() and .client() + + Incorporates ANYMAIL["AMAZON_SES_SESSION_PARAMS"] and + ANYMAIL["AMAZON_SES_CLIENT_PARAMS"] settings. + + Converts config dict to botocore.client.Config if needed + + May remove keys from kwargs, but won't modify original settings + """ + # (shared with ..webhooks.amazon_ses) + session_params = get_anymail_setting( + "session_params", esp_name=esp_name, kwargs=kwargs, default={} + ) + client_params = get_anymail_setting( + "client_params", esp_name=esp_name, kwargs=kwargs, default={} + ) + + # Add Anymail user-agent, and convert config dict to botocore.client.Config + client_params = client_params.copy() # don't modify source + config = Config( + user_agent_extra="django-anymail/{version}-{esp}".format( + esp=esp_name.lower().replace(" ", "-"), version=__version__ + ) + ) + if "config" in client_params: + # convert config dict to botocore.client.Config if needed + client_params_config = client_params["config"] + if not isinstance(client_params_config, Config): + client_params_config = Config(**client_params_config) + config = config.merge(client_params_config) + client_params["config"] = config + + return session_params, client_params diff --git a/docs/esps/amazon_ses.rst b/docs/esps/amazon_ses.rst index 470eae4..9b1c7b7 100644 --- a/docs/esps/amazon_ses.rst +++ b/docs/esps/amazon_ses.rst @@ -3,8 +3,21 @@ Amazon SES ========== -Anymail integrates with `Amazon Simple Email Service`_ (SES) using the `Boto 3`_ -AWS SDK for Python, and includes sending, tracking, and inbound receiving capabilities. +Anymail integrates with the `Amazon Simple Email Service`_ (SES) using the `Boto 3`_ +AWS SDK for Python, and supports sending, tracking, and inbound receiving capabilities. + +.. versionchanged:: 9.1 + +.. note:: + + AWS has two versions of the SES API available for sending email. Anymail 9.0 + and earlier used the SES v1 API. Anymail 9.1 supports both SES v1 and v2, but + support for the v1 API is now deprecated and will be removed in a future Anymail + release (likely in late 2023). + + For new projects, you should use the SES v2 API. For existing projects that are + using the SES v1 API, see :ref:`amazon-ses-v2` below. + .. sidebar:: Alternatives @@ -21,21 +34,23 @@ Installation ------------ You must ensure the :pypi:`boto3` package is installed to use Anymail's Amazon SES -backend. Either include the "amazon_ses" option when you install Anymail: +backend. Either include the ``amazon_ses`` option when you install Anymail: .. code-block:: console $ pip install "django-anymail[amazon_ses]" -or separately run `pip install boto3`. +or separately run ``pip install boto3``. To send mail with Anymail's Amazon SES backend, set: .. code-block:: python - EMAIL_BACKEND = "anymail.backends.amazon_ses.EmailBackend" + EMAIL_BACKEND = "anymail.backends.amazon_sesv2.EmailBackend" -in your settings.py. +in your settings.py. (If you need to use the older Amazon SES v1 API, replace +``amazon_sesv2`` with just ``amazon_ses``---but be aware SES v1 support is +deprecated and will be dropped in a near-future Anymail release.) In addition, you must make sure boto3 is configured with AWS credentials having the necessary :ref:`amazon-ses-iam-permissions`. @@ -48,6 +63,61 @@ setting to customize the Boto session. .. _Credentials: https://boto3.readthedocs.io/en/stable/guide/configuration.html#configuring-credentials +.. _amazon-ses-v2: + +Migrating to the SES v2 API +--------------------------- + +.. versionchanged:: 9.1 + +Anymail is in the process of switching email sending from the original Amazon SES API (v1) +to the updated SES v2 API. Although the capabilities of the two API versions are virtually +identical, Amazon is implementing SES improvements (such as increased maximum message size) +only in the v2 API. + +If you used Anymail 9.0 or earlier to integrate with Amazon SES, you are using the +older SES v1 API. Your code will continue to work with Anymail 9.1, but SES v1 support +is now deprecated and will be removed in a future Anymail release (likely in late 2023). + +Migrating to SES v2 requires minimal code changes: + +1. Update your :ref:`IAM permissions ` to grant Anymail + access to the SES v2 sending actions: ``ses:SendEmail`` for ordinary sends, and/or + ``ses:SendBulkEmail`` to send using SES templates. (The IAM action + prefix is just ``ses`` for both the v1 and v2 APIs.) + + If you run into unexpected IAM authorization failures, see the note about + :ref:`misleading IAM permissions errors ` below. + +2. If your code uses Anymail's :attr:`~anymail.message.AnymailMessage.esp_extra` + to pass additional SES API parameters, or examines the raw + raw :attr:`~anymail.message.AnymailStatus.esp_response` after sending a message, + you may need to update it for the v2 API. Many parameters have different names + in the v2 API compared to the equivalent v1 calls, and the response formats are + slightly different. + + Among v1 parameters commonly used, ``ConfigurationSetName`` is unchanged in v2, + but v1's ``Tags`` and most ``*Arn`` parameters have been renamed in v2. + See AWS's docs for SES v1 `SendRawEmail`_ vs. v2 `SendEmail`_, or if you are sending + with SES templates, compare v1 `SendBulkTemplatedEmail`_ to v2 `SendBulkEmail`_. + +3. In your settings.py, update the :setting:`!EMAIL_BACKEND` to use ``amazon_sesv2``. + Change this: + + .. code-block:: python + + EMAIL_BACKEND = "anymail.backends.amazon_ses.EmailBackend" # SES v1 + + to this: + + .. code-block:: python + + EMAIL_BACKEND = "anymail.backends.amazon_sesv2.EmailBackend" # SES v2 + # ^^ + +The upgrade for SES v2 affects only sending email. There are no changes required +for status tracking webhooks or receiving inbound email. + .. _amazon-ses-quirks: @@ -76,7 +146,7 @@ Limitations and quirks :attr:`~anymail.message.AnymailMessage.track_clicks` are not supported. Although Amazon SES *does* support open and click tracking, it doesn't offer a simple mechanism to override the settings for individual messages. If you - need this feature, provide a custom ConfigurationSetName in Anymail's + need this feature, provide a custom ``ConfigurationSetName`` in Anymail's :ref:`esp_extra `. **No delayed sending** @@ -98,11 +168,19 @@ Limitations and quirks ``message.attach_alternative("...AMPHTML content...", "text/x-amp-html")`` (and be sure to also include regular HTML and text bodies, too). -**Spoofed To header and multiple From emails allowed** +**Envelope-sender is forwarded** + Anymail's :attr:`~anymail.message.AnymailMessage.envelope_sender` becomes + Amazon SES's ``FeedbackForwardingEmailAddress`` (for the SES v2 API; or for SES v1 + either ``Source`` or ``ReturnPath``). That address will receive bounce and other + delivery notifications, but will not appear in the message sent to the recipient. + SES always generates its own anonymized envelope sender (mailfrom) for each outgoing + message, and then forwards that address to your envelope-sender. See + `Email feedback forwarding destination`_ in the SES docs. + +**Spoofed To header allowed** Amazon SES is one of the few ESPs that supports spoofing the :mailheader:`To` header - (see :ref:`message-headers`) and supplying multiple addresses in a message's `from_email`. - (Most ISPs consider these to be very strong spam signals, and using either them will almost - certainly prevent delivery of your mail.) + (see :ref:`message-headers`). (But be aware that most ISPs consider this a strong spam + signal, and using it will likely prevent delivery of your email.) **Template limitations** Messages sent with templates have a number of additional limitations, such as not @@ -112,6 +190,10 @@ Limitations and quirks .. _throttles sending: https://docs.aws.amazon.com/ses/latest/DeveloperGuide/manage-sending-limits.html +.. _Email feedback forwarding destination: + https://docs.aws.amazon.com/ses/latest/dg/monitor-sending-activity-using-notifications-email.html#monitor-sending-activity-using-notifications-email-destination + + .. _amazon-ses-tags: Tags and metadata @@ -168,11 +250,12 @@ setting is disabled by default. If you use it, then only a single tag is support and both the tag and the name must be limited to alphanumeric, hyphen, and underscore characters. -For more complex use cases, set the SES `Tags` parameter directly in Anymail's -:ref:`esp_extra `. See the example below. (Because custom headers do not -work with SES's SendBulkTemplatedEmail call, esp_extra Tags is the only way to attach -data to SES messages also using Anymail's :attr:`~anymail.message.AnymailMessage.template_id` -and :attr:`~anymail.message.AnymailMessage.merge_data` features, and the +For more complex use cases, set the SES ``EmailTags`` parameter (or ``DefaultEmailTags`` +for template sends) directly in Anymail's :ref:`esp_extra `. See +the example below. (Because custom headers do not work with SES's SendBulkEmail call, +esp_extra ``DefaultEmailTags`` is the only way to attach data to SES messages also using +Anymail's :attr:`~anymail.message.AnymailMessage.template_id` and +:attr:`~anymail.message.AnymailMessage.merge_data` features, and :attr:`~anymail.message.AnymailMessage.merge_metadata` cannot be supported.) @@ -191,33 +274,42 @@ esp_extra support To use Amazon SES features not directly supported by Anymail, you can set a message's :attr:`~anymail.message.AnymailMessage.esp_extra` to -a `dict` that will be merged into the params for the `SendRawEmail`_ -or `SendBulkTemplatedEmail`_ SES API call. +a `dict` that will be shallow-merged into the params for the `SendEmail`_ +or `SendBulkEmail`_ SES v2 API call. (Or if you are using the SES v1 API, +`SendRawEmail`_ or `SendBulkTemplatedEmail`_.) -Example: +Examples (for a non-template send using the SES v2 API): .. code-block:: python message.esp_extra = { - # Override AMAZON_SES_CONFIGURATION_SET_NAME for this message + # Override AMAZON_SES_CONFIGURATION_SET_NAME for this message: 'ConfigurationSetName': 'NoOpenOrClickTrackingConfigSet', - # Authorize a custom sender - 'SourceArn': 'arn:aws:ses:us-east-1:123456789012:identity/example.com', - # Set Amazon SES Message Tags - 'Tags': [ + # Authorize a custom sender: + 'FromEmailAddressIdentityArn': 'arn:aws:ses:us-east-1:123456789012:identity/example.com', + # Set SES Message Tags (change to 'DefaultEmailTags' for template sends): + 'EmailTags': [ # (Names and values must be A-Z a-z 0-9 - and _ only) {'Name': 'UserID', 'Value': str(user_id)}, {'Name': 'TestVariation', 'Value': 'Subject-Emoji-Trial-A'}, ], + # Set options for unsubscribe links: + 'ListManagementOptions': { + 'ContactListName': 'RegisteredUsers', + 'TopicName': 'DailyUpdates', + }, } (You can also set `"esp_extra"` in Anymail's :ref:`global send defaults ` to apply it to all messages.) +.. _SendEmail: + https://docs.aws.amazon.com/ses/latest/APIReference-V2/API_SendEmail.html +.. _SendBulkEmail: + https://docs.aws.amazon.com/ses/latest/APIReference-V2/API_SendBulkEmail.html .. _SendRawEmail: https://docs.aws.amazon.com/ses/latest/APIReference/API_SendRawEmail.html - .. _SendBulkTemplatedEmail: https://docs.aws.amazon.com/ses/latest/APIReference/API_SendBulkTemplatedEmail.html @@ -232,8 +324,8 @@ and :ref:`batch sending ` with per-recipient merge data. See Amazon's `Sending personalized email`_ guide for more information. When you set a message's :attr:`~anymail.message.AnymailMessage.template_id` -to the name of one of your SES templates, Anymail will use the SES -`SendBulkTemplatedEmail`_ call to send template messages personalized with data +to the name of one of your SES templates, Anymail will use the SES v2 `SendBulkEmail`_ +(or v1 `SendBulkTemplatedEmail`_) call to send template messages personalized with data from Anymail's normalized :attr:`~anymail.message.AnymailMessage.merge_data` and :attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes. @@ -338,7 +430,8 @@ Finally, switch to Amazon's **Simple Email Service** console: .. code-block:: python ANYMAIL = { - ... + # ... other settings ... + # Use the name from step 5a above: "AMAZON_SES_CONFIGURATION_SET_NAME": "TrackingConfigSet", } @@ -652,7 +745,12 @@ IAM permissions Anymail requires IAM permissions that will allow it to use these actions: -* To send mail: +* To send mail with the SES v2 API: + + * Ordinary (non-templated) sends: ``ses:SendEmail`` + * Template/merge sends: ``ses:SendBulkEmail`` + +* To send mail with the older SES v1 API (deprecated in Anymail 9.1): * Ordinary (non-templated) sends: ``ses:SendRawEmail`` * Template/merge sends: ``ses:SendBulkTemplatedEmail`` @@ -677,6 +775,10 @@ This IAM policy covers all of those: { "Version": "2012-10-17", "Statement": [{ + "Effect": "Allow", + "Action": ["ses:SendEmail", "ses:SendBulkEmail"], + "Resource": "*" + }, { "Effect": "Allow", "Action": ["ses:SendRawEmail", "ses:SendBulkTemplatedEmail"], "Resource": "*" @@ -691,11 +793,35 @@ This IAM policy covers all of those: }] } +.. _amazon-ses-iam-errors: + +.. note:: **Misleading IAM error messages** + + Permissions errors for the SES v2 API often refer to the equivalent SES v1 API name, + which can be confusing. For example, this error (emphasis added): + + .. parsed-literal:: + + An error occurred (AccessDeniedException) when calling the **SendEmail** operation: + User 'arn:...' is not authorized to perform **'ses:SendRawEmail'** on resource 'arn:...' + + actually indicates problems with IAM policies for the v2 ``ses:SendEmail`` action, + *not* the v1 ``ses:SendRawEmail`` action. (The correct action appears as the "operation" + in the first line of the error message.) + Following the principle of `least privilege`_, you should omit permissions for any features you aren't using, and you may want to add additional restrictions: +* If you are not using the older Amazon SES v1 API, you can omit permissions + that allow ``ses:SendRawEmail`` and ``ses:SendBulkTemplatedEmail``. (See + :ref:`amazon-ses-v2` above.) + * For Amazon SES sending, you can add conditions to restrict senders, recipients, times, or other properties. See Amazon's `Controlling access to Amazon SES`_ guide. + (Be aware that the SES v2 ``SendBulkEmail`` API does not support condition keys + that restrict email addresses, and using them can cause misleading error messages. + All other SES APIs used by Anymail *do* support address restrictions, including + the SES v2 ``SendEmail`` API used for non-template sends.) * For auto-confirming webhooks, you might limit the resource to SNS topics owned by your AWS account, and/or specific topic names or patterns. E.g., diff --git a/tests/test_amazon_sesv2_backend.py b/tests/test_amazon_sesv2_backend.py new file mode 100644 index 0000000..4084ceb --- /dev/null +++ b/tests/test_amazon_sesv2_backend.py @@ -0,0 +1,959 @@ +import json +from datetime import datetime +from email.encoders import encode_7or8bit +from email.mime.application import MIMEApplication +from unittest.mock import ANY, patch + +from django.core import mail +from django.core.mail import BadHeaderError +from django.test import SimpleTestCase, override_settings, tag + +from anymail.exceptions import AnymailAPIError, AnymailUnsupportedFeature +from anymail.inbound import AnymailInboundMessage +from anymail.message import AnymailMessage, attach_inline_image_file + +from .utils import ( + SAMPLE_IMAGE_FILENAME, + AnymailTestMixin, + sample_image_content, + sample_image_path, +) + + +@tag("amazon_ses") +@override_settings(EMAIL_BACKEND="anymail.backends.amazon_sesv2.EmailBackend") +class AmazonSESBackendMockAPITestCase(AnymailTestMixin, SimpleTestCase): + """TestCase that uses the Amazon SES EmailBackend with a mocked boto3 client""" + + def setUp(self): + super().setUp() + + # Mock boto3.session.Session().client('sesv2').send_raw_email (and any other + # client operations). (We could also use botocore.stub.Stubber, but mock works + # well with our test structure.) + self.patch_boto3_session = patch( + "anymail.backends.amazon_sesv2.boto3.session.Session", autospec=True + ) + self.mock_session = self.patch_boto3_session.start() # boto3.session.Session + self.addCleanup(self.patch_boto3_session.stop) + #: boto3.session.Session().client + self.mock_client = self.mock_session.return_value.client + #: boto3.session.Session().client('sesv2', ...) + self.mock_client_instance = self.mock_client.return_value + self.set_mock_response() + + # Simple message useful for many tests + self.message = mail.EmailMultiAlternatives( + "Subject", "Text Body", "from@example.com", ["to@example.com"] + ) + + DEFAULT_SEND_RESPONSE = { + "MessageId": "1111111111111111-bbbbbbbb-3333-7777-aaaa-eeeeeeeeeeee-000000", + "ResponseMetadata": { + "RequestId": "900dd7f3-0399-4a1b-9d9f-bed91f46924a", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "date": "Tue, 21 Feb 2023 22:59:46 GMT", + "content-type": "application/json", + "content-length": "76", + "connection": "keep-alive", + "x-amzn-requestid": "900dd7f3-0399-4a1b-9d9f-bed91f46924a", + }, + "RetryAttempts": 0, + }, + } + + def set_mock_response(self, response=None, operation_name="send_email"): + mock_operation = getattr(self.mock_client_instance, operation_name) + mock_operation.return_value = response or self.DEFAULT_SEND_RESPONSE + return mock_operation.return_value + + def set_mock_failure(self, response, operation_name="send_email"): + from botocore.exceptions import ClientError + + mock_operation = getattr(self.mock_client_instance, operation_name) + mock_operation.side_effect = ClientError( + response, operation_name=operation_name + ) + + def get_session_params(self): + if self.mock_session.call_args is None: + raise AssertionError("boto3 Session was not created") + (args, kwargs) = self.mock_session.call_args + if args: + raise AssertionError( + "boto3 Session created with unexpected positional args %r" % args + ) + return kwargs + + def get_client_params(self, service="sesv2"): + """Returns kwargs params passed to mock boto3 client constructor + + Fails test if boto3 client wasn't constructed with named service + """ + if self.mock_client.call_args is None: + raise AssertionError("boto3 client was not created") + (args, kwargs) = self.mock_client.call_args + if len(args) != 1: + raise AssertionError( + "boto3 client created with unexpected positional args %r" % args + ) + if args[0] != service: + raise AssertionError( + "boto3 client created with service %r, not %r" % (args[0], service) + ) + return kwargs + + def get_send_params(self, operation_name="send_email"): + """Returns kwargs params passed to the mock send API. + + Fails test if API wasn't called. + """ + self.mock_client.assert_called_with("sesv2", config=ANY) + mock_operation = getattr(self.mock_client_instance, operation_name) + if mock_operation.call_args is None: + raise AssertionError("API was not called") + (args, kwargs) = mock_operation.call_args + return kwargs + + def get_sent_message(self): + """Returns a parsed version of the send_email Content.Raw.Data param""" + params = self.get_send_params(operation_name="send_email") + raw_mime = params["Content"]["Raw"]["Data"] + parsed = AnymailInboundMessage.parse_raw_mime_bytes(raw_mime) + return parsed + + def assert_esp_not_called(self, msg=None, operation_name="send_email"): + mock_operation = getattr(self.mock_client_instance, operation_name) + if mock_operation.called: + raise AssertionError(msg or "ESP API was called and shouldn't have been") + + +@tag("amazon_ses") +class AmazonSESBackendStandardEmailTests(AmazonSESBackendMockAPITestCase): + """Test backend support for Django standard email features""" + + def test_send_mail(self): + """Test basic API for simple send""" + mail.send_mail( + "Subject here", + "Here is the message.", + "from@example.com", + ["to@example.com"], + fail_silently=False, + ) + params = self.get_send_params() + # send_email takes a fully-formatted MIME message. + # This is a simple (if inexact) way to check for expected headers and body: + raw_mime = params["Content"]["Raw"]["Data"] + self.assertIsInstance(raw_mime, bytes) # SendEmail expects Data as bytes + self.assertIn(b"\nFrom: from@example.com\n", raw_mime) + self.assertIn(b"\nTo: to@example.com\n", raw_mime) + self.assertIn(b"\nSubject: Subject here\n", raw_mime) + self.assertIn(b"\n\nHere is the message", raw_mime) + # Destination must include all recipients: + self.assertEqual(params["Destination"], {"ToAddresses": ["to@example.com"]}) + + # Since the SES backend generates the MIME message using Django's + # EmailMessage.message().to_string(), there's not really a need + # to exhaustively test all the various standard email features. + # (EmailMessage.message() is well tested in the Django codebase.) + # Instead, just spot-check a few things... + + def test_destinations(self): + self.message.to = ["to1@example.com", '"Recipient, second" '] + self.message.cc = ["cc1@example.com", "Also cc "] + self.message.bcc = ["bcc1@example.com", "BCC 2 "] + self.message.send() + params = self.get_send_params() + self.assertEqual( + params["Destination"], + { + "ToAddresses": [ + "to1@example.com", + '"Recipient, second" ', + ], + "CcAddresses": ["cc1@example.com", "Also cc "], + "BccAddresses": ["bcc1@example.com", "BCC 2 "], + }, + ) + # Bcc's shouldn't appear in the message itself: + self.assertNotIn(b"bcc", params["Content"]["Raw"]["Data"]) + + def test_non_ascii_headers(self): + self.message.subject = "Thử tin nhắn" # utf-8 in subject header + self.message.to = ['"Người nhận" '] # utf-8 in display name + self.message.cc = ["cc@thư.example.com"] # utf-8 in domain + self.message.send() + params = self.get_send_params() + raw_mime = params["Content"]["Raw"]["Data"] + # Non-ASCII headers must use MIME encoded-word syntax: + self.assertIn(b"\nSubject: =?utf-8?b?VGjhu60gdGluIG5o4bqvbg==?=\n", raw_mime) + # Non-ASCII display names as well: + self.assertIn( + b"\nTo: =?utf-8?b?TmfGsOG7nWkgbmjhuq1u?= \n", raw_mime + ) + # Non-ASCII address domains must use Punycode: + self.assertIn(b"\nCc: cc@xn--th-e0a.example.com\n", raw_mime) + # SES doesn't support non-ASCII in the username@ part + # (RFC 6531 "SMTPUTF8" extension) + + # Destinations must include all recipients (addr-spec only, must use Punycode): + self.assertEqual( + params["Destination"], + { + "ToAddresses": ["=?utf-8?b?TmfGsOG7nWkgbmjhuq1u?= "], + "CcAddresses": ["cc@xn--th-e0a.example.com"], + }, + ) + + def test_attachments(self): + # These are \u2022 bullets ("\N{BULLET}") below: + text_content = "• Item one\n• Item two\n• Item three" + self.message.attach( + filename="Une pièce jointe.txt", # utf-8 chars in filename + content=text_content, + mimetype="text/plain", + ) + + # Should guess mimetype if not provided... + png_content = b"PNG\xb4 pretend this is the contents of a png file" + self.message.attach(filename="test.png", content=png_content) + + # Should work with a MIMEBase object (also tests no filename)... + pdf_content = b"PDF\xb4 pretend this is valid pdf params" + mimeattachment = MIMEApplication(pdf_content, "pdf") # application/pdf + mimeattachment["Content-Disposition"] = "attachment" + self.message.attach(mimeattachment) + + self.message.send() + sent_message = self.get_sent_message() + attachments = sent_message.attachments + self.assertEqual(len(attachments), 3) + + self.assertEqual(attachments[0].get_content_type(), "text/plain") + self.assertEqual(attachments[0].get_filename(), "Une pièce jointe.txt") + self.assertEqual(attachments[0].get_param("charset"), "utf-8") + self.assertEqual(attachments[0].get_content_text(), text_content) + + self.assertEqual(attachments[1].get_content_type(), "image/png") + # not inline: + self.assertEqual(attachments[1].get_content_disposition(), "attachment") + self.assertEqual(attachments[1].get_filename(), "test.png") + self.assertEqual(attachments[1].get_content_bytes(), png_content) + + self.assertEqual(attachments[2].get_content_type(), "application/pdf") + self.assertIsNone(attachments[2].get_filename()) # no filename specified + self.assertEqual(attachments[2].get_content_bytes(), pdf_content) + + def test_embedded_images(self): + image_filename = SAMPLE_IMAGE_FILENAME + image_path = sample_image_path(image_filename) + image_data = sample_image_content(image_filename) + + cid = attach_inline_image_file(self.message, image_path, domain="example.com") + html_content = ( + '

This has an inline image.

' % cid + ) + self.message.attach_alternative(html_content, "text/html") + + self.message.send() + sent_message = self.get_sent_message() + + self.assertEqual(sent_message.html, html_content) + + inlines = sent_message.inline_attachments + self.assertEqual(len(inlines), 1) + self.assertEqual(inlines[cid].get_content_type(), "image/png") + self.assertEqual(inlines[cid].get_filename(), image_filename) + self.assertEqual(inlines[cid].get_content_bytes(), image_data) + + # Make sure neither the html nor the inline image is treated as an attachment: + params = self.get_send_params() + raw_mime = params["Content"]["Raw"]["Data"] + self.assertNotIn(b"\nContent-Disposition: attachment", raw_mime) + + def test_multiple_html_alternatives(self): + # Multiple alternatives *are* allowed + self.message.attach_alternative("

First html is OK

", "text/html") + self.message.attach_alternative("

And so is second

", "text/html") + self.message.send() + params = self.get_send_params() + raw_mime = params["Content"]["Raw"]["Data"] + # just check the alternative smade it into the message + # (assume that Django knows how to format them properly) + self.assertIn(b"\n\n

First html is OK

\n", raw_mime) + self.assertIn(b"\n\n

And so is second

\n", raw_mime) + + def test_alternative(self): + # Non-HTML alternatives (including AMP) *are* allowed + self.message.attach_alternative("

AMP HTML

", "text/x-amp-html") + self.message.send() + params = self.get_send_params() + raw_mime = params["Content"]["Raw"]["Data"] + # just check the alternative made it into the message + # (assume that Python email knows how to format it properly) + self.assertIn(b"\nContent-Type: text/x-amp-html", raw_mime) + + def test_multiple_from(self): + # Amazon allows multiple addresses in the From header, + # but must specify a single one for the FromEmailAddress + self.message.from_email = "First , from2@example.com" + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "multiple from emails" + ): + self.message.send() + + def test_commas_in_subject(self): + """ + There used to be a Python email header bug that added unwanted spaces + after commas in long subjects + """ + self.message.subject = ( + "100,000,000 isn't a number you'd really want" + " to break up in this email subject, right?" + ) + self.message.send() + sent_message = self.get_sent_message() + self.assertEqual(sent_message["Subject"], self.message.subject) + + def test_no_cte_8bit(self): + """Anymail works around an Amazon SES bug that can corrupt non-ASCII bodies.""" + # (see detailed comments in the backend code) + + # The generated MIMEText for each of these ends up using CTE 8bit by default: + self.message.body = "Это text body" + self.message.attach_alternative("

Это html body

", "text/html") + self.message.attach("sample.csv", "Это attachment", "text/csv") + + # Also force a CTE 8bit attachment (which normally defaults to CTE base64): + att = MIMEApplication("Это data".encode("utf8"), "data", encode_7or8bit) + self.assertEqual(att["Content-Transfer-Encoding"], "8bit") + self.message.attach(att) + + self.message.send() + sent_message = self.get_sent_message() + + # Make sure none of the resulting parts use `Content-Transfer-Encoding: 8bit`. + # (Technically, either quoted-printable or base64 would be OK, but base64 text + # parts have a reputation for triggering spam filters, so just require + # quoted-printable for them.) + part_encodings = [ + (part.get_content_type(), part["Content-Transfer-Encoding"]) + for part in sent_message.walk() + ] + self.assertEqual( + part_encodings, + [ + ("multipart/mixed", None), + ("multipart/alternative", None), + ("text/plain", "quoted-printable"), + ("text/html", "quoted-printable"), + ("text/csv", "quoted-printable"), + ("application/data", "base64"), + ], + ) + + def test_api_failure(self): + error_response = { + "Error": { + "Code": "MessageRejected", + "Message": "Email address is not verified. The following identities" + " failed the check in region US-EAST-1: to@example.com", + }, + "ResponseMetadata": { + "RequestId": "c44b0ae2-e086-45ca-8820-b76a9b9f430a", + "HTTPStatusCode": 403, + "HTTPHeaders": { + "date": "Tue, 21 Feb 2023 23:49:31 GMT", + "content-type": "application/json", + "content-length": "196", + "connection": "keep-alive", + "x-amzn-requestid": "c44b0ae2-e086-45ca-8820-b76a9b9f430a", + "x-amzn-errortype": "MessageRejected", + }, + "RetryAttempts": 0, + }, + } + + self.set_mock_failure(error_response) + with self.assertRaises(AnymailAPIError) as cm: + self.message.send() + err = cm.exception + # AWS error is included in Anymail message: + self.assertIn( + "Email address is not verified. The following identities failed " + "the check in region US-EAST-1: to@example.com", + str(err), + ) + # Raw AWS response is available on the exception: + self.assertEqual(err.response, error_response) + + def test_api_failure_fail_silently(self): + # Make sure fail_silently is respected + self.set_mock_failure( + { + "Error": { + "Type": "Sender", + "Code": "InvalidParameterValue", + "Message": "That is not allowed", + } + } + ) + sent = self.message.send(fail_silently=True) + self.assertEqual(sent, 0) + + def test_prevents_header_injection(self): + # Since we build the raw MIME message, we're responsible for preventing header + # injection. django.core.mail.EmailMessage.message() implements most of that + # (for the SMTP backend); spot check some likely cases just to be sure... + with self.assertRaises(BadHeaderError): + mail.send_mail( + "Subject\r\ninjected", "Body", "from@example.com", ["to@example.com"] + ) + with self.assertRaises(BadHeaderError): + mail.send_mail( + "Subject", + "Body", + '"Display-Name\nInjected" ', + ["to@example.com"], + ) + with self.assertRaises(BadHeaderError): + mail.send_mail( + "Subject", + "Body", + "from@example.com", + ['"Display-Name\rInjected" '], + ) + with self.assertRaises(BadHeaderError): + mail.EmailMessage( + "Subject", + "Body", + "from@example.com", + ["to@example.com"], + headers={"X-Header": "custom header value\r\ninjected"}, + ).send() + + +@tag("amazon_ses") +class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase): + """Test backend support for Anymail added features""" + + def test_envelope_sender(self): + self.message.envelope_sender = "bounce-handler@bounces.example.com" + self.message.send() + params = self.get_send_params() + self.assertEqual( + params["FeedbackForwardingEmailAddress"], + "bounce-handler@bounces.example.com", + ) + + def test_spoofed_to(self): + # Amazon SES is one of the few ESPs that actually permits the To header + # to differ from the envelope recipient... + self.message.to = ["Envelope "] + self.message.extra_headers["To"] = "Spoofed " + self.message.send() + params = self.get_send_params() + raw_mime = params["Content"]["Raw"]["Data"] + self.assertEqual( + params["Destination"], + {"ToAddresses": ["Envelope "]}, + ) + self.assertIn(b"\nTo: Spoofed \n", raw_mime) + self.assertNotIn(b"envelope-to@example.com", raw_mime) + + def test_metadata(self): + self.message.metadata = { + "User ID": 12345, + # that \n is a header-injection test: + "items": "Correct horse,Battery,\nStaple", + "Cart-Total": "22.70", + } + self.message.send() + + # Metadata is passed as JSON in a message header field: + sent_message = self.get_sent_message() + self.assertJSONEqual( + sent_message["X-Metadata"], + '{"User ID": 12345,' + ' "items": "Correct horse,Battery,\\nStaple",' + ' "Cart-Total": "22.70"}', + ) + + def test_send_at(self): + # Amazon SES does not support delayed sending + self.message.send_at = datetime(2016, 3, 4, 5, 6, 7) + with self.assertRaisesMessage(AnymailUnsupportedFeature, "send_at"): + self.message.send() + + def test_tags(self): + self.message.tags = ["Transactional", "Cohort 12/2017"] + self.message.send() + + # Tags are added as multiple X-Tag message headers: + sent_message = self.get_sent_message() + self.assertCountEqual( + sent_message.get_all("X-Tag"), ["Transactional", "Cohort 12/2017"] + ) + + # Tags are *not* by default used as Amazon SES "Message Tags": + params = self.get_send_params() + self.assertNotIn("Tags", params) + + @override_settings(ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign") + def test_amazon_message_tags(self): + """ + The Anymail AMAZON_SES_MESSAGE_TAG_NAME setting enables a single Message Tag + """ + self.message.tags = ["Welcome"] + self.message.send() + params = self.get_send_params() + self.assertEqual( + params["EmailTags"], [{"Name": "Campaign", "Value": "Welcome"}] + ) + + # Multiple Anymail tags are not supported when using this feature + self.message.tags = ["Welcome", "Variation_A"] + with self.assertRaisesMessage( + AnymailUnsupportedFeature, + "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting", + ): + self.message.send() + + def test_tracking(self): + # Amazon SES doesn't support overriding click/open-tracking settings + # on individual messages through any standard API params. + # (You _can_ use a ConfigurationSet to control this; see esp_extra below.) + self.message.track_clicks = True + with self.assertRaisesMessage(AnymailUnsupportedFeature, "track_clicks"): + self.message.send() + delattr(self.message, "track_clicks") + + self.message.track_opens = True + with self.assertRaisesMessage(AnymailUnsupportedFeature, "track_opens"): + self.message.send() + + def test_merge_data(self): + # Amazon SES only supports merging when using templates (see below) + self.message.merge_data = {} + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "merge_data without template_id" + ): + self.message.send() + delattr(self.message, "merge_data") + + self.message.merge_global_data = {"group": "Users", "site": "ExampleCo"} + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "global_merge_data without template_id" + ): + self.message.send() + + @override_settings( + # only way to use tags with template_id: + ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign" + ) + def test_template(self): + """With template_id, Anymail switches to SESv2 SendBulkEmail""" + # SendBulkEmail uses a completely different API call and payload + # structure, so this re-tests a bunch of Anymail features that were handled + # differently above. (See test_amazon_ses_integration for a more realistic + # template example.) + raw_response = { + "BulkEmailEntryResults": [ + { + "Status": "SUCCESS", + "MessageId": "1111111111111111-bbbbbbbb-3333-7777", + }, + { + "Status": "ACCOUNT_DAILY_QUOTA_EXCEEDED", + "Error": "Daily message quota exceeded", + }, + ], + "ResponseMetadata": self.DEFAULT_SEND_RESPONSE["ResponseMetadata"], + } + self.set_mock_response(raw_response, operation_name="send_bulk_email") + message = AnymailMessage( + template_id="welcome_template", + from_email='"Example, Inc." ', + to=["alice@example.com", "罗伯特 "], + cc=["cc@example.com"], + reply_to=["reply1@example.com", "Reply 2 "], + merge_data={ + "alice@example.com": {"name": "Alice", "group": "Developers"}, + "bob@example.com": {"name": "Bob"}, # and leave group undefined + "nobody@example.com": {"name": "Not a recipient for this message"}, + }, + merge_global_data={"group": "Users", "site": "ExampleCo"}, + # (only works with AMAZON_SES_MESSAGE_TAG_NAME when using template): + tags=["WelcomeVariantA"], + envelope_sender="bounce@example.com", + esp_extra={ + "FromEmailAddressIdentityArn": ( + "arn:aws:ses:us-east-1:123456789012:identity/example.com" + ) + }, + ) + message.send() + + # templates use a different API call... + self.assert_esp_not_called(operation_name="send_email") + params = self.get_send_params(operation_name="send_bulk_email") + self.assertEqual( + params["DefaultContent"]["Template"]["TemplateName"], "welcome_template" + ) + self.assertEqual( + params["FromEmailAddress"], '"Example, Inc." ' + ) + bulk_entries = params["BulkEmailEntries"] + self.assertEqual(len(bulk_entries), 2) + self.assertEqual( + bulk_entries[0]["Destination"], + {"ToAddresses": ["alice@example.com"], "CcAddresses": ["cc@example.com"]}, + ) + self.assertEqual( + json.loads( + bulk_entries[0]["ReplacementEmailContent"]["ReplacementTemplate"][ + "ReplacementTemplateData" + ] + ), + {"name": "Alice", "group": "Developers"}, + ) + self.assertEqual( + bulk_entries[1]["Destination"], + { + # SES requires RFC2047: + "ToAddresses": ["=?utf-8?b?572X5Lyv54m5?= "], + "CcAddresses": ["cc@example.com"], + }, + ) + self.assertEqual( + json.loads( + bulk_entries[1]["ReplacementEmailContent"]["ReplacementTemplate"][ + "ReplacementTemplateData" + ] + ), + {"name": "Bob"}, + ) + self.assertEqual( + json.loads(params["DefaultContent"]["Template"]["TemplateData"]), + {"group": "Users", "site": "ExampleCo"}, + ) + self.assertEqual( + params["ReplyToAddresses"], + ["reply1@example.com", "Reply 2 "], + ) + self.assertEqual( + params["DefaultEmailTags"], + [{"Name": "Campaign", "Value": "WelcomeVariantA"}], + ) + self.assertEqual(params["FeedbackForwardingEmailAddress"], "bounce@example.com") + # esp_extra: + self.assertEqual( + params["FromEmailAddressIdentityArn"], + "arn:aws:ses:us-east-1:123456789012:identity/example.com", + ) + + self.assertEqual(message.anymail_status.status, {"queued", "failed"}) + self.assertEqual( + # different for each recipient + message.anymail_status.message_id, + {"1111111111111111-bbbbbbbb-3333-7777", None}, + ) + self.assertEqual( + message.anymail_status.recipients["alice@example.com"].status, "queued" + ) + self.assertEqual( + message.anymail_status.recipients["bob@example.com"].status, "failed" + ) + self.assertEqual( + message.anymail_status.recipients["alice@example.com"].message_id, + "1111111111111111-bbbbbbbb-3333-7777", + ) + self.assertIsNone( + message.anymail_status.recipients["bob@example.com"].message_id + ) + self.assertEqual(message.anymail_status.esp_response, raw_response) + + def test_template_failure(self): + """Failures to all recipients raise a similar error to non-template sends""" + raw_response = { + "BulkEmailEntryResults": [ + { + "Status": "TEMPLATE_DOES_NOT_EXIST", + "Error": "No template named 'oops'", + }, + { + "Status": "TEMPLATE_DOES_NOT_EXIST", + "Error": "No template named 'oops'", + }, + ], + "ResponseMetadata": self.DEFAULT_SEND_RESPONSE["ResponseMetadata"], + } + self.set_mock_response(raw_response, operation_name="send_bulk_email") + message = AnymailMessage( + template_id="oops", + from_email="from@example.com", + to=["alice@example.com", "bob@example.com"], + ) + with self.assertRaisesMessage(AnymailAPIError, "No template named 'oops'"): + message.send() + + def test_template_unsupported(self): + """A lot of options are not compatible with SendBulkTemplatedEmail""" + message = AnymailMessage(template_id="welcome_template", to=["to@example.com"]) + + message.subject = "nope, can't change template subject" + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "overriding template subject" + ): + message.send() + message.subject = None + + message.body = "nope, can't change text body" + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "overriding template body content" + ): + message.send() + message.content_subtype = "html" + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "overriding template body content" + ): + message.send() + message.body = None + + message.attach("attachment.txt", "this is an attachment", "text/plain") + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "attachments with template" + ): + message.send() + message.attachments = [] + + message.extra_headers = {"X-Custom": "header"} + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "extra_headers with template" + ): + message.send() + message.extra_headers = {} + + message.metadata = {"meta": "data"} + with self.assertRaisesMessage( + AnymailUnsupportedFeature, "metadata with template" + ): + message.send() + message.metadata = None + + message.tags = ["tag 1", "tag 2"] + with self.assertRaisesMessage(AnymailUnsupportedFeature, "tags with template"): + message.send() + message.tags = None + + def test_send_anymail_message_without_template(self): + # Make sure SendEmail is used for non-template_id messages + message = AnymailMessage( + from_email="from@example.com", to=["to@example.com"], subject="subject" + ) + message.send() + self.assert_esp_not_called(operation_name="send_bulk_email") + # fails if send_email not called: + self.get_send_params(operation_name="send_email") + + def test_default_omits_options(self): + """Make sure by default we don't send any ESP-specific options. + + Options not specified by the caller should be omitted entirely from + the API call (*not* sent as False or empty). This ensures + that your ESP account settings apply by default. + """ + self.message.send() + params = self.get_send_params() + self.assertNotIn("BulkEmailEntries", params) + self.assertNotIn("ConfigurationSetName", params) + self.assertNotIn("DefaultContent", params) + self.assertNotIn("DefaultContent", params) + self.assertNotIn("DefaultEmailTags", params) + self.assertNotIn("EmailTags", params) + self.assertNotIn("FeedbackForwardingEmailAddress", params) + self.assertNotIn("FeedbackForwardingEmailAddressIdentityArn", params) + self.assertNotIn("FromEmailAddressIdentityArn", params) + self.assertNotIn("ListManagementOptions", params) + self.assertNotIn("ReplyToAddresses", params) + + sent_message = self.get_sent_message() + # custom headers not added if not needed: + self.assertNotIn("X-Metadata", sent_message) + self.assertNotIn("X-Tag", sent_message) + + def test_esp_extra(self): + # Values in esp_extra are merged into the Amazon SES SendRawEmail parameters + self.message.esp_extra = { + # E.g., if you've set up a configuration set + # that disables open/click tracking: + "ConfigurationSetName": "NoTrackingConfigurationSet", + } + self.message.send() + params = self.get_send_params() + self.assertEqual(params["ConfigurationSetName"], "NoTrackingConfigurationSet") + + def test_send_attaches_anymail_status(self): + """The anymail_status should be attached to the message when it is sent""" + msg = mail.EmailMessage( + "Subject", + "Message", + "from@example.com", + ["to1@example.com"], + ) + sent = msg.send() + self.assertEqual(sent, 1) + self.assertEqual(msg.anymail_status.status, {"queued"}) + self.assertEqual( + msg.anymail_status.message_id, + "1111111111111111-bbbbbbbb-3333-7777-aaaa-eeeeeeeeeeee-000000", + ) + self.assertEqual( + msg.anymail_status.recipients["to1@example.com"].status, "queued" + ) + self.assertEqual( + msg.anymail_status.recipients["to1@example.com"].message_id, + "1111111111111111-bbbbbbbb-3333-7777-aaaa-eeeeeeeeeeee-000000", + ) + self.assertEqual(msg.anymail_status.esp_response, self.DEFAULT_SEND_RESPONSE) + + # Amazon SES doesn't report rejected addresses at send time in a form that can be + # distinguished from other API errors. If SES rejects *any* recipient you'll get + # an AnymailAPIError, and the message won't be sent to *all* recipients. + + # noinspection PyUnresolvedReferences + def test_send_unparsable_response(self): + """ + If the send succeeds, but result is unexpected format, + should raise an API exception + """ + response_content = {"wrong": "format"} + self.set_mock_response(response_content) + with self.assertRaisesMessage( + AnymailAPIError, "parsing Amazon SES send result" + ): + self.message.send() + self.assertIsNone(self.message.anymail_status.status) + self.assertIsNone(self.message.anymail_status.message_id) + self.assertEqual(self.message.anymail_status.recipients, {}) + self.assertEqual(self.message.anymail_status.esp_response, response_content) + + +@tag("amazon_ses") +class AmazonSESBackendConfigurationTests(AmazonSESBackendMockAPITestCase): + """Test configuration options""" + + def test_boto_default_config(self): + """By default, boto3 gets credentials from the environment or its config files + + See http://boto3.readthedocs.io/en/stable/guide/configuration.html + """ + self.message.send() + + session_params = self.get_session_params() + # no additional params passed to boto3.session.Session(): + self.assertEqual(session_params, {}) + + client_params = self.get_client_params() + # Ignore botocore.config.Config, which doesn't support == + config = client_params.pop("config") + # no additional params passed to session.client('ses'): + self.assertEqual(client_params, {}) + self.assertRegex( + config.user_agent_extra, r"django-anymail/\d(\.\w+){1,}-amazon-ses" + ) + + @override_settings( + ANYMAIL={ + "AMAZON_SES_CLIENT_PARAMS": { + # Example for testing; it's not a good idea to hardcode credentials in + # your code. Safer: `os.getenv("MY_SPECIAL_AWS_KEY_ID")` etc. + "aws_access_key_id": "test-access-key-id", + "aws_secret_access_key": "test-secret-access-key", + "region_name": "ap-northeast-1", + # config can be given as dict of botocore.config.Config params + "config": { + "read_timeout": 30, + "retries": {"max_attempts": 2}, + }, + } + } + ) + def test_client_params_in_setting(self): + """ + The Anymail AMAZON_SES_CLIENT_PARAMS setting specifies + boto3 session.client() params for Anymail + """ + self.message.send() + client_params = self.get_client_params() + # Ignore botocore.config.Config, which doesn't support == + config = client_params.pop("config") + self.assertEqual( + client_params, + { + "aws_access_key_id": "test-access-key-id", + "aws_secret_access_key": "test-secret-access-key", + "region_name": "ap-northeast-1", + }, + ) + self.assertEqual(config.read_timeout, 30) + self.assertEqual(config.retries, {"max_attempts": 2}) + + def test_client_params_in_connection_init(self): + """ + You can also supply credentials specifically + for a particular EmailBackend connection instance + """ + from botocore.config import Config + + boto_config = Config(connect_timeout=30) + conn = mail.get_connection( + "anymail.backends.amazon_sesv2.EmailBackend", + client_params={ + "aws_session_token": "test-session-token", + "config": boto_config, + }, + ) + conn.send_messages([self.message]) + + client_params = self.get_client_params() + # Ignore botocore.config.Config, which doesn't support == + config = client_params.pop("config") + self.assertEqual(client_params, {"aws_session_token": "test-session-token"}) + self.assertEqual(config.connect_timeout, 30) + + @override_settings( + ANYMAIL={"AMAZON_SES_SESSION_PARAMS": {"profile_name": "anymail-testing"}} + ) + def test_session_params_in_setting(self): + """ + The Anymail AMAZON_SES_SESSION_PARAMS setting + specifies boto3.session.Session() params for Anymail + """ + self.message.send() + + session_params = self.get_session_params() + self.assertEqual(session_params, {"profile_name": "anymail-testing"}) + + client_params = self.get_client_params() + # Ignore botocore.config.Config, which doesn't support == + client_params.pop("config") + # no additional params passed to session.client('ses'): + self.assertEqual(client_params, {}) + + @override_settings( + ANYMAIL={"AMAZON_SES_CONFIGURATION_SET_NAME": "MyConfigurationSet"} + ) + def test_config_set_setting(self): + """You can supply a default ConfigurationSetName""" + self.message.send() + params = self.get_send_params() + self.assertEqual(params["ConfigurationSetName"], "MyConfigurationSet") + + # override on individual message using esp_extra + self.message.esp_extra = {"ConfigurationSetName": "CustomConfigurationSet"} + self.message.send() + params = self.get_send_params() + self.assertEqual(params["ConfigurationSetName"], "CustomConfigurationSet") diff --git a/tests/test_amazon_sesv2_integration.py b/tests/test_amazon_sesv2_integration.py new file mode 100644 index 0000000..c66ad46 --- /dev/null +++ b/tests/test_amazon_sesv2_integration.py @@ -0,0 +1,199 @@ +import os +import unittest +from email.utils import formataddr + +from django.test import SimpleTestCase, override_settings, tag + +from anymail.exceptions import AnymailAPIError +from anymail.message import AnymailMessage + +from .utils import AnymailTestMixin, sample_image_path + +ANYMAIL_TEST_AMAZON_SES_ACCESS_KEY_ID = os.getenv( + "ANYMAIL_TEST_AMAZON_SES_ACCESS_KEY_ID" +) +ANYMAIL_TEST_AMAZON_SES_SECRET_ACCESS_KEY = os.getenv( + "ANYMAIL_TEST_AMAZON_SES_SECRET_ACCESS_KEY" +) +ANYMAIL_TEST_AMAZON_SES_REGION_NAME = os.getenv( + "ANYMAIL_TEST_AMAZON_SES_REGION_NAME", "us-east-1" +) +ANYMAIL_TEST_AMAZON_SES_DOMAIN = os.getenv("ANYMAIL_TEST_AMAZON_SES_DOMAIN") + + +@unittest.skipUnless( + ANYMAIL_TEST_AMAZON_SES_ACCESS_KEY_ID + and ANYMAIL_TEST_AMAZON_SES_SECRET_ACCESS_KEY + and ANYMAIL_TEST_AMAZON_SES_DOMAIN, + "Set ANYMAIL_TEST_AMAZON_SES_ACCESS_KEY_ID and" + " ANYMAIL_TEST_AMAZON_SES_SECRET_ACCESS_KEY and ANYMAIL_TEST_AMAZON_SES_DOMAIN" + " environment variables to run Amazon SES integration tests", +) +@override_settings( + EMAIL_BACKEND="anymail.backends.amazon_sesv2.EmailBackend", + ANYMAIL={ + "AMAZON_SES_CLIENT_PARAMS": { + # This setting provides Anymail-specific AWS credentials to boto3.client(), + # overriding any credentials in the environment or boto config. It's often + # *not* the best approach. See the Anymail and boto3 docs for other options. + "aws_access_key_id": ANYMAIL_TEST_AMAZON_SES_ACCESS_KEY_ID, + "aws_secret_access_key": ANYMAIL_TEST_AMAZON_SES_SECRET_ACCESS_KEY, + "region_name": ANYMAIL_TEST_AMAZON_SES_REGION_NAME, + # Can supply any other boto3.client params, + # including botocore.config.Config as dict + "config": {"retries": {"max_attempts": 2}}, + }, + # actual config set in Anymail test account: + "AMAZON_SES_CONFIGURATION_SET_NAME": "TestConfigurationSet", + }, +) +@tag("amazon_ses", "live") +class AmazonSESBackendIntegrationTests(AnymailTestMixin, SimpleTestCase): + """Amazon SES API integration tests + + These tests run against the **live** Amazon SES API, using the environment + variables `ANYMAIL_TEST_AMAZON_SES_ACCESS_KEY_ID` and + `ANYMAIL_TEST_AMAZON_SES_SECRET_ACCESS_KEY` as AWS credentials. + If those variables are not set, these tests won't run. + + (You can also set the environment variable `ANYMAIL_TEST_AMAZON_SES_REGION_NAME` + to test SES using a region other than the default "us-east-1".) + + Amazon SES doesn't offer a test mode -- it tries to send everything you ask. + To avoid stacking up a pile of undeliverable @example.com + emails, the tests use Amazon's @simulator.amazonses.com addresses. + https://docs.aws.amazon.com/ses/latest/DeveloperGuide/mailbox-simulator.html + """ + + def setUp(self): + super().setUp() + self.from_email = f"test@{ANYMAIL_TEST_AMAZON_SES_DOMAIN}" + self.message = AnymailMessage( + "Anymail Amazon SES integration test", + "Text content", + self.from_email, + ["success@simulator.amazonses.com"], + ) + self.message.attach_alternative("

HTML content

", "text/html") + + def test_simple_send(self): + # Example of getting the Amazon SES send status and message id from the message + sent_count = self.message.send() + self.assertEqual(sent_count, 1) + + anymail_status = self.message.anymail_status + sent_status = anymail_status.recipients[ + "success@simulator.amazonses.com" + ].status + message_id = anymail_status.recipients[ + "success@simulator.amazonses.com" + ].message_id + + # Amazon SES always queues (or raises an error): + self.assertEqual(sent_status, "queued") + # Amazon SES message ids are groups of hex chars: + self.assertRegex(message_id, r"[0-9a-f-]+") + # set of all recipient statuses: + self.assertEqual(anymail_status.status, {sent_status}) + self.assertEqual(anymail_status.message_id, message_id) + + def test_all_options(self): + message = AnymailMessage( + subject="Anymail Amazon SES all-options integration test", + body="This is the text body", + from_email=formataddr(("Test From, with comma", self.from_email)), + to=[ + "success+to1@simulator.amazonses.com", + "Recipient 2 ", + ], + cc=[ + "success+cc1@simulator.amazonses.com", + "Copy 2 ", + ], + bcc=[ + "success+bcc1@simulator.amazonses.com", + "Blind Copy 2 ", + ], + reply_to=["reply1@example.com", "Reply 2 "], + headers={"X-Anymail-Test": "value"}, + metadata={"meta1": "simple_string", "meta2": 2}, + tags=["Re-engagement", "Cohort 12/2017"], + envelope_sender=f"bounce-handler@{ANYMAIL_TEST_AMAZON_SES_DOMAIN}", + ) + message.attach("attachment1.txt", "Here is some\ntext for you", "text/plain") + message.attach("attachment2.csv", "ID,Name\n1,Amy Lina", "text/csv") + cid = message.attach_inline_image_file(sample_image_path()) + message.attach_alternative( + "

HTML: with link" + "and image: " % cid, + "text/html", + ) + + message.attach_alternative( + "Amazon SES SendRawEmail actually supports multiple alternative parts", + "text/x-note-for-email-geeks", + ) + + message.send() + self.assertEqual(message.anymail_status.status, {"queued"}) + + def test_stored_template(self): + # Using a template created like this: + # boto3.client('sesv2').create_email_template( + # TemplateName="TestTemplate", + # TemplateContent={ + # "Subject": "Your order {{order}} shipped", + # "Html": "

Dear {{name}}:

" + # "

Your order {{order}} shipped {{ship_date}}.

", + # "Text": "Dear {{name}}:\r\n" + # "Your order {{order}} shipped {{ship_date}}." + # }, + # ) + message = AnymailMessage( + template_id="TestTemplate", + from_email=formataddr(("Test From", self.from_email)), + to=[ + "First Recipient ", + "success+to2@simulator.amazonses.com", + ], + merge_data={ + "success+to1@simulator.amazonses.com": { + "order": 12345, + "name": "Test Recipient", + }, + "success+to2@simulator.amazonses.com": {"order": 6789}, + }, + merge_global_data={"name": "Customer", "ship_date": "today"}, # default + ) + message.send() + recipient_status = message.anymail_status.recipients + self.assertEqual( + recipient_status["success+to1@simulator.amazonses.com"].status, "queued" + ) + self.assertRegex( + recipient_status["success+to1@simulator.amazonses.com"].message_id, + r"[0-9a-f-]+", + ) + self.assertEqual( + recipient_status["success+to2@simulator.amazonses.com"].status, "queued" + ) + self.assertRegex( + recipient_status["success+to2@simulator.amazonses.com"].message_id, + r"[0-9a-f-]+", + ) + + @override_settings( + ANYMAIL={ + "AMAZON_SES_CLIENT_PARAMS": { + "aws_access_key_id": "test-invalid-access-key-id", + "aws_secret_access_key": "test-invalid-secret-access-key", + "region_name": ANYMAIL_TEST_AMAZON_SES_REGION_NAME, + } + } + ) + def test_invalid_aws_credentials(self): + # Make sure the exception message includes AWS's response: + with self.assertRaisesMessage( + AnymailAPIError, "The security token included in the request is invalid" + ): + self.message.send()