Amazon SES: support headers with template

Use new SES v2 SendBulkEmail ReplacementHeaders param
to support features that require custom headers,
including `extra_headers`, `metadata`,
`merge_metadata` and `tags`.

Update integration tests and docs

Closes #375

CHANGELOG.rst

@@ -48,6 +48,10 @@ Features
   headers with template sends. (Requires boto3 >= 1.34.98.)
   (Thanks to `@carrerasrodrigo`_ the implementation.)
 
+* **Amazon SES:** Allow extra headers, ``metadata``, ``merge_metadata``,
+  and ``tags`` when sending with a ``template_id``.
+  (Requires boto3 v1.34.98 or later.)
+
 
 v10.3
 -----

anymail/backends/amazon_ses.py

@@ -2,6 +2,8 @@ import email.charset
 import email.encoders
 import email.policy
 
+from requests.structures import CaseInsensitiveDict
+
 from .. import __version__ as ANYMAIL_VERSION
 from ..exceptions import AnymailAPIError, AnymailImproperlyInstalled
 from ..message import AnymailRecipientStatus
@@ -339,10 +341,14 @@ class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload):
 
     def init_payload(self):
         super().init_payload()
-        # late-bind recipients and merge_data in finalize_payload
+        # late-bind in finalize_payload:
         self.recipients = {"to": [], "cc": [], "bcc": []}
         self.merge_data = {}
+        self.headers = {}
         self.merge_headers = {}
+        self.metadata = {}
+        self.merge_metadata = {}
+        self.tags = []
 
     def finalize_payload(self):
         # Build BulkEmailEntries from recipients and merge_data.
@@ -372,11 +378,26 @@ class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload):
                 },
             }
 
-            if len(self.merge_headers) > 0:
+            replacement_headers = []
-                entry["ReplacementHeaders"] = [
+            if self.headers or to.addr_spec in self.merge_headers:
-                    {"Name": key, "Value": value}
+                headers = CaseInsensitiveDict(self.headers)
-                    for key, value in self.merge_headers.get(to.addr_spec, {}).items()
+                headers.update(self.merge_headers.get(to.addr_spec, {}))
+                replacement_headers += [
+                    {"Name": key, "Value": value} for key, value in headers.items()
                 ]
+            if self.metadata or to.addr_spec in self.merge_metadata:
+                metadata = self.metadata.copy()
+                metadata.update(self.merge_metadata.get(to.addr_spec, {}))
+                if metadata:
+                    replacement_headers.append(
+                        {"Name": "X-Metadata", "Value": self.serialize_json(metadata)}
+                    )
+            if self.tags:
+                replacement_headers += [
+                    {"Name": "X-Tag", "Value": tag} for tag in self.tags
+                ]
+            if replacement_headers:
+                entry["ReplacementHeaders"] = replacement_headers
             self.params["BulkEmailEntries"].append(entry)
 
     def parse_recipient_status(self, response):
@@ -446,7 +467,7 @@ class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload):
             self.params["ReplyToAddresses"] = [email.address for email in emails]
 
     def set_extra_headers(self, headers):
-        self.unsupported_feature("extra_headers with template")
+        self.headers = headers
 
     def set_text_body(self, body):
         if body:
@@ -468,15 +489,19 @@ class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload):
         self.params["FeedbackForwardingEmailAddress"] = email.addr_spec
 
     def set_metadata(self, metadata):
-        # no custom headers with SendBulkEmail
+        self.metadata = metadata
-        self.unsupported_feature("metadata with template")
+
+    def set_merge_metadata(self, merge_metadata):
+        self.merge_metadata = merge_metadata
 
     def set_tags(self, tags):
-        # no custom headers with SendBulkEmail, but support
+        self.tags = tags
-        # AMAZON_SES_MESSAGE_TAG_NAME if used (see tags/metadata in
+
-        # AmazonSESV2SendEmailPayload for more info)
+        # Also *optionally* pass a single Message Tag if the AMAZON_SES_MESSAGE_TAG_NAME
-        if tags:
+        # Anymail setting is set (default no). The AWS API restricts tag content in this
-            if self.backend.message_tag_name is not None:
+        # case. (This is useful for dashboard segmentation; use esp_extra["Tags"] for
+        # anything more complex.)
+        if tags and self.backend.message_tag_name is not None:
             if len(tags) > 1:
                 self.unsupported_feature(
                     "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting"
@@ -484,11 +509,6 @@ class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload):
             self.params["DefaultEmailTags"] = [
                 {"Name": self.backend.message_tag_name, "Value": tags[0]}
             ]
-            else:
-                self.unsupported_feature(
-                    "tags with template (unless using the"
-                    " AMAZON_SES_MESSAGE_TAG_NAME setting)"
-                )
 
     def set_template_id(self, template_id):
         # DefaultContent.Template.TemplateName

docs/esps/amazon_ses.rst

@@ -68,6 +68,11 @@ setting to customize the Boto session.
 Limitations and quirks
 ----------------------
 
+.. versionchanged:: 11.0
+
+    Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata`
+    is now supported.
+
 **Hard throttling**
   Like most ESPs, Amazon SES `throttles sending`_ for new customers. But unlike
   most ESPs, SES does not queue and slowly release throttled messages. Instead, it
@@ -80,11 +85,6 @@ Limitations and quirks
   :attr:`~anymail.message.AnymailMessage.tags` feature. See :ref:`amazon-ses-tags`
   below for more information and additional options.
 
-**No merge_metadata**
-  Amazon SES's batch sending API does not support the custom headers Anymail uses
-  for metadata, so Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata`
-  feature is not available. (See :ref:`amazon-ses-tags` below for more information.)
-
 **Open and click tracking overrides**
   Anymail's :attr:`~anymail.message.AnymailMessage.track_opens` and
   :attr:`~anymail.message.AnymailMessage.track_clicks` are not supported.
@@ -126,7 +126,7 @@ Limitations and quirks
   signal, and using it will likely prevent delivery of your email.)
 
 **Template limitations**
-  Messages sent with templates have a number of additional limitations, such as not
+  Messages sent with templates have some additional limitations, such as not
   supporting attachments. See :ref:`amazon-ses-templates` below.
 
 
@@ -195,12 +195,7 @@ characters.
 
 For more complex use cases, set the SES ``EmailTags`` parameter (or ``DefaultEmailTags``
 for template sends) directly in Anymail's :ref:`esp_extra <amazon-ses-esp-extra>`. See
-the example below. (Because custom headers do not work with SES's SendBulkEmail call,
+the example below.
-esp_extra ``DefaultEmailTags`` is the only way to attach data to SES messages also using
-Anymail's :attr:`~anymail.message.AnymailMessage.template_id` and
-:attr:`~anymail.message.AnymailMessage.merge_data` features, and
-:attr:`~anymail.message.AnymailMessage.merge_metadata` cannot be supported.)
-
 
 .. _Introducing Sending Metrics:
     https://aws.amazon.com/blogs/ses/introducing-sending-metrics/
@@ -264,9 +259,10 @@ See Amazon's `Sending personalized email`_ guide for more information.
 When you set a message's :attr:`~anymail.message.AnymailMessage.template_id`
 to the name of one of your SES templates, Anymail will use the SES v2 `SendBulkEmail`_
 call to send template messages personalized with data
-from Anymail's normalized :attr:`~anymail.message.AnymailMessage.merge_data`
+from Anymail's normalized :attr:`~anymail.message.AnymailMessage.merge_data`,
-and :attr:`~anymail.message.AnymailMessage.merge_global_data`
+:attr:`~anymail.message.AnymailMessage.merge_global_data`,
-message attributes.
+:attr:`~anymail.message.AnymailMessage.merge_metadata`, and
+:attr:`~anymail.message.AnymailMessage.merge_headers` message attributes.
 
   .. code-block:: python
 
@@ -284,17 +280,21 @@ message attributes.
           'ship_date': "May 15",
       }
 
-Amazon's templated email APIs don't support several features available for regular email.
+Amazon's templated email APIs don't support a few features available for regular email.
 When :attr:`~anymail.message.AnymailMessage.template_id` is used:
 
-* Attachments and alternative parts (including AMPHTML) are not supported
+* Attachments and inline images are not supported
-* Extra headers are not supported
+* Alternative parts (including AMPHTML) are not supported
 * Overriding the template's subject or body is not supported
-* Anymail's :attr:`~anymail.message.AnymailMessage.metadata` is not supported
+
-* Anymail's :attr:`~anymail.message.AnymailMessage.tags` are only supported
+.. versionchanged:: 11.0
-  with the :setting:`AMAZON_SES_MESSAGE_TAG_NAME <ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME>`
+
-  setting; only a single tag is allowed, and the tag is not directly available
+    Extra headers, :attr:`~anymail.message.AnymailMessage.metadata`,
-  to webhooks. (See :ref:`amazon-ses-tags` above.)
+    :attr:`~anymail.message.AnymailMessage.merge_metadata`, and
+    :attr:`~anymail.message.AnymailMessage.tags` are now fully supported
+    when using :attr:`~anymail.message.AnymailMessage.template_id`.
+    (This requires :pypi:`boto3` v1.34.98 or later, which enables the
+    ReplacementHeaders parameter for SendBulkEmail.)
 
 .. _Sending personalized email:
    https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-personalized-email-api.html

tests/test_amazon_ses_backend.py

@@ -568,60 +568,6 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
         ):
             self.message.send()
 
-    @override_settings(
-        # only way to use tags with template_id:
-        ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign"
-    )
-    def test_template_dont_add_merge_headers(self):
-        """With template_id, Anymail switches to SESv2 SendBulkEmail"""
-        # SendBulkEmail uses a completely different API call and payload
-        # structure, so this re-tests a bunch of Anymail features that were handled
-        # differently above. (See test_amazon_ses_integration for a more realistic
-        # template example.)
-        raw_response = {
-            "BulkEmailEntryResults": [
-                {
-                    "Status": "SUCCESS",
-                    "MessageId": "1111111111111111-bbbbbbbb-3333-7777",
-                },
-                {
-                    "Status": "ACCOUNT_DAILY_QUOTA_EXCEEDED",
-                    "Error": "Daily message quota exceeded",
-                },
-            ],
-            "ResponseMetadata": self.DEFAULT_SEND_RESPONSE["ResponseMetadata"],
-        }
-        self.set_mock_response(raw_response, operation_name="send_bulk_email")
-        message = AnymailMessage(
-            template_id="welcome_template",
-            from_email='"Example, Inc." <from@example.com>',
-            to=["alice@example.com", "罗伯特 <bob@example.com>"],
-            cc=["cc@example.com"],
-            reply_to=["reply1@example.com", "Reply 2 <reply2@example.com>"],
-            merge_data={
-                "alice@example.com": {"name": "Alice", "group": "Developers"},
-                "bob@example.com": {"name": "Bob"},  # and leave group undefined
-                "nobody@example.com": {"name": "Not a recipient for this message"},
-            },
-            merge_global_data={"group": "Users", "site": "ExampleCo"},
-            # (only works with AMAZON_SES_MESSAGE_TAG_NAME when using template):
-            tags=["WelcomeVariantA"],
-            envelope_sender="bounce@example.com",
-            esp_extra={
-                "FromEmailAddressIdentityArn": (
-                    "arn:aws:ses:us-east-1:123456789012:identity/example.com"
-                )
-            },
-        )
-        message.send()
-
-        params = self.get_send_params(operation_name="send_bulk_email")
-        self.assertNotIn("ReplacementHeaders", params["BulkEmailEntries"][0])
-
-    @override_settings(
-        # only way to use tags with template_id:
-        ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign"
-    )
     def test_template(self):
         """With template_id, Anymail switches to SESv2 SendBulkEmail"""
         # SendBulkEmail uses a completely different API call and payload
@@ -648,24 +594,29 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
             to=["alice@example.com", "罗伯特 <bob@example.com>"],
             cc=["cc@example.com"],
             reply_to=["reply1@example.com", "Reply 2 <reply2@example.com>"],
+            headers={
+                "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
+                "List-Unsubscribe": "<mailto:unsubscribe@example.com>",
+            },
+            merge_headers={
+                "alice@example.com": {
+                    "List-Unsubscribe": "<https://example.com/a/>",
+                },
+                "bob@example.com": {
+                    "List-Unsubscribe": "<https://example.com/b/>",
+                },
+            },
             merge_data={
                 "alice@example.com": {"name": "Alice", "group": "Developers"},
                 "bob@example.com": {"name": "Bob"},  # and leave group undefined
                 "nobody@example.com": {"name": "Not a recipient for this message"},
             },
-            merge_headers={
-                "alice@example.com": {
-                    "List-Unsubscribe": "<https://example.com/a/>",
-                    "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
-                },
-                "nobody@example.com": {
-                    "List-Unsubscribe": "<mailto:unsubscribe@example.com>",
-                    "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
-                },
-            },
             merge_global_data={"group": "Users", "site": "ExampleCo"},
-            # (only works with AMAZON_SES_MESSAGE_TAG_NAME when using template):
+            tags=["Welcome Variant A", "Cohort 12/2017"],
-            tags=["WelcomeVariantA"],
+            metadata={"meta1": "test"},
+            merge_metadata={
+                "alice@example.com": {"meta2": "meta-alice"},
+            },
             envelope_sender="bounce@example.com",
             esp_extra={
                 "FromEmailAddressIdentityArn": (
@@ -715,19 +666,40 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
             {"name": "Bob"},
         )
 
-        self.assertEqual(
+        self.assertCountEqual(
             bulk_entries[0]["ReplacementHeaders"],
             [
-                {"Name": "List-Unsubscribe", "Value": "<https://example.com/a/>"},
+                # From extra_headers and merge_headers:
                 {
                     "Name": "List-Unsubscribe-Post",
                     "Value": "List-Unsubscribe=One-Click",
                 },
+                {"Name": "List-Unsubscribe", "Value": "<https://example.com/a/>"},
+                # From metadata and merge_metadata:
+                {
+                    "Name": "X-Metadata",
+                    "Value": '{"meta1": "test", "meta2": "meta-alice"}',
+                },
+                # From tags:
+                {"Name": "X-Tag", "Value": "Welcome Variant A"},
+                {"Name": "X-Tag", "Value": "Cohort 12/2017"},
             ],
         )
-        self.assertEqual(
+        self.assertCountEqual(
             bulk_entries[1]["ReplacementHeaders"],
-            [],
+            [
+                # From extra_headers and merge_headers:
+                {
+                    "Name": "List-Unsubscribe-Post",
+                    "Value": "List-Unsubscribe=One-Click",
+                },
+                {"Name": "List-Unsubscribe", "Value": "<https://example.com/b/>"},
+                # From metadata (no merge_metadata for bob@):
+                {"Name": "X-Metadata", "Value": '{"meta1": "test"}'},
+                # From tags:
+                {"Name": "X-Tag", "Value": "Welcome Variant A"},
+                {"Name": "X-Tag", "Value": "Cohort 12/2017"},
+            ],
         )
         self.assertEqual(
             json.loads(params["DefaultContent"]["Template"]["TemplateData"]),
@@ -737,10 +709,6 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
             params["ReplyToAddresses"],
             ["reply1@example.com", "Reply 2 <reply2@example.com>"],
         )
-        self.assertEqual(
-            params["DefaultEmailTags"],
-            [{"Name": "Campaign", "Value": "WelcomeVariantA"}],
-        )
         self.assertEqual(params["FeedbackForwardingEmailAddress"], "bounce@example.com")
         # esp_extra:
         self.assertEqual(
@@ -769,6 +737,69 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
         )
         self.assertEqual(message.anymail_status.esp_response, raw_response)
 
+    def test_template_omits_unused_replacement_headers(self):
+        """If headers are not needed, the ReplacementHeaders param should be omitted"""
+        # bob@example.com requires ReplacementHeaders; alice@example.com doesn't
+        raw_response = {
+            "BulkEmailEntryResults": [
+                {
+                    "Status": "SUCCESS",
+                    "MessageId": "1111111111111111-bbbbbbbb-3333-7777",
+                },
+                {
+                    "Status": "SUCCESS",
+                    "MessageId": "1111111111111111-bbbbbbbb-4444-8888",
+                },
+            ],
+            "ResponseMetadata": self.DEFAULT_SEND_RESPONSE["ResponseMetadata"],
+        }
+        self.set_mock_response(raw_response, operation_name="send_bulk_email")
+        message = AnymailMessage(
+            template_id="welcome_template",
+            from_email='"Example, Inc." <from@example.com>',
+            to=["alice@example.com", "罗伯特 <bob@example.com>"],
+            reply_to=["reply1@example.com", "Reply 2 <reply2@example.com>"],
+            merge_headers={
+                "alice@example.com": {},
+                "bob@example.com": {"X-Test": "test"},
+            },
+            merge_global_data={"group": "Users", "site": "ExampleCo"},
+        )
+        message.send()
+
+        params = self.get_send_params(operation_name="send_bulk_email")
+        self.assertNotIn("ReplacementHeaders", params["BulkEmailEntries"][0])
+        self.assertIn("ReplacementHeaders", params["BulkEmailEntries"][1])
+
+    @override_settings(
+        # This will pass DefaultEmailTags: Name "Campaign"
+        ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME="Campaign"
+    )
+    def test_template_default_email_tag(self):
+        raw_response = {
+            "BulkEmailEntryResults": [
+                {
+                    "Status": "SUCCESS",
+                    "MessageId": "1111111111111111-bbbbbbbb-3333-7777",
+                },
+            ],
+            "ResponseMetadata": self.DEFAULT_SEND_RESPONSE["ResponseMetadata"],
+        }
+        self.set_mock_response(raw_response, operation_name="send_bulk_email")
+        message = AnymailMessage(
+            template_id="welcome_template",
+            from_email='"Example, Inc." <from@example.com>',
+            to=["alice@example.com"],
+            tags=["WelcomeVariantA"],
+        )
+        message.send()
+
+        params = self.get_send_params(operation_name="send_bulk_email")
+        self.assertEqual(
+            params["DefaultEmailTags"],
+            [{"Name": "Campaign", "Value": "WelcomeVariantA"}],
+        )
+
     def test_template_failure(self):
         """Failures to all recipients raise a similar error to non-template sends"""
         raw_response = {
@@ -794,7 +825,7 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
             message.send()
 
     def test_template_unsupported(self):
-        """A lot of options are not compatible with SendBulkTemplatedEmail"""
+        """Some options are not compatible with SendBulkTemplatedEmail"""
         message = AnymailMessage(template_id="welcome_template", to=["to@example.com"])
 
         message.subject = "nope, can't change template subject"
@@ -823,25 +854,6 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
             message.send()
         message.attachments = []
 
-        message.extra_headers = {"X-Custom": "header"}
-        with self.assertRaisesMessage(
-            AnymailUnsupportedFeature, "extra_headers with template"
-        ):
-            message.send()
-        message.extra_headers = {}
-
-        message.metadata = {"meta": "data"}
-        with self.assertRaisesMessage(
-            AnymailUnsupportedFeature, "metadata with template"
-        ):
-            message.send()
-        message.metadata = None
-
-        message.tags = ["tag 1", "tag 2"]
-        with self.assertRaisesMessage(AnymailUnsupportedFeature, "tags with template"):
-            message.send()
-        message.tags = None
-
     def test_send_anymail_message_without_template(self):
         # Make sure SendEmail is used for non-template_id messages
         message = AnymailMessage(

tests/test_amazon_ses_integration.py

@@ -164,6 +164,20 @@ class AmazonSESBackendIntegrationTests(AnymailTestMixin, SimpleTestCase):
                 "success+to2@simulator.amazonses.com": {"order": 6789},
             },
             merge_global_data={"name": "Customer", "ship_date": "today"},  # default
+            headers={
+                "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
+            },
+            merge_headers={
+                "success+to1@simulator.amazonses.com": {
+                    "List-Unsubscribe": "<https://example.com/unsubscribe/to1>"
+                },
+                "success+to2@simulator.amazonses.com": {
+                    "List-Unsubscribe": "<https://example.com/unsubscribe/to2>"
+                },
+            },
+            tags=["Live integration test", "Template send"],
+            metadata={"test": "data"},
+            merge_metadata={"success+to2@simulator.amazonses.com": {"user-id": "2"}},
         )
         message.send()
         recipient_status = message.anymail_status.recipients