Brevo: add batch send support

Closes #353

CHANGELOG.rst

@@ -25,6 +25,18 @@ Release history
 ^^^^^^^^^^^^^^^
     ..  This extra heading level keeps the ToC from becoming unmanageably long
 
+vNext
+-----
+
+*unreleased changes*
+
+Features
+~~~~~~~~
+
+* **Brevo:** Add support for batch sending
+  (`docs <https://anymail.dev/en/latest/esps/brevo/#batch-sending-merge-and-esp-templates>`__).
+
+
 v10.2
 -----
 

anymail/backends/sendinblue.py

@@ -39,11 +39,16 @@ class EmailBackend(AnymailRequestsBackend):
         # SendinBlue doesn't give any detail on a success
         # https://developers.sendinblue.com/docs/responses
         message_id = None
+        message_ids = []
 
         if response.content != b"":
             parsed_response = self.deserialize_json_response(response, payload, message)
             try:
                 message_id = parsed_response["messageId"]
+            except (KeyError, TypeError):
+                try:
+                    # batch send
+                    message_ids = parsed_response["messageIds"]
                 except (KeyError, TypeError) as err:
                     raise AnymailRequestsAPIError(
                         "Invalid SendinBlue API response format",
@@ -54,12 +59,21 @@ class EmailBackend(AnymailRequestsBackend):
                     ) from err
 
         status = AnymailRecipientStatus(message_id=message_id, status="queued")
-        return {recipient.addr_spec: status for recipient in payload.all_recipients}
+        recipient_status = {
+            recipient.addr_spec: status for recipient in payload.all_recipients
+        }
+        if message_ids:
+            for to, message_id in zip(payload.to_recipients, message_ids):
+                recipient_status[to.addr_spec] = AnymailRecipientStatus(
+                    message_id=message_id, status="queued"
+                )
+        return recipient_status
 
 
 class SendinBluePayload(RequestsPayload):
     def __init__(self, message, defaults, backend, *args, **kwargs):
         self.all_recipients = []  # used for backend.parse_recipient_status
+        self.to_recipients = []  # used for backend.parse_recipient_status
 
         http_headers = kwargs.pop("headers", {})
         http_headers["api-key"] = backend.api_key
@@ -74,9 +88,32 @@ class SendinBluePayload(RequestsPayload):
 
     def init_payload(self):
         self.data = {"headers": CaseInsensitiveDict()}  # becomes json
+        self.merge_data = {}
+        self.metadata = {}
+        self.merge_metadata = {}
 
     def serialize_data(self):
         """Performs any necessary serialization on self.data, and returns the result."""
+        if self.is_batch():
+            # Burst data["to"] into data["messageVersions"]
+            to_list = self.data.pop("to", [])
+            self.data["messageVersions"] = [
+                {"to": [to], "params": self.merge_data.get(to["email"])}
+                for to in to_list
+            ]
+            if self.merge_metadata:
+                # Merge global metadata with any per-recipient metadata.
+                # (Top-level X-Mailin-custom header is already set to global metadata,
+                # and will apply for recipients without a "headers" override.)
+                for version in self.data["messageVersions"]:
+                    to_email = version["to"][0]["email"]
+                    if to_email in self.merge_metadata:
+                        recipient_metadata = self.metadata.copy()
+                        recipient_metadata.update(self.merge_metadata[to_email])
+                        version["headers"] = {
+                            "X-Mailin-custom": self.serialize_json(recipient_metadata)
+                        }
+
         if not self.data["headers"]:
             del self.data["headers"]  # don't send empty headers
         return self.serialize_json(self.data)
@@ -102,6 +139,8 @@ class SendinBluePayload(RequestsPayload):
         if emails:
             self.data[recipient_type] = [self.email_object(email) for email in emails]
             self.all_recipients += emails  # used for backend.parse_recipient_status
+            if recipient_type == "to":
+                self.to_recipients = emails  # used for backend.parse_recipient_status
 
     def set_subject(self, subject):
         if subject != "":  # see note in set_text_body about template rendering
@@ -158,8 +197,8 @@ class SendinBluePayload(RequestsPayload):
         self.data.update(extra)
 
     def set_merge_data(self, merge_data):
-        """SendinBlue doesn't support special attributes for each recipient"""
+        # Late bound in serialize_data:
-        self.unsupported_feature("merge_data")
+        self.merge_data = merge_data
 
     def set_merge_global_data(self, merge_global_data):
         self.data["params"] = merge_global_data
@@ -167,6 +206,11 @@ class SendinBluePayload(RequestsPayload):
     def set_metadata(self, metadata):
         # SendinBlue expects a single string payload
         self.data["headers"]["X-Mailin-custom"] = self.serialize_json(metadata)
+        self.metadata = metadata  # needed in serialize_data for batch send
+
+    def set_merge_metadata(self, merge_metadata):
+        # Late-bound in serialize_data:
+        self.merge_metadata = merge_metadata
 
     def set_send_at(self, send_at):
         try:

docs/esps/brevo.rst

@@ -149,10 +149,33 @@ Brevo can handle.
   If you are ignoring unsupported features and have multiple reply addresses,
   Anymail will use only the first one.
 
-**Metadata**
+**Metadata exposed in message headers**
   Anymail passes :attr:`~anymail.message.AnymailMessage.metadata` to Brevo
   as a JSON-encoded string using their :mailheader:`X-Mailin-custom` email header.
-  The metadata is available in tracking webhooks.
+  This header is included in the sent message, so **metadata will be visible to
+  message recipients** if they view the raw message source.
+
+**Special headers**
+  Brevo uses special email headers to control certain features.
+  You can set these using Django's
+  :class:`EmailMessage.headers <django.core.mail.EmailMessage>`:
+
+    .. code-block:: python
+
+        message = EmailMessage(
+            ...,
+            headers = {
+                "sender.ip": "10.10.1.150",  # use a dedicated IP
+                "idempotencyKey": "...uuid...",  # batch send deduplication
+            }
+        )
+
+        # Note the constructor param is called `headers`, but the
+        # corresponding attribute is named `extra_headers`:
+        message.extra_headers = {
+            "sender.ip": "10.10.1.222",
+            "idempotencyKey": "...uuid...",
+        }
 
 **Delayed sending**
   .. versionadded:: 9.0
@@ -174,30 +197,33 @@ Brevo can handle.
 Batch sending/merge and ESP templates
 -------------------------------------
 
-Brevo supports :ref:`ESP stored templates <esp-stored-templates>` populated with
+.. versionchanged:: 10.3
-global merge data for all recipients, but does not offer :ref:`batch sending <batch-send>`
+
-with per-recipient merge data. Anymail's :attr:`~anymail.message.AnymailMessage.merge_data`
+    Added support for batch sending with :attr:`~anymail.message.AnymailMessage.merge_data`
-and :attr:`~anymail.message.AnymailMessage.merge_metadata` message attributes are not
+    and :attr:`~anymail.message.AnymailMessage.merge_metadata`.
-supported with the Brevo backend, but you can use Anymail's
+
-:attr:`~anymail.message.AnymailMessage.merge_global_data` with Brevo templates.
+Brevo supports :ref:`ESP stored templates <esp-stored-templates>` and
+:ref:`batch sending <batch-send>` with per-recipient merge data.
 
 To use a Brevo template, set the message's
 :attr:`~anymail.message.AnymailMessage.template_id` to the numeric
-Brevo template ID, and supply substitution attributes using
+Brevo template ID, and supply substitution params using Anymail's normalized
-the message's :attr:`~anymail.message.AnymailMessage.merge_global_data`:
+:attr:`~anymail.message.AnymailMessage.merge_data` and
+:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes:
 
   .. code-block:: python
 
       message = EmailMessage(
-          to=["alice@example.com"]  # single recipient...
+          # (subject and body come from the template, so don't include those)
-          # ...multiple to emails would all get the same message
+          to=["alice@example.com", "Bob <bob@example.com>"]
-          # (and would all see each other's emails in the "to" header)
       )
       message.template_id = 3   # use this Brevo template
       message.from_email = None  # to use the template's default sender
+      message.merge_data = {
+          'alice@example.com': {'name': "Alice", 'order_no': "12345"},
+          'bob@example.com': {'name': "Bob", 'order_no': "54321"},
+      }
       message.merge_global_data = {
-          'name': "Alice",
-          'order_no': "12345",
           'ship_date': "May 15",
       }
 
@@ -214,6 +240,31 @@ If you want to use the template's sender, be sure to set ``from_email`` to ``Non
 You can also override the template's subject and reply-to address (but not body)
 using standard :class:`~django.core.mail.EmailMessage` attributes.
 
+Brevo also supports batch-sending without using an ESP-stored template. In this
+case, each recipient will receive the same content (Brevo doesn't support inline
+templates) but will see only their own *To* email address. Setting either of
+:attr:`~anymail.message.AnymailMessage.merge_data` or
+:attr:`~anymail.message.AnymailMessage.merge_metadata`---even to an empty
+dict---will cause Anymail to use Brevo's batch send option (``"messageVersions"``).
+
+You can use Anymail's
+:attr:`~anymail.message.AnymailMessage.merge_metadata` to supply custom tracking
+data for each recipient:
+
+  .. code-block:: python
+
+      message = EmailMessage(
+          to=["alice@example.com", "Bob <bob@example.com>"],
+          from_email="...", subject="...", body="..."
+      )
+      message.merge_metadata = {
+          'alice@example.com': {'user_id': "12345"},
+          'bob@example.com': {'user_id': "54321"},
+      }
+
+To use Brevo's "`idempotencyKey`_" with a batch send, set it in the
+message's headers: ``message.extra_headers = {"idempotencyKey": "...uuid..."}``.
+
 .. caution::
 
     **Sendinblue "old template language" not supported**
@@ -241,6 +292,9 @@ using standard :class:`~django.core.mail.EmailMessage` attributes.
 .. _Brevo Template Language:
     https://help.brevo.com/hc/en-us/articles/360000946299
 
+.. _idempotencyKey:
+    https://developers.brevo.com/docs/heterogenous-versions-batch-emails
+
 .. _convert each old template:
     https://help.brevo.com/hc/en-us/articles/360000991960
 

tests/test_sendinblue_backend.py

@@ -18,7 +18,7 @@ from anymail.exceptions import (
     AnymailSerializationError,
     AnymailUnsupportedFeature,
 )
-from anymail.message import attach_inline_image_file
+from anymail.message import AnymailMessage, attach_inline_image_file
 
 from .mock_requests_backend import (
     RequestsBackendMockAPITestCase,
@@ -478,13 +478,60 @@ class SendinBlueBackendAnymailFeatureTests(SendinBlueBackendMockAPITestCase):
         self.assertEqual(data["subject"], "My Subject")
         self.assertEqual(data["to"], [{"email": "to@example.com", "name": "Recipient"}])
 
-    def test_merge_data(self):
+    _mock_batch_response = {
-        self.message.merge_data = {
+        "messageIds": [
-            "alice@example.com": {":name": "Alice", ":group": "Developers"},
+            "<202403182259.64789700810.1@smtp-relay.mailin.fr>",
-            "bob@example.com": {":name": "Bob"},  # and leave :group undefined
+            "<202403182259.64789700810.2@smtp-relay.mailin.fr>",
+        ]
     }
-        with self.assertRaises(AnymailUnsupportedFeature):
+
-            self.message.send()
+    def test_merge_data(self):
+        self.set_mock_response(json_data=self._mock_batch_response)
+        message = AnymailMessage(
+            from_email="from@example.com",
+            template_id=1234567,
+            to=["alice@example.com", "Bob <bob@example.com>"],
+            merge_data={
+                "alice@example.com": {"name": "Alice", "group": "Developers"},
+                "bob@example.com": {"name": "Bob"},  # and leave group undefined
+                "nobody@example.com": {"name": "Not a recipient for this message"},
+            },
+            merge_global_data={"group": "Users", "site": "ExampleCo"},
+        )
+        message.send()
+
+        # batch send uses same API endpoint as regular send:
+        self.assert_esp_called("/v3/smtp/email")
+        data = self.get_api_call_json()
+        versions = data["messageVersions"]
+        self.assertEqual(len(versions), 2)
+        self.assertEqual(
+            versions[0],
+            {
+                "to": [{"email": "alice@example.com"}],
+                "params": {"name": "Alice", "group": "Developers"},
+            },
+        )
+        self.assertEqual(
+            versions[1],
+            {
+                "to": [{"email": "bob@example.com", "name": "Bob"}],
+                "params": {"name": "Bob"},
+            },
+        )
+        self.assertEqual(data["params"], {"group": "Users", "site": "ExampleCo"})
+
+        recipients = message.anymail_status.recipients
+        self.assertEqual(recipients["alice@example.com"].status, "queued")
+        self.assertEqual(
+            recipients["alice@example.com"].message_id,
+            "<202403182259.64789700810.1@smtp-relay.mailin.fr>",
+        )
+        self.assertEqual(recipients["bob@example.com"].status, "queued")
+        self.assertEqual(
+            recipients["bob@example.com"].message_id,
+            "<202403182259.64789700810.2@smtp-relay.mailin.fr>",
+        )
 
     def test_merge_global_data(self):
         self.message.merge_global_data = {"a": "b"}
@@ -492,6 +539,38 @@ class SendinBlueBackendAnymailFeatureTests(SendinBlueBackendMockAPITestCase):
         data = self.get_api_call_json()
         self.assertEqual(data["params"], {"a": "b"})
 
+    def test_merge_metadata(self):
+        self.set_mock_response(json_data=self._mock_batch_response)
+        self.message.to = ["alice@example.com", "Bob <bob@example.com>"]
+        self.message.merge_metadata = {
+            "alice@example.com": {"order_id": 123, "tier": "premium"},
+            "bob@example.com": {"order_id": 678},
+        }
+        self.message.metadata = {"notification_batch": "zx912"}
+        self.message.send()
+
+        data = self.get_api_call_json()
+        versions = data["messageVersions"]
+        self.assertEqual(len(versions), 2)
+        self.assertEqual(versions[0]["to"], [{"email": "alice@example.com"}])
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertEqual(
+            json.loads(versions[0]["headers"]["X-Mailin-custom"]),
+            {"order_id": 123, "tier": "premium", "notification_batch": "zx912"},
+        )
+        self.assertEqual(
+            versions[1]["to"], [{"name": "Bob", "email": "bob@example.com"}]
+        )
+        self.assertEqual(
+            json.loads(versions[1]["headers"]["X-Mailin-custom"]),
+            {"order_id": 678, "notification_batch": "zx912"},
+        )
+        # default metadata still sent in base headers:
+        self.assertEqual(
+            json.loads(data["headers"]["X-Mailin-custom"]),
+            {"notification_batch": "zx912"},
+        )
+
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
 
@@ -502,35 +581,24 @@ class SendinBlueBackendAnymailFeatureTests(SendinBlueBackendMockAPITestCase):
         self.message.send()
         data = self.get_api_call_json()
         self.assertNotIn("attachment", data)
-        self.assertNotIn("tag", data)
+        self.assertNotIn("bcc", data)
+        self.assertNotIn("cc", data)
         self.assertNotIn("headers", data)
+        self.assertNotIn("messageVersions", data)
+        self.assertNotIn("params", data)
         self.assertNotIn("replyTo", data)
-        self.assertNotIn("atributes", data)
+        self.assertNotIn("schedule", data)
+        self.assertNotIn("tags", data)
+        self.assertNotIn("templateId", data)
 
     def test_esp_extra(self):
-        # SendinBlue doesn't offer any esp-extra but we will test
-        # with some extra of SendGrid to see if it's work in the future
         self.message.esp_extra = {
-            "ip_pool_name": "transactional",
+            "batchId": "5c6cfa04-eed9-42c2-8b5c-6d470d978e9d",
-            "asm": {  # subscription management
-                "group_id": 1,
-            },
-            "tracking_settings": {
-                "subscription_tracking": {
-                    "enable": True,
-                    "substitution_tag": "[unsubscribe_url]",
-                },
-            },
         }
         self.message.send()
         data = self.get_api_call_json()
         # merged from esp_extra:
-        self.assertEqual(data["ip_pool_name"], "transactional")
+        self.assertEqual(data["batchId"], "5c6cfa04-eed9-42c2-8b5c-6d470d978e9d")
-        self.assertEqual(data["asm"], {"group_id": 1})
-        self.assertEqual(
-            data["tracking_settings"]["subscription_tracking"],
-            {"enable": True, "substitution_tag": "[unsubscribe_url]"},
-        )
 
     # noinspection PyUnresolvedReferences
     def test_send_attaches_anymail_status(self):

tests/test_sendinblue_integration.py

@@ -98,40 +98,42 @@ class SendinBlueBackendIntegrationTests(AnymailTestMixin, SimpleTestCase):
             template_id=5,
             # Override template sender:
             from_email=formataddr(("Sender", self.from_email)),
-            # No batch send (so max one recipient suggested):
+            to=["Recipient 1 <test+to1@anymail.dev>", "test+to2@anymail.dev"],
-            to=["Recipient <test+to1@anymail.dev>"],
             reply_to=["Do not reply <reply@example.dev>"],
             tags=["using-template"],
-            headers={"X-Anymail-Test": "group: A, variation: C"},
-            merge_global_data={
             # The Anymail test template includes `{{ params.SHIP_DATE }}`
             # and `{{ params.ORDER_ID }}` substitutions
-                "SHIP_DATE": "yesterday",
+            merge_data={
-                "ORDER_ID": "12345",
+                "test+to1@anymail.dev": {"ORDER_ID": "12345"},
+                "test+to2@anymail.dev": {"ORDER_ID": "23456"},
+            },
+            merge_global_data={"SHIP_DATE": "yesterday"},
+            metadata={"customer-id": "unknown", "meta2": 2},
+            merge_metadata={
+                "test+to1@anymail.dev": {"customer-id": "ZXK9123"},
+                "test+to2@anymail.dev": {"customer-id": "ZZT4192"},
             },
-            metadata={"customer-id": "ZXK9123", "meta2": 2},
         )
 
-        # Normal attachments don't work with Brevo templates:
+        message.attach("attachment1.txt", "Here is some\ntext", "text/plain")
-        #   message.attach("attachment1.txt", "Here is some\ntext", "text/plain")
-        # If you can host the attachment content on some publicly-accessible URL,
-        # this *non-portable* alternative allows sending attachments with templates:
-        message.esp_extra = {
-            "attachment": [
-                {
-                    "name": "attachment1.txt",
-                    # URL where Brevo can download the attachment content while
-                    # sending (must be content-type: text/plain):
-                    "url": "https://raw.githubusercontent.com/anymail/django-anymail/"
-                    "main/docs/_readme/template.txt",
-                }
-            ]
-        }
 
         message.send()
         # SendinBlue always queues:
         self.assertEqual(message.anymail_status.status, {"queued"})
-        self.assertRegex(message.anymail_status.message_id, r"\<.+@.+\>")
+        recipient_status = message.anymail_status.recipients
+        self.assertEqual(recipient_status["test+to1@anymail.dev"].status, "queued")
+        self.assertEqual(recipient_status["test+to2@anymail.dev"].status, "queued")
+        self.assertRegex(
+            recipient_status["test+to1@anymail.dev"].message_id, r"\<.+@.+\>"
+        )
+        self.assertRegex(
+            recipient_status["test+to2@anymail.dev"].message_id, r"\<.+@.+\>"
+        )
+        # Each recipient gets their own message_id:
+        self.assertNotEqual(
+            recipient_status["test+to1@anymail.dev"].message_id,
+            recipient_status["test+to2@anymail.dev"].message_id,
+        )
 
     @override_settings(ANYMAIL_SENDINBLUE_API_KEY="Hey, that's not an API key!")
     def test_invalid_api_key(self):