Postmark: Support merge_data and batch sending.

Use Postmark /email/batch or /email/batchWithTemplates APIs when
merge_data provided.

Parse Postmark batch-send API responses, and improve accuracy of
parsing individual recipient status from all responses.

Closes #122

CHANGELOG.rst

@@ -30,6 +30,13 @@ v4.2
 
 *In development*
 
+Features
+~~~~~~~~
+
+*  **Postmark:** Support per-recipient template `merge_data` and batch sending. (Batch
+   sending can be used with or without a template. See
+   `docs <https://anymail.readthedocs.io/en/stable/esps/postmark/#postmark-templates>`__.)
+
 Fixes
 ~~~~~
 

anymail/backends/postmark.py

@@ -2,7 +2,7 @@ import re
 
 from ..exceptions import AnymailRequestsAPIError
 from ..message import AnymailRecipientStatus
-from ..utils import get_anymail_setting
+from ..utils import get_anymail_setting, parse_address_list
 
 from .base_requests import AnymailRequestsBackend, RequestsPayload
 
@@ -33,58 +33,90 @@ class EmailBackend(AnymailRequestsBackend):
             super(EmailBackend, self).raise_for_status(response, payload, message)
 
     def parse_recipient_status(self, response, payload, message):
+        # default to "unknown" status for each recipient, unless/until we find otherwise
+        unknown_status = AnymailRecipientStatus(message_id=None, status='unknown')
+        recipient_status = {to.addr_spec: unknown_status for to in payload.to_emails}
+
         parsed_response = self.deserialize_json_response(response, payload, message)
-        try:
-            error_code = parsed_response["ErrorCode"]
-            msg = parsed_response["Message"]
-        except (KeyError, TypeError):
-            raise AnymailRequestsAPIError("Invalid Postmark API response format",
-                                          email_message=message, payload=payload, response=response,
-                                          backend=self)
+        if not isinstance(parsed_response, list):
+            # non-batch calls return a single response object
+            parsed_response = [parsed_response]
 
-        message_id = parsed_response.get("MessageID", None)
-        rejected_emails = []
+        for one_response in parsed_response:
+            try:
+                # these fields should always be present
+                error_code = one_response["ErrorCode"]
+                msg = one_response["Message"]
+            except (KeyError, TypeError):
+                raise AnymailRequestsAPIError("Invalid Postmark API response format",
+                                              email_message=message, payload=payload, response=response,
+                                              backend=self)
 
-        if error_code == 300:  # Invalid email request
-            # Either the From address or at least one recipient was invalid. Email not sent.
-            if "'From' address" in msg:
-                # Normal error
+            if error_code == 0:
+                # At least partial success, and (some) email was sent.
+                try:
+                    to_header = one_response["To"]
+                    message_id = one_response["MessageID"]
+                except KeyError:
+                    raise AnymailRequestsAPIError("Invalid Postmark API success response format",
+                                                  email_message=message, payload=payload,
+                                                  response=response, backend=self)
+                for to in parse_address_list(to_header):
+                    recipient_status[to.addr_spec.lower()] = AnymailRecipientStatus(
+                        message_id=message_id, status='sent')
+                # Sadly, have to parse human-readable message to figure out if everyone got it:
+                #   "Message OK, but will not deliver to these inactive addresses: {addr_spec, ...}.
+                #    Inactive recipients are ones that have generated a hard bounce or a spam complaint."
+                reject_addr_specs = self._addr_specs_from_error_msg(
+                    msg, r'inactive addresses:\s*(.*)\.\s*Inactive recipients')
+                for reject_addr_spec in reject_addr_specs:
+                    recipient_status[reject_addr_spec] = AnymailRecipientStatus(
+                        message_id=None, status='rejected')
+
+            elif error_code == 300:  # Invalid email request
+                # Either the From address or at least one recipient was invalid. Email not sent.
+                # response["To"] is not populated for this error; must examine response["Message"]:
+                #   "Invalid 'To' address: '{addr_spec}'."
+                #   "Error parsing 'To': Illegal email domain '{domain}' in address '{addr_spec}'."
+                #   "Error parsing 'To': Illegal email address '{addr_spec}'. It must contain the '@' symbol."
+                #   "Invalid 'From' address: '{email_address}'."
+                if "'From' address" in msg:
+                    # Normal error
+                    raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response,
+                                                  backend=self)
+                else:
+                    # Use AnymailRecipientsRefused logic
+                    invalid_addr_specs = self._addr_specs_from_error_msg(msg, r"address:?\s*'(.*)'")
+                    for invalid_addr_spec in invalid_addr_specs:
+                        recipient_status[invalid_addr_spec] = AnymailRecipientStatus(
+                            message_id=None, status='invalid')
+
+            elif error_code == 406:  # Inactive recipient
+                # All recipients were rejected as hard-bounce or spam-complaint. Email not sent.
+                # response["To"] is not populated for this error; must examine response["Message"]:
+                #   "You tried to send to a recipient that has been marked as inactive.\n
+                #    Found inactive addresses: {addr_spec, ...}.\n
+                #    Inactive recipients are ones that have generated a hard bounce or a spam complaint. "
+                reject_addr_specs = self._addr_specs_from_error_msg(
+                    msg, r'inactive addresses:\s*(.*)\.\s*Inactive recipients')
+                for reject_addr_spec in reject_addr_specs:
+                    recipient_status[reject_addr_spec] = AnymailRecipientStatus(
+                        message_id=None, status='rejected')
+
+            else:  # Other error
                 raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response,
                                               backend=self)
-            else:
-                # Use AnymailRecipientsRefused logic
-                default_status = 'invalid'
-        elif error_code == 406:  # Inactive recipient
-            # All recipients were rejected as hard-bounce or spam-complaint. Email not sent.
-            default_status = 'rejected'
-        elif error_code == 0:
-            # At least partial success, and email was sent.
-            # Sadly, have to parse human-readable message to figure out if everyone got it.
-            default_status = 'sent'
-            rejected_emails = self.parse_inactive_recipients(msg)
-        else:
-            raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response,
-                                          backend=self)
 
-        return {
-            recipient.addr_spec: AnymailRecipientStatus(
-                message_id=message_id,
-                status=('rejected' if recipient.addr_spec.lower() in rejected_emails
-                        else default_status)
-            )
-            for recipient in payload.all_recipients
-        }
+        return recipient_status
 
-    def parse_inactive_recipients(self, msg):
-        """Return a list of 'inactive' email addresses from a Postmark "OK" response
+    @staticmethod
+    def _addr_specs_from_error_msg(error_msg, pattern):
+        """Extract a list of email addr_specs from Postmark error_msg.
 
-        :param str msg: the "Message" from the Postmark API response
+        pattern must be a re whose first group matches a comma-separated
+        list of addr_specs in the message
         """
-        # Example msg with inactive recipients:
-        #   "Message OK, but will not deliver to these inactive addresses: one@xample.com, two@example.com."
-        #   " Inactive recipients are ones that have generated a hard bounce or a spam complaint."
-        # Example msg with everything OK: "OK"
-        match = re.search(r'inactive addresses:\s*(.*)\.\s*Inactive recipients', msg)
+        match = re.search(pattern, error_msg, re.MULTILINE)
         if match:
             emails = match.group(1)  # "one@xample.com, two@example.com"
             return [email.strip().lower() for email in emails.split(',')]
@@ -101,15 +133,23 @@ class PostmarkPayload(RequestsPayload):
             # 'X-Postmark-Server-Token': see get_request_params (and set_esp_extra)
         }
         self.server_token = backend.server_token  # added to headers later, so esp_extra can override
-        self.all_recipients = []  # used for backend.parse_recipient_status
+        self.to_emails = []
+        self.merge_data = None
         super(PostmarkPayload, self).__init__(message, defaults, backend, headers=headers, *args, **kwargs)
 
     def get_api_endpoint(self):
+        batch_send = self.merge_data is not None and len(self.to_emails) > 1
         if 'TemplateId' in self.data or 'TemplateModel' in self.data:
-            # This is the one Postmark API documented to have a trailing slash. (Typo?)
-            return "email/withTemplate/"
+            if batch_send:
+                return "email/batchWithTemplates"
+            else:
+                # This is the one Postmark API documented to have a trailing slash. (Typo?)
+                return "email/withTemplate/"
         else:
-            return "email"
+            if batch_send:
+                return "email/batch"
+            else:
+                return "email"
 
     def get_request_params(self, api_url):
         params = super(PostmarkPayload, self).get_request_params(api_url)
@@ -117,7 +157,26 @@ class PostmarkPayload(RequestsPayload):
         return params
 
     def serialize_data(self):
-        return self.serialize_json(self.data)
+        data = self.data
+        api_endpoint = self.get_api_endpoint()
+        if api_endpoint == "email/batchWithTemplates":
+            data = {"Messages": [self.data_for_recipient(to) for to in self.to_emails]}
+        elif api_endpoint == "email/batch":
+            data = [self.data_for_recipient(to) for to in self.to_emails]
+        return self.serialize_json(data)
+
+    def data_for_recipient(self, to):
+        data = self.data.copy()
+        data["To"] = to.address
+        if self.merge_data and to.addr_spec in self.merge_data:
+            recipient_data = self.merge_data[to.addr_spec]
+            if "TemplateModel" in data:
+                # merge recipient_data into merge_global_data
+                data["TemplateModel"] = data["TemplateModel"].copy()
+                data["TemplateModel"].update(recipient_data)
+            else:
+                data["TemplateModel"] = recipient_data
+        return data
 
     #
     # Payload construction
@@ -136,7 +195,8 @@ class PostmarkPayload(RequestsPayload):
         if emails:
             field = recipient_type.capitalize()
             self.data[field] = ', '.join([email.address for email in emails])
-            self.all_recipients += emails  # used for backend.parse_recipient_status
+            if recipient_type == "to":
+                self.to_emails = emails
 
     def set_subject(self, subject):
         self.data["Subject"] = subject
@@ -204,7 +264,9 @@ class PostmarkPayload(RequestsPayload):
             if field in self.data and not self.data[field]:
                 del self.data[field]
 
-    # merge_data: Postmark doesn't support per-recipient substitutions
+    def set_merge_data(self, merge_data):
+        # late-bind
+        self.merge_data = merge_data
 
     def set_merge_global_data(self, merge_global_data):
         self.data["TemplateModel"] = merge_global_data

docs/esps/postmark.rst

@@ -127,39 +127,49 @@ see :ref:`unsupported-features`.
 Batch sending/merge and ESP templates
 -------------------------------------
 
-Postmark supports :ref:`ESP stored templates <esp-stored-templates>`
-populated with global merge data for all recipients, but does not
-offer :ref:`batch sending <batch-send>` with per-recipient merge data.
-Anymail's :attr:`~anymail.message.AnymailMessage.merge_data`
-message attribute is not supported with the Postmark backend.
+Postmark offers both :ref:`ESP stored templates <esp-stored-templates>`
+and :ref:`batch sending <batch-send>` with per-recipient merge data.
+
+.. versionchanged:: 4.2
+
+    Added Postmark :attr:`~anymail.message.AnymailMessage.merge_data` and batch sending
+    support. (Earlier Anymail releases only supported
+    :attr:`~anymail.message.AnymailMessage.merge_global_data` with Postmark.)
 
 To use a Postmark template, set the message's
-:attr:`~anymail.message.AnymailMessage.template_id` to the numeric
-Postmark "TemplateID" and supply the "TemplateModel" using
-the :attr:`~anymail.message.AnymailMessage.merge_global_data`
-message attribute:
+:attr:`~anymail.message.AnymailMessage.template_id` to the numeric Postmark
+"TemplateID" (*not* its name or "TemplateAlias"). You can find a template's
+numeric id near the top right in Postmark's template editor.
+
+Supply the Postmark "TemplateModel" variables using Anymail's normalized
+:attr:`~anymail.message.AnymailMessage.merge_data` and
+:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes:
 
   .. code-block:: python
 
       message = EmailMessage(
           # (subject and body come from the template, so don't include those)
-          from_email="from@example.com",
-          to=["alice@example.com"]  # single recipient...
-          # ...multiple to emails would all get the same message
-          # (and would all see each other's emails in the "to" header)
+          to=["alice@example.com", "Bob <bob@example.com>"]
       )
-      message.template_id = 80801  # use this Postmark template
+      message.template_id = 80801  # Postmark template id
+      message.merge_data = {
+          'alice@example.com': {'name': "Alice", 'order_no': "12345"},
+          'bob@example.com': {'name': "Bob", 'order_no': "54321"},
+      }
       message.merge_global_data = {
-          'name': "Alice",
-          'order_no': "12345",
           'ship_date': "May 15",
-          'items': [
-              {'product': "Widget", 'price': "9.99"},
-              {'product': "Gadget", 'price': "17.99"},
-          ],
       }
 
 Postmark does not allow overriding the message's subject or body with a template.
+(You can customize the subject by including variables in the template's subject.)
+
+When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`,
+Anymail automatically switches to Postmark's batch send API, so that
+each "to" recipient sees only their own email address. (Any cc's or bcc's will be
+duplicated for *every* to-recipient.)
+
+If you want to use batch sending with a regular message (without a template), set
+merge data to an empty dict: `message.merge_data = {}`.
 
 See this `Postmark blog post on templates`_ for more information.
 

tests/test_postmark_backend.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+import json
 from base64 import b64encode
 from decimal import Decimal
 from email.mime.base import MIMEBase
@@ -386,11 +386,102 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         self.assertNotIn('TextBody', data)
 
     def test_merge_data(self):
-        self.message.merge_data = {
-            'alice@example.com': {'name': "Alice", 'group': "Developers"},
-        }
-        with self.assertRaisesMessage(AnymailUnsupportedFeature, 'merge_data'):
-            self.message.send()
+        self.set_mock_response(raw=json.dumps([{
+            "ErrorCode": 0,
+            "Message": "OK",
+            "To": "alice@example.com",
+            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
+            "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817",
+        }, {
+            "ErrorCode": 0,
+            "Message": "OK",
+            "To": "bob@example.com",
+            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
+            "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d",
+        }]).encode('utf-8'))
+
+        message = AnymailMessage(
+            from_email='from@example.com',
+            template_id=1234567,  # Postmark only supports merge_data content in a template
+            to=['alice@example.com', 'Bob <bob@example.com>'],
+            merge_data={
+                'alice@example.com': {'name': "Alice", 'group': "Developers"},
+                'bob@example.com': {'name': "Bob"},  # and leave group undefined
+                'nobody@example.com': {'name': "Not a recipient for this message"},
+            },
+            merge_global_data={'group': "Users", 'site': "ExampleCo"}
+        )
+        message.send()
+
+        self.assert_esp_called('/email/batchWithTemplates')
+        data = self.get_api_call_json()
+        messages = data["Messages"]
+        self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0], {
+            "From": "from@example.com",
+            "To": "alice@example.com",
+            "TemplateId": 1234567,
+            "TemplateModel": {"name": "Alice", "group": "Developers", "site": "ExampleCo"},
+        })
+        self.assertEqual(messages[1], {
+            "From": "from@example.com",
+            "To": "Bob <bob@example.com>",
+            "TemplateId": 1234567,
+            "TemplateModel": {"name": "Bob", "group": "Users", "site": "ExampleCo"},
+        })
+
+        recipients = message.anymail_status.recipients
+        self.assertEqual(recipients['alice@example.com'].status, 'sent')
+        self.assertEqual(recipients['alice@example.com'].message_id, 'b7bc2f4a-e38e-4336-af7d-e6c392c2f817')
+        self.assertEqual(recipients['bob@example.com'].status, 'sent')
+        self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d')
+
+    def test_merge_data_no_template(self):
+        # merge_data={} can be used to force batch sending without a template
+        self.set_mock_response(raw=json.dumps([{
+            "ErrorCode": 0,
+            "Message": "OK",
+            "To": "alice@example.com",
+            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
+            "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817",
+        }, {
+            "ErrorCode": 0,
+            "Message": "OK",
+            "To": "bob@example.com",
+            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
+            "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d",
+        }]).encode('utf-8'))
+
+        message = AnymailMessage(
+            from_email='from@example.com',
+            to=['alice@example.com', 'Bob <bob@example.com>'],
+            merge_data={},
+            subject="Test batch send",
+            body="Test body",
+        )
+        message.send()
+
+        self.assert_esp_called('/email/batch')
+        data = self.get_api_call_json()
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0], {
+            "From": "from@example.com",
+            "To": "alice@example.com",
+            "Subject": "Test batch send",
+            "TextBody": "Test body",
+        })
+        self.assertEqual(data[1], {
+            "From": "from@example.com",
+            "To": "Bob <bob@example.com>",
+            "Subject": "Test batch send",
+            "TextBody": "Test body",
+        })
+
+        recipients = message.anymail_status.recipients
+        self.assertEqual(recipients['alice@example.com'].status, 'sent')
+        self.assertEqual(recipients['alice@example.com'].message_id, 'b7bc2f4a-e38e-4336-af7d-e6c392c2f817')
+        self.assertEqual(recipients['bob@example.com'].status, 'sent')
+        self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d')
 
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
@@ -433,10 +524,11 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         response_content = b"""{
             "MessageID":"abcdef01-2345-6789-0123-456789abcdef",
             "ErrorCode":0,
+            "To":"Recipient <to1@example.com>",
             "Message":"OK"
         }"""
         self.set_mock_response(raw=response_content)
-        msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['to1@example.com'],)
+        msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['Recipient <to1@example.com>'],)
         sent = msg.send()
         self.assertEqual(sent, 1)
         self.assertEqual(msg.anymail_status.status, {'sent'})

tests/test_postmark_integration.py

@@ -58,11 +58,12 @@ class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
             reply_to=["reply1@example.com", "Reply 2 <reply2@example.com>"],
             headers={"X-Anymail-Test": "value"},
 
-            # no send_at, track_clicks support
+            # no send_at support
             metadata={"meta1": "simple string", "meta2": 2},
             tags=["tag 1"],  # max one tag
             track_opens=True,
             track_clicks=True,
+            merge_data={},  # force batch send (distinct message for each `to`)
         )
         message.attach("attachment1.txt", "Here is some\ntext for you", "text/plain")
         message.attach("attachment2.csv", "ID,Name\n1,Amy Lina", "text/csv")
@@ -74,6 +75,11 @@ class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
 
         message.send()
         self.assertEqual(message.anymail_status.status, {'sent'})
+        self.assertEqual(message.anymail_status.recipients['test+to1@anymail.info'].status, 'sent')
+        self.assertEqual(message.anymail_status.recipients['test+to2@anymail.info'].status, 'sent')
+        # distinct messages should have different message_ids:
+        self.assertNotEqual(message.anymail_status.recipients['test+to1@anymail.info'].message_id,
+                            message.anymail_status.recipients['test+to2@anymail.info'].message_id)
 
     def test_invalid_from(self):
         self.message.from_email = 'webmaster@localhost'  # Django's default From
@@ -90,9 +96,13 @@ class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
     def test_template(self):
         message = AnymailMessage(
             from_email="from@test-pm.anymail.info",
-            to=["test+to1@anymail.info"],
+            to=["test+to1@anymail.info", "Second Recipient <test+to2@anymail.info>"],
             template_id=POSTMARK_TEST_TEMPLATE_ID,
-            merge_global_data={"name": "Test Recipient", "order_no": "12345"},
+            merge_data={
+                "test+to1@anymail.info": {"name": "Recipient 1", "order_no": "12345"},
+                "test+to2@anymail.info": {"order_no": "6789"},
+            },
+            merge_global_data={"name": "Valued Customer"},
         )
         message.send()
         self.assertEqual(message.anymail_status.status, {'sent'})