From 9c493dba72988c841d13eeffde8be2d1f3ded2df Mon Sep 17 00:00:00 2001 From: medmunds Date: Thu, 6 Sep 2018 17:24:04 -0700 Subject: [PATCH] Postmark: Support merge_data and batch sending. Use Postmark /email/batch or /email/batchWithTemplates APIs when merge_data provided. Parse Postmark batch-send API responses, and improve accuracy of parsing individual recipient status from all responses. Closes #122 --- CHANGELOG.rst | 7 ++ anymail/backends/postmark.py | 164 ++++++++++++++++++++--------- docs/esps/postmark.rst | 50 +++++---- tests/test_postmark_backend.py | 106 +++++++++++++++++-- tests/test_postmark_integration.py | 16 ++- 5 files changed, 262 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 03ba543..cf56d20 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,6 +30,13 @@ v4.2 *In development* +Features +~~~~~~~~ + +* **Postmark:** Support per-recipient template `merge_data` and batch sending. (Batch + sending can be used with or without a template. See + `docs `__.) + Fixes ~~~~~ diff --git a/anymail/backends/postmark.py b/anymail/backends/postmark.py index e716bbb..2ae8e87 100644 --- a/anymail/backends/postmark.py +++ b/anymail/backends/postmark.py @@ -2,7 +2,7 @@ import re from ..exceptions import AnymailRequestsAPIError from ..message import AnymailRecipientStatus -from ..utils import get_anymail_setting +from ..utils import get_anymail_setting, parse_address_list from .base_requests import AnymailRequestsBackend, RequestsPayload @@ -33,58 +33,90 @@ class EmailBackend(AnymailRequestsBackend): super(EmailBackend, self).raise_for_status(response, payload, message) def parse_recipient_status(self, response, payload, message): + # default to "unknown" status for each recipient, unless/until we find otherwise + unknown_status = AnymailRecipientStatus(message_id=None, status='unknown') + recipient_status = {to.addr_spec: unknown_status for to in payload.to_emails} + parsed_response = self.deserialize_json_response(response, payload, message) - try: - error_code = parsed_response["ErrorCode"] - msg = parsed_response["Message"] - except (KeyError, TypeError): - raise AnymailRequestsAPIError("Invalid Postmark API response format", - email_message=message, payload=payload, response=response, - backend=self) + if not isinstance(parsed_response, list): + # non-batch calls return a single response object + parsed_response = [parsed_response] - message_id = parsed_response.get("MessageID", None) - rejected_emails = [] + for one_response in parsed_response: + try: + # these fields should always be present + error_code = one_response["ErrorCode"] + msg = one_response["Message"] + except (KeyError, TypeError): + raise AnymailRequestsAPIError("Invalid Postmark API response format", + email_message=message, payload=payload, response=response, + backend=self) - if error_code == 300: # Invalid email request - # Either the From address or at least one recipient was invalid. Email not sent. - if "'From' address" in msg: - # Normal error + if error_code == 0: + # At least partial success, and (some) email was sent. + try: + to_header = one_response["To"] + message_id = one_response["MessageID"] + except KeyError: + raise AnymailRequestsAPIError("Invalid Postmark API success response format", + email_message=message, payload=payload, + response=response, backend=self) + for to in parse_address_list(to_header): + recipient_status[to.addr_spec.lower()] = AnymailRecipientStatus( + message_id=message_id, status='sent') + # Sadly, have to parse human-readable message to figure out if everyone got it: + # "Message OK, but will not deliver to these inactive addresses: {addr_spec, ...}. + # Inactive recipients are ones that have generated a hard bounce or a spam complaint." + reject_addr_specs = self._addr_specs_from_error_msg( + msg, r'inactive addresses:\s*(.*)\.\s*Inactive recipients') + for reject_addr_spec in reject_addr_specs: + recipient_status[reject_addr_spec] = AnymailRecipientStatus( + message_id=None, status='rejected') + + elif error_code == 300: # Invalid email request + # Either the From address or at least one recipient was invalid. Email not sent. + # response["To"] is not populated for this error; must examine response["Message"]: + # "Invalid 'To' address: '{addr_spec}'." + # "Error parsing 'To': Illegal email domain '{domain}' in address '{addr_spec}'." + # "Error parsing 'To': Illegal email address '{addr_spec}'. It must contain the '@' symbol." + # "Invalid 'From' address: '{email_address}'." + if "'From' address" in msg: + # Normal error + raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response, + backend=self) + else: + # Use AnymailRecipientsRefused logic + invalid_addr_specs = self._addr_specs_from_error_msg(msg, r"address:?\s*'(.*)'") + for invalid_addr_spec in invalid_addr_specs: + recipient_status[invalid_addr_spec] = AnymailRecipientStatus( + message_id=None, status='invalid') + + elif error_code == 406: # Inactive recipient + # All recipients were rejected as hard-bounce or spam-complaint. Email not sent. + # response["To"] is not populated for this error; must examine response["Message"]: + # "You tried to send to a recipient that has been marked as inactive.\n + # Found inactive addresses: {addr_spec, ...}.\n + # Inactive recipients are ones that have generated a hard bounce or a spam complaint. " + reject_addr_specs = self._addr_specs_from_error_msg( + msg, r'inactive addresses:\s*(.*)\.\s*Inactive recipients') + for reject_addr_spec in reject_addr_specs: + recipient_status[reject_addr_spec] = AnymailRecipientStatus( + message_id=None, status='rejected') + + else: # Other error raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response, backend=self) - else: - # Use AnymailRecipientsRefused logic - default_status = 'invalid' - elif error_code == 406: # Inactive recipient - # All recipients were rejected as hard-bounce or spam-complaint. Email not sent. - default_status = 'rejected' - elif error_code == 0: - # At least partial success, and email was sent. - # Sadly, have to parse human-readable message to figure out if everyone got it. - default_status = 'sent' - rejected_emails = self.parse_inactive_recipients(msg) - else: - raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response, - backend=self) - return { - recipient.addr_spec: AnymailRecipientStatus( - message_id=message_id, - status=('rejected' if recipient.addr_spec.lower() in rejected_emails - else default_status) - ) - for recipient in payload.all_recipients - } + return recipient_status - def parse_inactive_recipients(self, msg): - """Return a list of 'inactive' email addresses from a Postmark "OK" response + @staticmethod + def _addr_specs_from_error_msg(error_msg, pattern): + """Extract a list of email addr_specs from Postmark error_msg. - :param str msg: the "Message" from the Postmark API response + pattern must be a re whose first group matches a comma-separated + list of addr_specs in the message """ - # Example msg with inactive recipients: - # "Message OK, but will not deliver to these inactive addresses: one@xample.com, two@example.com." - # " Inactive recipients are ones that have generated a hard bounce or a spam complaint." - # Example msg with everything OK: "OK" - match = re.search(r'inactive addresses:\s*(.*)\.\s*Inactive recipients', msg) + match = re.search(pattern, error_msg, re.MULTILINE) if match: emails = match.group(1) # "one@xample.com, two@example.com" return [email.strip().lower() for email in emails.split(',')] @@ -101,15 +133,23 @@ class PostmarkPayload(RequestsPayload): # 'X-Postmark-Server-Token': see get_request_params (and set_esp_extra) } self.server_token = backend.server_token # added to headers later, so esp_extra can override - self.all_recipients = [] # used for backend.parse_recipient_status + self.to_emails = [] + self.merge_data = None super(PostmarkPayload, self).__init__(message, defaults, backend, headers=headers, *args, **kwargs) def get_api_endpoint(self): + batch_send = self.merge_data is not None and len(self.to_emails) > 1 if 'TemplateId' in self.data or 'TemplateModel' in self.data: - # This is the one Postmark API documented to have a trailing slash. (Typo?) - return "email/withTemplate/" + if batch_send: + return "email/batchWithTemplates" + else: + # This is the one Postmark API documented to have a trailing slash. (Typo?) + return "email/withTemplate/" else: - return "email" + if batch_send: + return "email/batch" + else: + return "email" def get_request_params(self, api_url): params = super(PostmarkPayload, self).get_request_params(api_url) @@ -117,7 +157,26 @@ class PostmarkPayload(RequestsPayload): return params def serialize_data(self): - return self.serialize_json(self.data) + data = self.data + api_endpoint = self.get_api_endpoint() + if api_endpoint == "email/batchWithTemplates": + data = {"Messages": [self.data_for_recipient(to) for to in self.to_emails]} + elif api_endpoint == "email/batch": + data = [self.data_for_recipient(to) for to in self.to_emails] + return self.serialize_json(data) + + def data_for_recipient(self, to): + data = self.data.copy() + data["To"] = to.address + if self.merge_data and to.addr_spec in self.merge_data: + recipient_data = self.merge_data[to.addr_spec] + if "TemplateModel" in data: + # merge recipient_data into merge_global_data + data["TemplateModel"] = data["TemplateModel"].copy() + data["TemplateModel"].update(recipient_data) + else: + data["TemplateModel"] = recipient_data + return data # # Payload construction @@ -136,7 +195,8 @@ class PostmarkPayload(RequestsPayload): if emails: field = recipient_type.capitalize() self.data[field] = ', '.join([email.address for email in emails]) - self.all_recipients += emails # used for backend.parse_recipient_status + if recipient_type == "to": + self.to_emails = emails def set_subject(self, subject): self.data["Subject"] = subject @@ -204,7 +264,9 @@ class PostmarkPayload(RequestsPayload): if field in self.data and not self.data[field]: del self.data[field] - # merge_data: Postmark doesn't support per-recipient substitutions + def set_merge_data(self, merge_data): + # late-bind + self.merge_data = merge_data def set_merge_global_data(self, merge_global_data): self.data["TemplateModel"] = merge_global_data diff --git a/docs/esps/postmark.rst b/docs/esps/postmark.rst index 5714d36..44d459a 100644 --- a/docs/esps/postmark.rst +++ b/docs/esps/postmark.rst @@ -127,39 +127,49 @@ see :ref:`unsupported-features`. Batch sending/merge and ESP templates ------------------------------------- -Postmark supports :ref:`ESP stored templates ` -populated with global merge data for all recipients, but does not -offer :ref:`batch sending ` with per-recipient merge data. -Anymail's :attr:`~anymail.message.AnymailMessage.merge_data` -message attribute is not supported with the Postmark backend. +Postmark offers both :ref:`ESP stored templates ` +and :ref:`batch sending ` with per-recipient merge data. + +.. versionchanged:: 4.2 + + Added Postmark :attr:`~anymail.message.AnymailMessage.merge_data` and batch sending + support. (Earlier Anymail releases only supported + :attr:`~anymail.message.AnymailMessage.merge_global_data` with Postmark.) To use a Postmark template, set the message's -:attr:`~anymail.message.AnymailMessage.template_id` to the numeric -Postmark "TemplateID" and supply the "TemplateModel" using -the :attr:`~anymail.message.AnymailMessage.merge_global_data` -message attribute: +:attr:`~anymail.message.AnymailMessage.template_id` to the numeric Postmark +"TemplateID" (*not* its name or "TemplateAlias"). You can find a template's +numeric id near the top right in Postmark's template editor. + +Supply the Postmark "TemplateModel" variables using Anymail's normalized +:attr:`~anymail.message.AnymailMessage.merge_data` and +:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes: .. code-block:: python message = EmailMessage( # (subject and body come from the template, so don't include those) - from_email="from@example.com", - to=["alice@example.com"] # single recipient... - # ...multiple to emails would all get the same message - # (and would all see each other's emails in the "to" header) + to=["alice@example.com", "Bob "] ) - message.template_id = 80801 # use this Postmark template + message.template_id = 80801 # Postmark template id + message.merge_data = { + 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, + 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, + } message.merge_global_data = { - 'name': "Alice", - 'order_no': "12345", 'ship_date': "May 15", - 'items': [ - {'product': "Widget", 'price': "9.99"}, - {'product': "Gadget", 'price': "17.99"}, - ], } Postmark does not allow overriding the message's subject or body with a template. +(You can customize the subject by including variables in the template's subject.) + +When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`, +Anymail automatically switches to Postmark's batch send API, so that +each "to" recipient sees only their own email address. (Any cc's or bcc's will be +duplicated for *every* to-recipient.) + +If you want to use batch sending with a regular message (without a template), set +merge data to an empty dict: `message.merge_data = {}`. See this `Postmark blog post on templates`_ for more information. diff --git a/tests/test_postmark_backend.py b/tests/test_postmark_backend.py index c72bc9a..c2fd6dc 100644 --- a/tests/test_postmark_backend.py +++ b/tests/test_postmark_backend.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import json from base64 import b64encode from decimal import Decimal from email.mime.base import MIMEBase @@ -386,11 +386,102 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase): self.assertNotIn('TextBody', data) def test_merge_data(self): - self.message.merge_data = { - 'alice@example.com': {'name': "Alice", 'group': "Developers"}, - } - with self.assertRaisesMessage(AnymailUnsupportedFeature, 'merge_data'): - self.message.send() + self.set_mock_response(raw=json.dumps([{ + "ErrorCode": 0, + "Message": "OK", + "To": "alice@example.com", + "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", + "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817", + }, { + "ErrorCode": 0, + "Message": "OK", + "To": "bob@example.com", + "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", + "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d", + }]).encode('utf-8')) + + message = AnymailMessage( + from_email='from@example.com', + template_id=1234567, # Postmark only supports merge_data content in a template + to=['alice@example.com', 'Bob '], + merge_data={ + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, # and leave group undefined + 'nobody@example.com': {'name': "Not a recipient for this message"}, + }, + merge_global_data={'group': "Users", 'site': "ExampleCo"} + ) + message.send() + + self.assert_esp_called('/email/batchWithTemplates') + data = self.get_api_call_json() + messages = data["Messages"] + self.assertEqual(len(messages), 2) + self.assertEqual(messages[0], { + "From": "from@example.com", + "To": "alice@example.com", + "TemplateId": 1234567, + "TemplateModel": {"name": "Alice", "group": "Developers", "site": "ExampleCo"}, + }) + self.assertEqual(messages[1], { + "From": "from@example.com", + "To": "Bob ", + "TemplateId": 1234567, + "TemplateModel": {"name": "Bob", "group": "Users", "site": "ExampleCo"}, + }) + + recipients = message.anymail_status.recipients + self.assertEqual(recipients['alice@example.com'].status, 'sent') + self.assertEqual(recipients['alice@example.com'].message_id, 'b7bc2f4a-e38e-4336-af7d-e6c392c2f817') + self.assertEqual(recipients['bob@example.com'].status, 'sent') + self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d') + + def test_merge_data_no_template(self): + # merge_data={} can be used to force batch sending without a template + self.set_mock_response(raw=json.dumps([{ + "ErrorCode": 0, + "Message": "OK", + "To": "alice@example.com", + "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", + "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817", + }, { + "ErrorCode": 0, + "Message": "OK", + "To": "bob@example.com", + "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", + "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d", + }]).encode('utf-8')) + + message = AnymailMessage( + from_email='from@example.com', + to=['alice@example.com', 'Bob '], + merge_data={}, + subject="Test batch send", + body="Test body", + ) + message.send() + + self.assert_esp_called('/email/batch') + data = self.get_api_call_json() + self.assertEqual(len(data), 2) + self.assertEqual(data[0], { + "From": "from@example.com", + "To": "alice@example.com", + "Subject": "Test batch send", + "TextBody": "Test body", + }) + self.assertEqual(data[1], { + "From": "from@example.com", + "To": "Bob ", + "Subject": "Test batch send", + "TextBody": "Test body", + }) + + recipients = message.anymail_status.recipients + self.assertEqual(recipients['alice@example.com'].status, 'sent') + self.assertEqual(recipients['alice@example.com'].message_id, 'b7bc2f4a-e38e-4336-af7d-e6c392c2f817') + self.assertEqual(recipients['bob@example.com'].status, 'sent') + self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d') def test_default_omits_options(self): """Make sure by default we don't send any ESP-specific options. @@ -433,10 +524,11 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase): response_content = b"""{ "MessageID":"abcdef01-2345-6789-0123-456789abcdef", "ErrorCode":0, + "To":"Recipient ", "Message":"OK" }""" self.set_mock_response(raw=response_content) - msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['to1@example.com'],) + msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['Recipient '],) sent = msg.send() self.assertEqual(sent, 1) self.assertEqual(msg.anymail_status.status, {'sent'}) diff --git a/tests/test_postmark_integration.py b/tests/test_postmark_integration.py index 9ad80e7..8951b6b 100644 --- a/tests/test_postmark_integration.py +++ b/tests/test_postmark_integration.py @@ -58,11 +58,12 @@ class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): reply_to=["reply1@example.com", "Reply 2 "], headers={"X-Anymail-Test": "value"}, - # no send_at, track_clicks support + # no send_at support metadata={"meta1": "simple string", "meta2": 2}, tags=["tag 1"], # max one tag track_opens=True, track_clicks=True, + merge_data={}, # force batch send (distinct message for each `to`) ) message.attach("attachment1.txt", "Here is some\ntext for you", "text/plain") message.attach("attachment2.csv", "ID,Name\n1,Amy Lina", "text/csv") @@ -74,6 +75,11 @@ class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): message.send() self.assertEqual(message.anymail_status.status, {'sent'}) + self.assertEqual(message.anymail_status.recipients['test+to1@anymail.info'].status, 'sent') + self.assertEqual(message.anymail_status.recipients['test+to2@anymail.info'].status, 'sent') + # distinct messages should have different message_ids: + self.assertNotEqual(message.anymail_status.recipients['test+to1@anymail.info'].message_id, + message.anymail_status.recipients['test+to2@anymail.info'].message_id) def test_invalid_from(self): self.message.from_email = 'webmaster@localhost' # Django's default From @@ -90,9 +96,13 @@ class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): def test_template(self): message = AnymailMessage( from_email="from@test-pm.anymail.info", - to=["test+to1@anymail.info"], + to=["test+to1@anymail.info", "Second Recipient "], template_id=POSTMARK_TEST_TEMPLATE_ID, - merge_global_data={"name": "Test Recipient", "order_no": "12345"}, + merge_data={ + "test+to1@anymail.info": {"name": "Recipient 1", "order_no": "12345"}, + "test+to2@anymail.info": {"order_no": "6789"}, + }, + merge_global_data={"name": "Valued Customer"}, ) message.send() self.assertEqual(message.anymail_status.status, {'sent'})