From dbca13243f0d0866dd53c2288f27b0cfc6360705 Mon Sep 17 00:00:00 2001 From: medmunds Date: Fri, 24 Aug 2018 18:21:42 -0700 Subject: [PATCH] SendGrid: support new "dynamic" transactional templates Closes #120 --- CHANGELOG.rst | 15 +++ anymail/backends/sendgrid.py | 50 +++++++++- docs/esps/sendgrid.rst | 147 ++++++++++++++++++++--------- tests/test_sendgrid_backend.py | 71 +++++++++++++- tests/test_sendgrid_integration.py | 13 +-- 5 files changed, 237 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 783511b..c93f984 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,21 @@ Release history .. This extra heading level keeps the ToC from becoming unmanageably long +v4.1 +---- + +*In development* + +.. TODO: when releasing, change "latest" to "stable" in docs links + +Features +~~~~~~~~ + +* **SendGrid:** Support both new "dynamic" and original "legacy" transactional + templates. (See + `docs `__.) + + v4.0 ---- diff --git a/anymail/backends/sendgrid.py b/anymail/backends/sendgrid.py index 34e31f1..4210a0c 100644 --- a/anymail/backends/sendgrid.py +++ b/anymail/backends/sendgrid.py @@ -72,6 +72,7 @@ class SendGridPayload(RequestsPayload): self.all_recipients = [] # used for backend.parse_recipient_status self.generate_message_id = backend.generate_message_id self.workaround_name_quote_bug = backend.workaround_name_quote_bug + self.use_dynamic_template = False # how to represent merge_data self.message_id = None # Message-ID -- assigned in serialize_data unless provided in headers self.merge_field_format = backend.merge_field_format self.merge_data = None # late-bound per-recipient data @@ -113,6 +114,41 @@ class SendGridPayload(RequestsPayload): self.data.setdefault("custom_args", {})["anymail_id"] = self.message_id def build_merge_data(self): + if self.use_dynamic_template: + self.build_merge_data_dynamic() + else: + self.build_merge_data_legacy() + + def build_merge_data_dynamic(self): + """Set personalizations[...]['dynamic_template_data']""" + if self.merge_global_data is not None: + assert len(self.data["personalizations"]) == 1 + self.data["personalizations"][0].setdefault( + "dynamic_template_data", {}).update(self.merge_global_data) + + if self.merge_data is not None: + # Burst apart each to-email in personalizations[0] into a separate + # personalization, and add merge_data for that recipient + assert len(self.data["personalizations"]) == 1 + base_personalizations = self.data["personalizations"].pop() + to_list = base_personalizations.pop("to") # {email, name?} for each message.to + for recipient in to_list: + personalization = base_personalizations.copy() # captures cc, bcc, merge_global_data, esp_extra + personalization["to"] = [recipient] + try: + recipient_data = self.merge_data[recipient["email"]] + except KeyError: + pass # no merge_data for this recipient + else: + if "dynamic_template_data" in personalization: + # merge per-recipient data into (copy of) merge_global_data + personalization["dynamic_template_data"] = personalization["dynamic_template_data"].copy() + personalization["dynamic_template_data"].update(recipient_data) + else: + personalization["dynamic_template_data"] = recipient_data + self.data["personalizations"].append(personalization) + + def build_merge_data_legacy(self): """Set personalizations[...]['substitutions'] and data['sections']""" merge_field_format = self.merge_field_format or '{}' @@ -291,18 +327,26 @@ class SendGridPayload(RequestsPayload): def set_template_id(self, template_id): self.data["template_id"] = template_id + try: + self.use_dynamic_template = template_id.startswith("d-") + except AttributeError: + pass def set_merge_data(self, merge_data): - # Becomes personalizations[...]['substitutions'] in build_merge_data, - # after we know recipients and merge_field_format. + # Becomes personalizations[...]['dynamic_template_data'] + # or personalizations[...]['substitutions'] in build_merge_data, + # after we know recipients, template type, and merge_field_format. self.merge_data = merge_data def set_merge_global_data(self, merge_global_data): - # Becomes data['section'] in build_merge_data, after we know merge_field_format. + # Becomes personalizations[...]['dynamic_template_data'] + # or data['section'] in build_merge_data, after we know + # template type and merge_field_format. self.merge_global_data = merge_global_data def set_esp_extra(self, extra): self.merge_field_format = extra.pop("merge_field_format", self.merge_field_format) + self.use_dynamic_template = extra.pop("use_dynamic_template", self.use_dynamic_template) if "x-smtpapi" in extra: raise AnymailConfigurationError( "You are attempting to use SendGrid v2 API-style x-smtpapi params " diff --git a/docs/esps/sendgrid.rst b/docs/esps/sendgrid.rst index e69f370..41d0b43 100644 --- a/docs/esps/sendgrid.rst +++ b/docs/esps/sendgrid.rst @@ -75,11 +75,10 @@ See :ref:`Message-ID quirks ` below. .. rubric:: SENDGRID_MERGE_FIELD_FORMAT -If you use :ref:`merge data `, set this to a :meth:`str.format` -formatting string that indicates how merge fields are delimited -in your SendGrid templates. -For example, if your templates use the ``-field-`` hyphen delimiters -suggested in some SendGrid docs, you would set: +If you use :ref:`merge data ` with SendGrid's legacy transactional templates, +set this to a :meth:`str.format` formatting string that indicates how merge fields are +delimited in your legacy templates. For example, if your templates use the ``-field-`` +hyphen delimiters suggested in some SendGrid docs, you would set: .. code-block:: python @@ -95,9 +94,12 @@ a literal brace character, double it up. (For example, Handlebars-style The default `None` requires you include the delimiters directly in your :attr:`~anymail.message.AnymailMessage.merge_data` keys. You can also override this setting for individual messages. -See the notes on SendGrid :ref:`templates and merge ` +See the notes on SendGrid :ref:`templates and merge ` below. +This setting is not used (or necessary) with SendGrid's newer dynamic transactional +templates, which always use Handlebars syntax. + .. setting:: ANYMAIL_SENDGRID_API_URL @@ -210,16 +212,29 @@ Batch sending/merge and ESP templates SendGrid offers both :ref:`ESP stored templates ` and :ref:`batch sending ` with per-recipient merge data. -You can use a SendGrid stored template by setting a message's -:attr:`~anymail.message.AnymailMessage.template_id` to the -template's unique id. Alternatively, you can refer to merge fields -directly in an EmailMessage's subject and body---the message itself -is used as an on-the-fly template. +SendGrid has two types of stored templates for transactional email: -In either case, supply the merge data values with Anymail's -normalized :attr:`~anymail.message.AnymailMessage.merge_data` -and :attr:`~anymail.message.AnymailMessage.merge_global_data` -message attributes. +* Dynamic transactional templates, which were introduced in July, 2018, + use Handlebars template syntax and allow complex logic to be coded in + the template itself. + +* Legacy transactional templates, which allow only simple key-value substitution + and don't specify a particular template syntax. + +[Legacy templates were originally just called "transactional templates," and many older +references still use this terminology. But confusingly, SendGrid's dashboard and some +recent articles now use "transactional templates" to mean the newer, dynamic templates.] + +.. versionchanged:: 4.1 + + Added support for SendGrid dynamic transactional templates. (Earlier Anymail + releases work only with SendGrid's legacy transactional templates.) + +You can use either type of SendGrid stored template by setting a message's +:attr:`~anymail.message.AnymailMessage.template_id` to the template's unique id +(*not* its name). Supply the merge data values with Anymail's normalized +:attr:`~anymail.message.AnymailMessage.merge_data` and +:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes. .. code-block:: python @@ -228,7 +243,7 @@ message attributes. # omit subject and body (or set to None) to use template content to=["alice@example.com", "Bob "] ) - message.template_id = "5997fcf6-2b9f-484d-acd5-7e9a99f0dc1f" # SendGrid id + message.template_id = "d-5a963add2ec84305813ff860db277d7a" # SendGrid dynamic id message.merge_data = { 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, @@ -236,28 +251,6 @@ message attributes. message.merge_global_data = { 'ship_date': "May 15", } - message.esp_extra = { - # Tell Anymail this SendGrid template uses "-field-" to refer to merge fields. - # (We could also just set SENDGRID_MERGE_FIELD_FORMAT in our ANYMAIL settings.) - 'merge_field_format': "-{}-" - } - -SendGrid doesn't have a pre-defined merge field syntax, so you -must tell Anymail how substitution fields are delimited in your templates. -There are three ways you can do this: - - * Set `'merge_field_format'` in the message's - :attr:`~anymail.message.AnymailMessage.esp_extra` to a python :meth:`str.format` - string, as shown in the example above. (This applies only to that - particular EmailMessage.) - * *Or* set :setting:`SENDGRID_MERGE_FIELD_FORMAT ` - in your Anymail settings. This is usually the best approach, and will apply to all messages - sent through SendGrid. (You can still use esp_extra to override for individual messages.) - * *Or* include the field delimiters directly in *all* your - :attr:`~anymail.message.AnymailMessage.merge_data` and - :attr:`~anymail.message.AnymailMessage.merge_global_data` keys. - E.g.: ``{'-name-': "Alice", '-order_no-': "12345"}``. - (This can be error-prone, and difficult to move to other ESPs.) When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`, Anymail automatically changes how it communicates the "to" list to SendGrid, so that @@ -265,19 +258,83 @@ so that each recipient sees only their own email address. (Anymail creates a sep "personalization" for each recipient in the "to" list; any cc's or bcc's will be duplicated for *every* to-recipient.) -SendGrid templates allow you to mix your EmailMessage's `subject` and `body` +See the `SendGrid's transactional template overview`_ for more information. + +.. _SendGrid's transactional template overview: + https://sendgrid.com/docs/ui/sending-email/create-and-edit-transactional-templates/ + + +.. _sendgrid-legacy-templates: + +Legacy transactional templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With *legacy* transactional templates (only), SendGrid doesn't have a pre-defined merge +field syntax, so you must tell Anymail how substitution fields are delimited in your +templates. There are three ways you can do this: + + * Set `'merge_field_format'` in the message's + :attr:`~anymail.message.AnymailMessage.esp_extra` to a python :meth:`str.format` + string, as shown in the example below. (This applies only to that particular + EmailMessage.) + * *Or* set :setting:`SENDGRID_MERGE_FIELD_FORMAT ` + in your Anymail settings. This is usually the best approach, and will apply to all + legacy template messages sent through SendGrid. (You can still use esp_extra to + override for individual messages.) + * *Or* include the field delimiters directly in *all* your + :attr:`~anymail.message.AnymailMessage.merge_data` and + :attr:`~anymail.message.AnymailMessage.merge_global_data` keys. + E.g.: ``{'-name-': "Alice", '-order_no-': "12345"}``. + (This can be error-prone, and makes it difficult to transition to other ESPs or to + SendGrid's dynamic templates.) + + .. code-block:: python + + # ... + message.template_id = "5997fcf6-2b9f-484d-acd5-7e9a99f0dc1f" # SendGrid legacy id + message.merge_data = { + 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, + 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, + } + message.esp_extra = { + # Tell Anymail this SendGrid legacy template uses "-field-" for merge fields. + # (You could instead set SENDGRID_MERGE_FIELD_FORMAT in your ANYMAIL settings.) + 'merge_field_format': "-{}-" + } + +SendGrid legacy templates allow you to mix your EmailMessage's `subject` and `body` with the template subject and body (by using `<%subject%>` and `<%body%>` in your SendGrid template definition where you want the message-specific versions to appear). If you don't want to supply any additional subject or body content from your Django app, set those EmailMessage attributes to empty strings or `None`. -See the `SendGrid's template overview`_ and `transactional template docs`_ -for more information. -.. _SendGrid's template overview: - https://sendgrid.com/docs/User_Guide/Transactional_Templates/index.html -.. _transactional template docs: - https://sendgrid.com/docs/API_Reference/Web_API_v3/Transactional_Templates/smtpapi.html +On-the-fly templates +~~~~~~~~~~~~~~~~~~~~ + +Rather than define a stored ESP template, you can refer to merge fields directly +in an EmailMessage's subject and body, and SendGrid will treat this as an on-the-fly, +legacy-style template definition. (The on-the-fly template can't contain any dynamic +template logic, and like any legacy template you must specify the merge field format +in either Anymail settings or esp_extra as described above.) + + .. code-block:: python + + # on-the-fly template using merge fields in subject and body: + message = EmailMessage( + subject="Your order {{order_no}} has shipped", + body="Dear {{name}}:\nWe've shipped order {{order_no}}.", + to=["alice@example.com", "Bob "] + ) + # note: no template_id specified + message.merge_data = { + 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, + 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, + } + message.esp_extra = { + # here's how to get Handlebars-style {{merge}} fields with Python's str.format: + 'merge_field_format': "{{{{{}}}}}" # "{{ {{ {} }} }}" without the spaces + } .. _sendgrid-webhooks: diff --git a/tests/test_sendgrid_backend.py b/tests/test_sendgrid_backend.py index 218ccaa..b72352f 100644 --- a/tests/test_sendgrid_backend.py +++ b/tests/test_sendgrid_backend.py @@ -421,11 +421,71 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): self.assertNotIn('subject', data) def test_merge_data(self): + # A template_id starting with "d-" indicates you are using SendGrid's newer + # (non-legacy) "dynamic" transactional templates + self.message.template_id = "d-5a963add2ec84305813ff860db277d7a" + + self.message.from_email = 'from@example.com' + self.message.to = ['alice@example.com', 'Bob ', 'celia@example.com'] + self.message.cc = ['cc@example.com'] # gets applied to *each* recipient in a merge + + self.message.merge_data = { + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, # and leave group undefined + # and no data for celia@example.com + } + self.message.merge_global_data = { + 'group': "Users", + 'site': "ExampleCo", + } + self.message.send() + + data = self.get_api_call_json() + self.assertEqual(data['personalizations'], [ + {'to': [{'email': 'alice@example.com'}], + 'cc': [{'email': 'cc@example.com'}], # all recipients get the cc + 'dynamic_template_data': { + 'name': "Alice", 'group': "Developers", 'site': "ExampleCo"}}, + {'to': [{'email': 'bob@example.com', 'name': '"Bob"'}], + 'cc': [{'email': 'cc@example.com'}], + 'dynamic_template_data': { + 'name': "Bob", 'group': "Users", 'site': "ExampleCo"}}, + {'to': [{'email': 'celia@example.com'}], + 'cc': [{'email': 'cc@example.com'}], + 'dynamic_template_data': { + 'group': "Users", 'site': "ExampleCo"}}, + ]) + self.assertNotIn('sections', data) # 'sections' not used with dynamic templates + + def test_explicit_dynamic_template(self): + # undocumented esp_extra['use_dynamic_template'] can be used to force dynamic/legacy params + self.message.merge_data = {'to@example.com': {"test": "data"}} + + self.message.template_id = "apparently-not-dynamic" # doesn't start with "d-" + self.message.esp_extra = {"use_dynamic_template": True} + self.message.send() + data = self.get_api_call_json() + self.assertEqual(data['personalizations'], [ + {'to': [{'email': 'to@example.com'}], + 'dynamic_template_data': {"test": "data"}}]) + + self.message.template_id = "d-apparently-not-legacy" + self.message.esp_extra = {"use_dynamic_template": False, + "merge_field_format": "<%{}%>"} + self.message.send() + data = self.get_api_call_json() + self.assertEqual(data['personalizations'], [ + {'to': [{'email': 'to@example.com'}], + 'substitutions': {"<%test%>": "data"}}]) + + def test_legacy_merge_data(self): + # unless a new "dynamic template" is specified, Anymail assumes the legacy + # "substitutions" format for merge data self.message.from_email = 'from@example.com' self.message.to = ['alice@example.com', 'Bob ', 'celia@example.com'] self.message.cc = ['cc@example.com'] # gets applied to *each* recipient in a merge # SendGrid template_id is not required to use merge. - # You can just supply template content as the message (e.g.): + # You can just supply (legacy) template content as the message (e.g.): self.message.body = "Hi :name. Welcome to :group at :site." self.message.merge_data = { # You must either include merge field delimiters in the keys (':name' rather than just 'name') @@ -459,7 +519,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): }) @override_settings(ANYMAIL_SENDGRID_MERGE_FIELD_FORMAT=":{}") # :field as shown in SG examples - def test_merge_field_format_setting(self): + def test_legacy_merge_field_format_setting(self): # Provide merge field delimiters in settings.py self.message.to = ['alice@example.com', 'Bob '] self.message.merge_data = { @@ -477,7 +537,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): ]) self.assertEqual(data['sections'], {':site': "ExampleCo"}) - def test_merge_field_format_esp_extra(self): + def test_legacy_merge_field_format_esp_extra(self): # Provide merge field delimiters for an individual message self.message.to = ['alice@example.com', 'Bob '] self.message.merge_data = { @@ -498,7 +558,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): # Make sure our esp_extra merge_field_format doesn't get sent to SendGrid API: self.assertNotIn('merge_field_format', data) - def test_warn_if_no_merge_field_delimiters(self): + def test_legacy_warn_if_no_merge_field_delimiters(self): self.message.to = ['alice@example.com'] self.message.merge_data = { 'alice@example.com': {'name': "Alice", 'group': "Developers"}, @@ -506,7 +566,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): with self.assertWarnsRegex(AnymailWarning, r'SENDGRID_MERGE_FIELD_FORMAT'): self.message.send() - def test_warn_if_no_global_merge_field_delimiters(self): + def test_legacy_warn_if_no_global_merge_field_delimiters(self): self.message.merge_global_data = {'site': "ExampleCo"} with self.assertWarnsRegex(AnymailWarning, r'SENDGRID_MERGE_FIELD_FORMAT'): self.message.send() @@ -536,6 +596,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): for personalization in data['personalizations']: self.assertNotIn('custom_args', personalization) + self.assertNotIn('dynamic_template_data', personalization) self.assertNotIn('headers', personalization) self.assertNotIn('send_at', personalization) self.assertNotIn('substitutions', personalization) diff --git a/tests/test_sendgrid_integration.py b/tests/test_sendgrid_integration.py index f56fbd1..699196b 100644 --- a/tests/test_sendgrid_integration.py +++ b/tests/test_sendgrid_integration.py @@ -115,14 +115,15 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): message = AnymailMessage( from_email="Test From ", to=["to@sink.sendgrid.net"], + # Anymail's live test template has merge fields "name", "order_no", and "dept"... template_id=SENDGRID_TEST_TEMPLATE_ID, - # The test template in the Anymail Test account has a substitution "-field-": - merge_global_data={ - 'field': 'value from merge_global_data', - }, - esp_extra={ - 'merge_field_format': '-{}-', + merge_data={ + 'to@sink.sendgrid.net': { + 'name': "Test Recipient", + 'order_no': "12345", + }, }, + merge_global_data={'dept': "Fulfillment"}, ) message.send() self.assertEqual(message.anymail_status.status, {'queued'})