SendGrid: support new "dynamic" transactional templates

Closes #120

CHANGELOG.rst

@@ -26,6 +26,21 @@ Release history
     ..  This extra heading level keeps the ToC from becoming unmanageably long
 
 
+v4.1
+----
+
+*In development*
+
+.. TODO: when releasing, change "latest" to "stable" in docs links
+
+Features
+~~~~~~~~
+
+*  **SendGrid:** Support both new "dynamic" and original "legacy" transactional
+   templates. (See
+   `docs <https://anymail.readthedocs.io/en/latest/esps/sendgrid/#sendgrid-templates>`__.)
+
+
 v4.0
 ----
 

anymail/backends/sendgrid.py

@@ -72,6 +72,7 @@ class SendGridPayload(RequestsPayload):
         self.all_recipients = []  # used for backend.parse_recipient_status
         self.generate_message_id = backend.generate_message_id
         self.workaround_name_quote_bug = backend.workaround_name_quote_bug
+        self.use_dynamic_template = False  # how to represent merge_data
         self.message_id = None  # Message-ID -- assigned in serialize_data unless provided in headers
         self.merge_field_format = backend.merge_field_format
         self.merge_data = None  # late-bound per-recipient data
@@ -113,6 +114,41 @@ class SendGridPayload(RequestsPayload):
         self.data.setdefault("custom_args", {})["anymail_id"] = self.message_id
 
     def build_merge_data(self):
+        if self.use_dynamic_template:
+            self.build_merge_data_dynamic()
+        else:
+            self.build_merge_data_legacy()
+
+    def build_merge_data_dynamic(self):
+        """Set personalizations[...]['dynamic_template_data']"""
+        if self.merge_global_data is not None:
+            assert len(self.data["personalizations"]) == 1
+            self.data["personalizations"][0].setdefault(
+                "dynamic_template_data", {}).update(self.merge_global_data)
+
+        if self.merge_data is not None:
+            # Burst apart each to-email in personalizations[0] into a separate
+            # personalization, and add merge_data for that recipient
+            assert len(self.data["personalizations"]) == 1
+            base_personalizations = self.data["personalizations"].pop()
+            to_list = base_personalizations.pop("to")  # {email, name?} for each message.to
+            for recipient in to_list:
+                personalization = base_personalizations.copy()  # captures cc, bcc, merge_global_data, esp_extra
+                personalization["to"] = [recipient]
+                try:
+                    recipient_data = self.merge_data[recipient["email"]]
+                except KeyError:
+                    pass  # no merge_data for this recipient
+                else:
+                    if "dynamic_template_data" in personalization:
+                        # merge per-recipient data into (copy of) merge_global_data
+                        personalization["dynamic_template_data"] = personalization["dynamic_template_data"].copy()
+                        personalization["dynamic_template_data"].update(recipient_data)
+                    else:
+                        personalization["dynamic_template_data"] = recipient_data
+                self.data["personalizations"].append(personalization)
+
+    def build_merge_data_legacy(self):
         """Set personalizations[...]['substitutions'] and data['sections']"""
         merge_field_format = self.merge_field_format or '{}'
 
@@ -291,18 +327,26 @@ class SendGridPayload(RequestsPayload):
 
     def set_template_id(self, template_id):
         self.data["template_id"] = template_id
+        try:
+            self.use_dynamic_template = template_id.startswith("d-")
+        except AttributeError:
+            pass
 
     def set_merge_data(self, merge_data):
-        # Becomes personalizations[...]['substitutions'] in build_merge_data,
+        # Becomes personalizations[...]['dynamic_template_data']
-        # after we know recipients and merge_field_format.
+        # or personalizations[...]['substitutions'] in build_merge_data,
+        # after we know recipients, template type, and merge_field_format.
         self.merge_data = merge_data
 
     def set_merge_global_data(self, merge_global_data):
-        # Becomes data['section'] in build_merge_data, after we know merge_field_format.
+        # Becomes personalizations[...]['dynamic_template_data']
+        # or data['section'] in build_merge_data, after we know
+        # template type and merge_field_format.
         self.merge_global_data = merge_global_data
 
     def set_esp_extra(self, extra):
         self.merge_field_format = extra.pop("merge_field_format", self.merge_field_format)
+        self.use_dynamic_template = extra.pop("use_dynamic_template", self.use_dynamic_template)
         if "x-smtpapi" in extra:
             raise AnymailConfigurationError(
                 "You are attempting to use SendGrid v2 API-style x-smtpapi params "

docs/esps/sendgrid.rst

@@ -75,11 +75,10 @@ See :ref:`Message-ID quirks <sendgrid-message-id>` below.
 
 .. rubric:: SENDGRID_MERGE_FIELD_FORMAT
 
-If you use :ref:`merge data <merge-data>`, set this to a :meth:`str.format`
+If you use :ref:`merge data <merge-data>` with SendGrid's legacy transactional templates,
-formatting string that indicates how merge fields are delimited
+set this to a :meth:`str.format` formatting string that indicates how merge fields are
-in your SendGrid templates.
+delimited in your legacy templates. For example, if your templates use the ``-field-``
-For example, if your templates use the ``-field-`` hyphen delimiters
+hyphen delimiters suggested in some SendGrid docs, you would set:
-suggested in some SendGrid docs, you would set:
 
   .. code-block:: python
 
@@ -95,9 +94,12 @@ a literal brace character, double it up. (For example, Handlebars-style
 The default `None` requires you include the delimiters directly in your
 :attr:`~anymail.message.AnymailMessage.merge_data` keys.
 You can also override this setting for individual messages.
-See the notes on SendGrid :ref:`templates and merge <sendgrid-templates>`
+See the notes on SendGrid :ref:`templates and merge <sendgrid-legacy-templates>`
 below.
 
+This setting is not used (or necessary) with SendGrid's newer dynamic transactional
+templates, which always use Handlebars syntax.
+
 
 .. setting:: ANYMAIL_SENDGRID_API_URL
 
@@ -210,16 +212,29 @@ Batch sending/merge and ESP templates
 SendGrid offers both :ref:`ESP stored templates <esp-stored-templates>`
 and :ref:`batch sending <batch-send>` with per-recipient merge data.
 
-You can use a SendGrid stored template by setting a message's
+SendGrid has two types of stored templates for transactional email:
-:attr:`~anymail.message.AnymailMessage.template_id` to the
-template's unique id. Alternatively, you can refer to merge fields
-directly in an EmailMessage's subject and body---the message itself
-is used as an on-the-fly template.
 
-In either case, supply the merge data values with Anymail's
+* Dynamic transactional templates, which were introduced in July, 2018,
-normalized :attr:`~anymail.message.AnymailMessage.merge_data`
+  use Handlebars template syntax and allow complex logic to be coded in
-and :attr:`~anymail.message.AnymailMessage.merge_global_data`
+  the template itself.
-message attributes.
+
+* Legacy transactional templates, which allow only simple key-value substitution
+  and don't specify a particular template syntax.
+
+[Legacy templates were originally just called "transactional templates," and many older
+references still use this terminology. But confusingly, SendGrid's dashboard and some
+recent articles now use "transactional templates" to mean the newer, dynamic templates.]
+
+.. versionchanged:: 4.1
+
+    Added support for SendGrid dynamic transactional templates. (Earlier Anymail
+    releases work only with SendGrid's legacy transactional templates.)
+
+You can use either type of SendGrid stored template by setting a message's
+:attr:`~anymail.message.AnymailMessage.template_id` to the template's unique id
+(*not* its name). Supply the merge data values with Anymail's normalized
+:attr:`~anymail.message.AnymailMessage.merge_data` and
+:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes.
 
   .. code-block:: python
 
@@ -228,7 +243,7 @@ message attributes.
           # omit subject and body (or set to None) to use template content
           to=["alice@example.com", "Bob <bob@example.com>"]
       )
-      message.template_id = "5997fcf6-2b9f-484d-acd5-7e9a99f0dc1f"  # SendGrid id
+      message.template_id = "d-5a963add2ec84305813ff860db277d7a"  # SendGrid dynamic id
       message.merge_data = {
           'alice@example.com': {'name': "Alice", 'order_no': "12345"},
           'bob@example.com': {'name': "Bob", 'order_no': "54321"},
@@ -236,28 +251,6 @@ message attributes.
       message.merge_global_data = {
           'ship_date': "May 15",
       }
-      message.esp_extra = {
-          # Tell Anymail this SendGrid template uses "-field-" to refer to merge fields.
-          # (We could also just set SENDGRID_MERGE_FIELD_FORMAT in our ANYMAIL settings.)
-          'merge_field_format': "-{}-"
-      }
-
-SendGrid doesn't have a pre-defined merge field syntax, so you
-must tell Anymail how substitution fields are delimited in your templates.
-There are three ways you can do this:
-
-  * Set `'merge_field_format'` in the message's
-    :attr:`~anymail.message.AnymailMessage.esp_extra` to a python :meth:`str.format`
-    string, as shown in the example above. (This applies only to that
-    particular EmailMessage.)
-  * *Or* set :setting:`SENDGRID_MERGE_FIELD_FORMAT <ANYMAIL_SENDGRID_MERGE_FIELD_FORMAT>`
-    in your Anymail settings. This is usually the best approach, and will apply to all messages
-    sent through SendGrid. (You can still use esp_extra to override for individual messages.)
-  * *Or* include the field delimiters directly in *all* your
-    :attr:`~anymail.message.AnymailMessage.merge_data` and
-    :attr:`~anymail.message.AnymailMessage.merge_global_data` keys.
-    E.g.: ``{'-name-': "Alice", '-order_no-': "12345"}``.
-    (This can be error-prone, and difficult to move to other ESPs.)
 
 When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`,
 Anymail automatically changes how it communicates the "to" list to SendGrid, so that
@@ -265,19 +258,83 @@ so that each recipient sees only their own email address. (Anymail creates a sep
 "personalization" for each recipient in the "to" list; any cc's or bcc's will be
 duplicated for *every* to-recipient.)
 
-SendGrid templates allow you to mix your EmailMessage's `subject` and `body`
+See the `SendGrid's transactional template overview`_ for more information.
+
+.. _SendGrid's transactional template overview:
+    https://sendgrid.com/docs/ui/sending-email/create-and-edit-transactional-templates/
+
+
+.. _sendgrid-legacy-templates:
+
+Legacy transactional templates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+With *legacy* transactional templates (only), SendGrid doesn't have a pre-defined merge
+field syntax, so you must tell Anymail how substitution fields are delimited in your
+templates. There are three ways you can do this:
+
+  * Set `'merge_field_format'` in the message's
+    :attr:`~anymail.message.AnymailMessage.esp_extra` to a python :meth:`str.format`
+    string, as shown in the example below. (This applies only to that particular
+    EmailMessage.)
+  * *Or* set :setting:`SENDGRID_MERGE_FIELD_FORMAT <ANYMAIL_SENDGRID_MERGE_FIELD_FORMAT>`
+    in your Anymail settings. This is usually the best approach, and will apply to all
+    legacy template messages sent through SendGrid. (You can still use esp_extra to
+    override for individual messages.)
+  * *Or* include the field delimiters directly in *all* your
+    :attr:`~anymail.message.AnymailMessage.merge_data` and
+    :attr:`~anymail.message.AnymailMessage.merge_global_data` keys.
+    E.g.: ``{'-name-': "Alice", '-order_no-': "12345"}``.
+    (This can be error-prone, and makes it difficult to transition to other ESPs or to
+    SendGrid's dynamic templates.)
+
+  .. code-block:: python
+
+      # ...
+      message.template_id = "5997fcf6-2b9f-484d-acd5-7e9a99f0dc1f"  # SendGrid legacy id
+      message.merge_data = {
+          'alice@example.com': {'name': "Alice", 'order_no': "12345"},
+          'bob@example.com': {'name': "Bob", 'order_no': "54321"},
+      }
+      message.esp_extra = {
+          # Tell Anymail this SendGrid legacy template uses "-field-" for merge fields.
+          # (You could instead set SENDGRID_MERGE_FIELD_FORMAT in your ANYMAIL settings.)
+          'merge_field_format': "-{}-"
+      }
+
+SendGrid legacy templates allow you to mix your EmailMessage's `subject` and `body`
 with the template subject and body (by using `<%subject%>` and `<%body%>` in
 your SendGrid template definition where you want the message-specific versions
 to appear). If you don't want to supply any additional subject or body content
 from your Django app, set those EmailMessage attributes to empty strings or `None`.
 
-See the `SendGrid's template overview`_ and `transactional template docs`_
-for more information.
 
-.. _SendGrid's template overview:
+On-the-fly templates
-    https://sendgrid.com/docs/User_Guide/Transactional_Templates/index.html
+~~~~~~~~~~~~~~~~~~~~
-.. _transactional template docs:
+
-    https://sendgrid.com/docs/API_Reference/Web_API_v3/Transactional_Templates/smtpapi.html
+Rather than define a stored ESP template, you can refer to merge fields directly
+in an EmailMessage's subject and body, and SendGrid will treat this as an on-the-fly,
+legacy-style template definition. (The on-the-fly template can't contain any dynamic
+template logic, and like any legacy template you must specify the merge field format
+in either Anymail settings or esp_extra as described above.)
+
+  .. code-block:: python
+
+      # on-the-fly template using merge fields in subject and body:
+      message = EmailMessage(
+          subject="Your order {{order_no}} has shipped",
+          body="Dear {{name}}:\nWe've shipped order {{order_no}}.",
+          to=["alice@example.com", "Bob <bob@example.com>"]
+      )
+      # note: no template_id specified
+      message.merge_data = {
+          'alice@example.com': {'name': "Alice", 'order_no': "12345"},
+          'bob@example.com': {'name': "Bob", 'order_no': "54321"},
+      }
+      message.esp_extra = {
+          # here's how to get Handlebars-style {{merge}} fields with Python's str.format:
+          'merge_field_format': "{{{{{}}}}}"  # "{{ {{ {} }} }}" without the spaces
+      }
 
 
 .. _sendgrid-webhooks:

tests/test_sendgrid_backend.py

@@ -421,11 +421,71 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         self.assertNotIn('subject', data)
 
     def test_merge_data(self):
+        # A template_id starting with "d-" indicates you are using SendGrid's newer
+        # (non-legacy) "dynamic" transactional templates
+        self.message.template_id = "d-5a963add2ec84305813ff860db277d7a"
+
+        self.message.from_email = 'from@example.com'
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>', 'celia@example.com']
+        self.message.cc = ['cc@example.com']  # gets applied to *each* recipient in a merge
+
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},  # and leave group undefined
+            # and no data for celia@example.com
+        }
+        self.message.merge_global_data = {
+            'group': "Users",
+            'site': "ExampleCo",
+        }
+        self.message.send()
+
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'alice@example.com'}],
+             'cc': [{'email': 'cc@example.com'}],  # all recipients get the cc
+             'dynamic_template_data': {
+                 'name': "Alice", 'group': "Developers", 'site': "ExampleCo"}},
+            {'to': [{'email': 'bob@example.com', 'name': '"Bob"'}],
+             'cc': [{'email': 'cc@example.com'}],
+             'dynamic_template_data': {
+                 'name': "Bob", 'group': "Users", 'site': "ExampleCo"}},
+            {'to': [{'email': 'celia@example.com'}],
+             'cc': [{'email': 'cc@example.com'}],
+             'dynamic_template_data': {
+                 'group': "Users", 'site': "ExampleCo"}},
+        ])
+        self.assertNotIn('sections', data)  # 'sections' not used with dynamic templates
+
+    def test_explicit_dynamic_template(self):
+        # undocumented esp_extra['use_dynamic_template'] can be used to force dynamic/legacy params
+        self.message.merge_data = {'to@example.com': {"test": "data"}}
+
+        self.message.template_id = "apparently-not-dynamic"  # doesn't start with "d-"
+        self.message.esp_extra = {"use_dynamic_template": True}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'to@example.com'}],
+             'dynamic_template_data': {"test": "data"}}])
+
+        self.message.template_id = "d-apparently-not-legacy"
+        self.message.esp_extra = {"use_dynamic_template": False,
+                                  "merge_field_format": "<%{}%>"}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'to@example.com'}],
+             'substitutions': {"<%test%>": "data"}}])
+
+    def test_legacy_merge_data(self):
+        # unless a new "dynamic template" is specified, Anymail assumes the legacy
+        # "substitutions" format for merge data
         self.message.from_email = 'from@example.com'
         self.message.to = ['alice@example.com', 'Bob <bob@example.com>', 'celia@example.com']
         self.message.cc = ['cc@example.com']  # gets applied to *each* recipient in a merge
         # SendGrid template_id is not required to use merge.
-        # You can just supply template content as the message (e.g.):
+        # You can just supply (legacy) template content as the message (e.g.):
         self.message.body = "Hi :name. Welcome to :group at :site."
         self.message.merge_data = {
             # You must either include merge field delimiters in the keys (':name' rather than just 'name')
@@ -459,7 +519,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         })
 
     @override_settings(ANYMAIL_SENDGRID_MERGE_FIELD_FORMAT=":{}")  # :field as shown in SG examples
-    def test_merge_field_format_setting(self):
+    def test_legacy_merge_field_format_setting(self):
         # Provide merge field delimiters in settings.py
         self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
         self.message.merge_data = {
@@ -477,7 +537,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         ])
         self.assertEqual(data['sections'], {':site': "ExampleCo"})
 
-    def test_merge_field_format_esp_extra(self):
+    def test_legacy_merge_field_format_esp_extra(self):
         # Provide merge field delimiters for an individual message
         self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
         self.message.merge_data = {
@@ -498,7 +558,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         # Make sure our esp_extra merge_field_format doesn't get sent to SendGrid API:
         self.assertNotIn('merge_field_format', data)
 
-    def test_warn_if_no_merge_field_delimiters(self):
+    def test_legacy_warn_if_no_merge_field_delimiters(self):
         self.message.to = ['alice@example.com']
         self.message.merge_data = {
             'alice@example.com': {'name': "Alice", 'group': "Developers"},
@@ -506,7 +566,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         with self.assertWarnsRegex(AnymailWarning, r'SENDGRID_MERGE_FIELD_FORMAT'):
             self.message.send()
 
-    def test_warn_if_no_global_merge_field_delimiters(self):
+    def test_legacy_warn_if_no_global_merge_field_delimiters(self):
         self.message.merge_global_data = {'site': "ExampleCo"}
         with self.assertWarnsRegex(AnymailWarning, r'SENDGRID_MERGE_FIELD_FORMAT'):
             self.message.send()
@@ -536,6 +596,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
 
         for personalization in data['personalizations']:
             self.assertNotIn('custom_args', personalization)
+            self.assertNotIn('dynamic_template_data', personalization)
             self.assertNotIn('headers', personalization)
             self.assertNotIn('send_at', personalization)
             self.assertNotIn('substitutions', personalization)

tests/test_sendgrid_integration.py

@@ -115,14 +115,15 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
         message = AnymailMessage(
             from_email="Test From <from@example.com>",
             to=["to@sink.sendgrid.net"],
+            # Anymail's live test template has merge fields "name", "order_no", and "dept"...
             template_id=SENDGRID_TEST_TEMPLATE_ID,
-            # The test template in the Anymail Test account has a substitution "-field-":
+            merge_data={
-            merge_global_data={
+                'to@sink.sendgrid.net': {
-                'field': 'value from merge_global_data',
+                    'name': "Test Recipient",
+                    'order_no': "12345",
                 },
-            esp_extra={
-                'merge_field_format': '-{}-',
             },
+            merge_global_data={'dept': "Fulfillment"},
         )
         message.send()
         self.assertEqual(message.anymail_status.status, {'queued'})