Mailgun: make merge_data work with stored handlebars templates

Mailgun has two different template mechanisms and two different ways
of providing substitution variables to them. Update Anymail's
normalized merge_data handling to work with either (while preserving
existing batch send and metadata capabilities that also use Mailgun's
custom data and recipient variables parameters).

Completes work started by @anstosa in #156.
Closes #155.

CHANGELOG.rst

@@ -25,6 +25,19 @@ Release history
 ^^^^^^^^^^^^^^^
     ..  This extra heading level keeps the ToC from becoming unmanageably long
 
+vNext
+-----
+
+*Not yet released*
+
+Features
+~~~~~~~~
+
+* **Mailgun:** Support Mailgun's new (ESP stored) handlebars templates via `template_id`.
+  See `docs <https://anymail.readthedocs.io/en/latest/esps/mailgun/#batch-sending-merge-and-esp-templates>`__.
+  (Thanks `@anstosa`_.)
+
+
 v6.1
 ----
 
@@ -964,6 +977,7 @@ Features
 .. _#153: https://github.com/anymail/issues/153
 
 .. _@ailionx: https://github.com/ailionx
+.. _@anstosa: https://github.com/anstosa
 .. _@calvin: https://github.com/calvin
 .. _@costela: https://github.com/costela
 .. _@decibyte: https://github.com/decibyte

anymail/backends/mailgun.py

@@ -119,51 +119,115 @@ class MailgunPayload(RequestsPayload):
         return params
 
     def serialize_data(self):
-        if self.is_batch() or self.merge_global_data:
         self.populate_recipient_variables()
         return self.data
 
+    # A not-so-brief digression about Mailgun's batch sending, template personalization,
+    # and metadata tracking capabilities...
+    #
+    # Mailgun has two kinds of templates:
+    #   * ESP-stored templates (handlebars syntax), referenced by template name in the
+    #     send API, with substitution data supplied as "custom data" variables.
+    #     Anymail's `template_id` maps to this feature.
+    #   * On-the-fly templating (`%recipient.KEY%` syntax), with template variables
+    #     appearing directly in the message headers and/or body, and data supplied
+    #     as "recipient variables" per-recipient personalizations. Mailgun docs also
+    #     sometimes refer to this data as "template variables," but it's distinct from
+    #     the substitution data used for stored handelbars templates.
+    #
+    # Mailgun has two mechanisms for supplying additional data with a message:
+    #   * "Custom data" is supplied via `v:KEY` and/or `h:X-Mailgun-Variables` fields.
+    #     Custom data is passed to tracking webhooks (as 'user-variables') and is
+    #     available for `{{substitutions}}` in ESP-stored handlebars templates.
+    #     Normally, the same custom data is applied to every recipient of a message.
+    #   * "Recipient variables" are supplied via the `recipient-variables` field, and
+    #     provide per-recipient data for batch sending. The recipient specific values
+    #     are available as `%recipient.KEY%` virtually anywhere in the message
+    #     (including header fields and other parameters).
+    #
+    # Anymail needs both mechanisms to map its normalized metadata and template merge_data
+    # features to Mailgun:
+    # (1) Anymail's `metadata` maps directly to Mailgun's custom data, where it can be
+    #     accessed from webhooks.
+    # (2) Anymail's `merge_metadata` (per-recipient metadata for batch sends) maps
+    #     *indirectly* through recipient-variables to Mailgun's custom data. To avoid
+    #     conflicts, the recipient-variables mapping prepends 'v:' to merge_metadata keys.
+    #     (E.g., Mailgun's custom-data "user" is set to "%recipient.v:user", which picks
+    #     up its per-recipient value from Mailgun's `recipient-variables[to_email]["v:user"]`.)
+    # (3) Anymail's `merge_data` (per-recipient template substitutions) maps directly to
+    #     Mailgun's `recipient-variables`, where it can be referenced in on-the-fly templates.
+    # (4) Anymail's `merge_global_data` (global template substitutions) is copied to
+    #     Mailgun's `recipient-variables` for every recipient, as the default for missing
+    #     `merge_data` keys.
+    # (5) Only if a stored template is used, `merge_data` and `merge_global_data` are
+    #     *also* mapped *indirectly* through recipient-variables to Mailgun's custom data,
+    #     where they can be referenced in handlebars {{substitutions}}.
+    #     (E.g., Mailgun's custom-data "name" is set to "%recipient.name%", which picks
+    #     up its per-recipient value from Mailgun's `recipient-variables[to_email]["name"]`.)
+    #
+    # If Anymail's `merge_data`, `template_id` (stored templates) and `metadata` (or
+    # `merge_metadata`) are used together, there's a possibility of conflicting keys in
+    # Mailgun's custom data. Anymail treats that conflict as an unsupported feature error.
+
     def populate_recipient_variables(self):
-        """Populate Mailgun recipient-variables from merge data and metadata"""
+        """Populate Mailgun recipient-variables and custom data from merge data and metadata"""
-        merge_metadata_keys = set()  # all keys used in any recipient's merge_metadata
+        # (numbers refer to detailed explanation above)
-        for recipient_metadata in self.merge_metadata.values():
+        # Mailgun parameters to construct:
-            merge_metadata_keys.update(recipient_metadata.keys())
+        recipient_variables = {}
-        metadata_vars = {key: "v:%s" % key for key in merge_metadata_keys}  # custom-var for key
+        custom_data = {}
 
-        # Set up custom-var substitutions for merge metadata
+        # (1) metadata --> Mailgun custom_data
-        # data['v:SomeMergeMetadataKey'] = '%recipient.v:SomeMergeMetadataKey%'
+        custom_data.update(self.metadata)
-        for var in metadata_vars.values():
-            self.data[var] = "%recipient.{var}%".format(var=var)
 
-        # Any (toplevel) metadata that is also in (any) merge_metadata must be be moved
+        # (2) merge_metadata --> Mailgun custom_data via recipient_variables
-        # into recipient-variables; and all merge_metadata vars must have defaults
+        if self.merge_metadata:
-        # (else they'll get the '%recipient.v:SomeMergeMetadataKey%' literal string).
+            def vkey(key):  # 'v:key'
-        base_metadata = {metadata_vars[key]: self.metadata.get(key, '')
+                return 'v:{}'.format(key)
+
+            merge_metadata_keys = flatset(  # all keys used in any recipient's merge_metadata
+                recipient_data.keys() for recipient_data in self.merge_metadata.values())
+            custom_data.update({  # custom_data['key'] = '%recipient.v:key%' indirection
+                key: '%recipient.{}%'.format(vkey(key))
+                for key in merge_metadata_keys})
+            base_recipient_data = {  # defaults for each recipient must cover all keys
+                vkey(key): self.metadata.get(key, '')
                 for key in merge_metadata_keys}
+            for email in self.to_emails:
+                this_recipient_data = base_recipient_data.copy()
+                this_recipient_data.update({
+                    vkey(key): value
+                    for key, value in self.merge_metadata.get(email, {}).items()})
+                recipient_variables.setdefault(email, {}).update(this_recipient_data)
 
-        recipient_vars = {}
+        # (3) and (4) merge_data, merge_global_data --> Mailgun recipient_variables
-        for addr in self.to_emails:
+        if self.merge_data or self.merge_global_data:
-            # For each recipient, Mailgun recipient-variables[addr] is merger of:
+            merge_data_keys = flatset(  # all keys used in any recipient's merge_data
-            # 1. metadata, for any keys that appear in merge_metadata
+                recipient_data.keys() for recipient_data in self.merge_data.values())
-            recipient_data = base_metadata.copy()
+            merge_data_keys = merge_data_keys.union(self.merge_global_data.keys())
+            base_recipient_data = {  # defaults for each recipient must cover all keys
+                key: self.merge_global_data.get(key, '')
+                for key in merge_data_keys}
+            for email in self.to_emails:
+                this_recipient_data = base_recipient_data.copy()
+                this_recipient_data.update(self.merge_data.get(email, {}))
+                recipient_variables.setdefault(email, {}).update(this_recipient_data)
 
-            # 2. merge_metadata[addr], with keys prefixed with 'v:'
+            # (5) if template, also map Mailgun custom_data to per-recipient_variables
-            if addr in self.merge_metadata:
+            if self.data.get('template') is not None:
-                recipient_data.update({
+                conflicts = merge_data_keys.intersection(custom_data.keys())
-                    metadata_vars[key]: value for key, value in self.merge_metadata[addr].items()
+                if conflicts:
-                })
+                    self.unsupported_feature(
+                        "conflicting merge_data and metadata keys (%s) when using template_id"
+                        % ', '.join("'%s'" % key for key in conflicts))
+                custom_data.update({  # custom_data['key'] = '%recipient.key%' indirection
+                    key: '%recipient.{}%'.format(key)
+                    for key in merge_data_keys})
 
-            # 3. merge_global_data (because Mailgun doesn't support global variables)
+        # populate Mailgun params
-            recipient_data.update(self.merge_global_data)
+        self.data.update({'v:%s' % key: value
-
+                          for key, value in custom_data.items()})
-            # 4. merge_data[addr]
+        if recipient_variables or self.is_batch():
-            if addr in self.merge_data:
+            self.data['recipient-variables'] = self.serialize_json(recipient_variables)
-                recipient_data.update(self.merge_data[addr])
-
-            if recipient_data:
-                recipient_vars[addr] = recipient_data
-
-        self.data['recipient-variables'] = self.serialize_json(recipient_vars)
 
     #
     # Payload construction
@@ -285,3 +349,12 @@ def isascii(s):
     except UnicodeEncodeError:
         return False
     return True
+
+
+def flatset(iterables):
+    """Return a set of the items in a single-level flattening of iterables
+
+    >>> flatset([1, 2], [2, 3])
+    set(1, 2, 3)
+    """
+    return set(item for iterable in iterables for item in iterable)

docs/esps/index.rst

@@ -46,7 +46,7 @@ Email Service Provider                        |Amazon SES|  |Mailgun|    |Mailje
 
 .. rubric:: :ref:`templates-and-merge`
 ---------------------------------------------------------------------------------------------------------------------------------------------------
-:attr:`~AnymailMessage.template_id`           Yes           No           Yes         Yes          Yes         Yes         Yes           Yes
+:attr:`~AnymailMessage.template_id`           Yes           Yes          Yes         Yes          Yes         Yes         Yes           Yes
 :attr:`~AnymailMessage.merge_data`            Yes           Yes          Yes         Yes          No          Yes         No            Yes
 :attr:`~AnymailMessage.merge_global_data`     Yes           (emulated)   Yes         Yes          Yes         Yes         Yes           Yes
 

docs/esps/mailgun.rst

@@ -217,6 +217,13 @@ Limitations and quirks
   the message to send, so it won't be present in your Mailgun API logs or the metadata
   that is sent to tracking webhooks.)
 
+**Additional limitations on merge_data with template_id**
+  If you are using Mailgun's stored handlebars templates (Anymail's
+  :attr:`~anymail.message.AnymailMessage.template_id`), :attr:`~anymail.message.AnymailMessage.merge_data`
+  cannot contain complex types or have any keys that conflict with
+  :attr:`~anymail.message.AnymailMessage.metadata`. See :ref:`mailgun-template-limitations`
+  below for more details.
+
 **merge_metadata values default to empty string**
   If you use Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata` feature,
   and you supply metadata keys for some recipients but not others, Anymail will first
@@ -233,20 +240,43 @@ Limitations and quirks
 
 .. _mailgun-templates:
 
-Batch sending/merge
+Batch sending/merge and ESP templates
 -------------------------------------
 
-Mailgun supports :ref:`batch sending <batch-send>` with per-recipient
+Mailgun supports :ref:`ESP stored templates <esp-stored-templates>`, on-the-fly
-merge data. You can refer to Mailgun "recipient variables" in your
+templating, and :ref:`batch sending <batch-send>` with per-recipient merge data.
-message subject and body, and supply the values with Anymail's
+
-normalized :attr:`~anymail.message.AnymailMessage.merge_data`
+.. versionchanged:: 6.2
-and :attr:`~anymail.message.AnymailMessage.merge_global_data`
+
-message attributes:
+  Added support for Mailgun's stored (handlebars) templates.
+
+Mailgun has two different syntaxes for substituting data into templates:
+
+* "Recipient variables" look like ``%recipient.name%``, and are used with on-the-fly
+  templates. You can refer to a recipient variable inside a message's body, subject,
+  or other message attributes defined in your Django code. See `Mailgun batch sending`_
+  for more information. (Note that Mailgun's docs also sometimes refer to recipient
+  variables as "template *variables*," and there are some additional predefined ones
+  described in their docs.)
+
+* "Template *substitutions*" look like ``{{ name }}``, and can *only* be used in
+  handlebars templates that are defined and stored in your Mailgun account (via
+  the Mailgun dashboard or API). You refer to a stored template using Anymail's
+  :attr:`~anymail.message.AnymailMessage.template_id` in your Django code.
+  See `Mailgun templates`_ for more information.
+
+With either type of template, you supply the substitution data using Anymail's
+normalized :attr:`~anymail.message.AnymailMessage.merge_data` and
+:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes. Anymail
+will figure out the correct Mailgun API parameters to use.
+
+Here's an example defining an on-the-fly template that uses Mailgun recipient variables:
 
   .. code-block:: python
 
       message = EmailMessage(
-          ...
+          from_email="shipping@example.com",
+          # Use %recipient.___% syntax in subject and body:
           subject="Your order %recipient.order_no% has shipped",
           body="""Hi %recipient.name%,
                   We shipped your order %recipient.order_no%
@@ -262,15 +292,97 @@ message attributes:
           'ship_date': "May 15"  # Anymail maps globals to all recipients
       }
 
-Mailgun does not natively support global merge data. Anymail emulates
+And here's an example that uses the same data with a stored template, which could refer
-the capability by copying any `merge_global_data` values to each
+to ``{{ name }}``, ``{{ order_no }}``, and ``{{ ship_date }}`` in its definition:
-recipient's section in Mailgun's "recipient-variables" API parameter.
+
+  .. code-block:: python
+
+      message = EmailMessage(
+          from_email="shipping@example.com",
+          # The message body and html_body come from from the stored template.
+          # (You can still use %recipient.___% fields in the subject:)
+          subject="Your order %recipient.order_no% has shipped",
+          to=["alice@example.com", "Bob <bob@example.com>"]
+      )
+      message.template_id = 'shipping-notification'  # name of template in our account
+      # The substitution data is exactly the same as in the previous example:
+      message.merge_data = {
+          'alice@example.com': {'name': "Alice", 'order_no': "12345"},
+          'bob@example.com': {'name': "Bob", 'order_no': "54321"},
+      }
+      message.merge_global_data = {
+          'ship_date': "May 15"  # Anymail maps globals to all recipients
+      }
+
+When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`,
+Anymail supplies Mailgun's ``recipient-variables`` parameter, which puts Mailgun
+in batch sending mode so that each "to" recipient sees only their own email address.
+(Any cc's or bcc's will be duplicated for *every* to-recipient.)
+
+If you want to use batch sending with a regular message (without a template), set
+merge data to an empty dict: `message.merge_data = {}`.
+
+Mailgun does not natively support global merge data. Anymail emulates
+the capability by copying any :attr:`~anymail.message.AnymailMessage.merge_global_data`
+values to every recipient.
+
+.. _mailgun-template-limitations:
+
+Limitations with stored handlebars templates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although Anymail tries to insulate you from Mailgun's relatively complicated API
+parameters for template substitutions in batch sends, there are two cases it can't
+handle. These *only* apply to stored handlebars templates (when you've set Anymail's
+:attr:`~anymail.message.AnymailMessage.template_id` attribute).
+
+First, metadata and template merge data substitutions use the same underlying
+"custom data" API parameters when a handlebars template is used. If you have any
+duplicate keys between your tracking metadata
+(:attr:`~anymail.message.AnymailMessage.metadata`/:attr:`~anymail.message.AnymailMessage.merge_metadata`)
+and your template merge data
+(:attr:`~anymail.message.AnymailMessage.merge_data`/:attr:`~anymail.message.AnymailMessage.merge_global_data`),
+Anymail will raise an :exc:`~anymail.exceptions.AnymailUnsupportedFeature` error.
+
+Second, Mailgun's API does not allow complex data types like lists or dicts to be
+passed as template substitutions for a batch send (confirmed with Mailgun support
+8/2019). Your Anymail :attr:`~anymail.message.AnymailMessage.merge_data` and
+:attr:`~anymail.message.AnymailMessage.merge_global_data` should only use simple
+types like string or number. This means you cannot use the handlebars ``{{#each item}}``
+block helper or dotted field notation like ``{{object.field}}`` with data passed
+through Anymail's normalized merge data attributes.
+
+Most ESPs do not support complex merge data types, so trying to do that is not recommended
+anyway, for portability reasons. But if you *do* want to pass complex types to Mailgun
+handlebars templates, and you're only sending to one recipient at a time, here's a
+(non-portable!) workaround:
+
+  .. code-block:: python
+
+      # Using complex substitutions with Mailgun handlebars templates.
+      # This works only for a single recipient, and is not at all portable between ESPs.
+      message = EmailMessage(
+          from_email="shipping@example.com",
+          to=["alice@example.com"]  # single recipient *only* (no batch send)
+          subject="Your order has shipped",  # recipient variables *not* available
+      )
+      message.template_id = 'shipping-notification'  # name of template in our account
+      substitutions = {
+          'items': [  # complex substitution data
+              {'product': "Anvil", 'quantity': 1},
+              {'product': "Tacks", 'quantity': 100},
+          ],
+          'ship_date': "May 15",
+      }
+      # Do *not* set Anymail's message.merge_data, merge_global_data, or merge_metadata.
+      # Instead add Mailgun custom variables directly:
+      message.extra_headers['X-Mailgun-Variables'] = json.dumps(substitutions)
 
-See the `Mailgun batch sending`_ docs for more information.
 
 .. _Mailgun batch sending:
-    https://documentation.mailgun.com/user_manual.html#batch-sending
+    https://documentation.mailgun.com/en/latest/user_manual.html#batch-sending
-
+.. _Mailgun templates:
+    https://documentation.mailgun.com/en/latest/user_manual.html#templates
 
 .. _mailgun-webhooks:
 

tests/test_mailgun_backend.py

@@ -490,7 +490,39 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
             'bob@example.com': {'test': "value"},
         })
 
+    def test_merge_data_with_template(self):
+        # Mailgun *stored* (handlebars) templates get their variable substitutions
+        # from Mailgun's custom-data (not recipient-variables). To support batch sends
+        # with stored templates, Anymail sets up custom-data to pull values from
+        # recipient-variables. (Note this same Mailgun custom-data is also used for
+        # webhook metadata tracking.)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.template_id = 'welcome_template'
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},  # and leave group undefined
+        }
+        self.message.merge_global_data = {
+            'group': "Users",  # default
+            'site': "ExampleCo",
+        }
+        self.message.send()
+        data = self.get_api_call_data()
+        # custom-data variables for merge_data refer to recipient-variables:
+        self.assertEqual(data['v:name'], '%recipient.name%')
+        self.assertEqual(data['v:group'], '%recipient.group%')
+        self.assertEqual(data['v:site'], '%recipient.site%')
+        # recipient-variables populates them:
+        self.assertJSONEqual(data['recipient-variables'], {
+            'alice@example.com': {'name': "Alice", 'group': "Developers", 'site': "ExampleCo"},
+            'bob@example.com': {'name': "Bob", 'group': "Users", 'site': "ExampleCo"},
+        })
+
     def test_merge_metadata(self):
+        # Per-recipient custom-data uses the same recipient-variables mechanism
+        # as above, but prepends 'v:' to the recipient-data keys for metadata to
+        # keep them separate.
+        # (For on-the-fly templates -- not stored handlebars templates.)
         self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
         self.message.merge_metadata = {
             'alice@example.com': {'order_id': 123, 'tier': 'premium'},
@@ -528,10 +560,54 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
         self.assertJSONEqual(data['recipient-variables'], {
             'alice@example.com': {'name': "Alice", 'group': "Developers",
                                   'v:order_id': 123, 'v:tier': 'premium'},
-            'bob@example.com': {'name': "Bob",  # undefined merge_data --> omitted
+            'bob@example.com': {'name': "Bob", 'group': '',  # undefined merge_data --> empty string
                                 'v:order_id': 678, 'v:tier': ''},  # undefined metadata --> empty string
         })
 
+    def test_merge_data_with_merge_metadata_and_template(self):
+        # This case gets tricky, because when a stored template is used, the per-recipient
+        # merge_metadata and merge_data both end up in the same Mailgun custom-data keys.
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.template_id = 'order_notification'
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},  # and leave group undefined
+        }
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},  # and leave tier undefined
+        }
+        self.message.send()
+
+        data = self.get_api_call_data()
+        # custom-data covers both merge_data and merge_metadata:
+        self.assertEqual(data['v:name'], '%recipient.name%')  # from merge_data
+        self.assertEqual(data['v:group'], '%recipient.group%')  # from merge_data
+        self.assertEqual(data['v:order_id'], '%recipient.v:order_id%')  # from merge_metadata
+        self.assertEqual(data['v:tier'], '%recipient.v:tier%')  # from merge_metadata
+        self.assertJSONEqual(data['recipient-variables'], {
+            'alice@example.com': {'name': "Alice", 'group': "Developers",
+                                  'v:order_id': 123, 'v:tier': 'premium'},
+            'bob@example.com': {'name': "Bob", 'group': '',  # undefined merge_data --> empty string
+                                'v:order_id': 678, 'v:tier': ''},  # undefined metadata --> empty string
+        })
+
+    def test_conflicting_merge_data_with_merge_metadata_and_template(self):
+        # When a stored template is used, the same Mailgun custom-data must hold both
+        # per-recipient merge_data and metadata, so there's potential for conflict.
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.template_id = 'order_notification'
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},
+        }
+        self.message.metadata = {'group': "Order processing subsystem"}
+        with self.assertRaisesMessage(
+            AnymailUnsupportedFeature,
+            "conflicting merge_data and metadata keys ('group') when using template_id"
+        ):
+            self.message.send()
+
     def test_force_batch(self):
         # Mailgun uses presence of recipient-variables to indicate batch send
         self.message.to = ['alice@example.com', 'Bob <bob@example.com>']

tests/test_mailgun_integration.py

@@ -161,6 +161,27 @@ class MailgunBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
         # (We could try fetching the message from event["storage"]["url"]
         # to verify content and other headers.)
 
+    def test_stored_template(self):
+        message = AnymailMessage(
+            template_id='test-template',  # name of a real template named in Anymail's Mailgun test account
+            subject='Your order %recipient.order%',  # Mailgun templates don't define subject
+            from_email='Test From <from@example.com>',  # Mailgun templates don't define sender
+            to=["test+to1@anymail.info"],
+            # metadata and merge_data must not have any conflicting keys when using template_id
+            metadata={"meta1": "simple string", "meta2": 2},
+            merge_data={
+                'test+to1@anymail.info': {
+                    'name': "Test Recipient",
+                }
+            },
+            merge_global_data={
+                'order': '12345',
+            },
+        )
+        message.send()
+        recipient_status = message.anymail_status.recipients
+        self.assertEqual(recipient_status['test+to1@anymail.info'].status, 'queued')
+
     # As of Anymail 0.10, this test is no longer possible, because
     # Anymail now raises AnymailInvalidAddress without even calling Mailgun
     # def test_invalid_from(self):