Mailgun: make merge_data work with stored handlebars templates

Mailgun has two different template mechanisms and two different ways
of providing substitution variables to them. Update Anymail's
normalized merge_data handling to work with either (while preserving
existing batch send and metadata capabilities that also use Mailgun's
custom data and recipient variables parameters).

Completes work started by @anstosa in #156.
Closes #155.
This commit is contained in:
Mike Edmunds
2019-09-03 11:51:19 -07:00
committed by GitHub
parent 8143b76041
commit df29ee2da6
6 changed files with 349 additions and 53 deletions

View File

@@ -25,6 +25,19 @@ Release history
^^^^^^^^^^^^^^^
.. This extra heading level keeps the ToC from becoming unmanageably long
vNext
-----
*Not yet released*
Features
~~~~~~~~
* **Mailgun:** Support Mailgun's new (ESP stored) handlebars templates via `template_id`.
See `docs <https://anymail.readthedocs.io/en/latest/esps/mailgun/#batch-sending-merge-and-esp-templates>`__.
(Thanks `@anstosa`_.)
v6.1
----
@@ -964,6 +977,7 @@ Features
.. _#153: https://github.com/anymail/issues/153
.. _@ailionx: https://github.com/ailionx
.. _@anstosa: https://github.com/anstosa
.. _@calvin: https://github.com/calvin
.. _@costela: https://github.com/costela
.. _@decibyte: https://github.com/decibyte

View File

@@ -119,51 +119,115 @@ class MailgunPayload(RequestsPayload):
return params
def serialize_data(self):
if self.is_batch() or self.merge_global_data:
self.populate_recipient_variables()
return self.data
# A not-so-brief digression about Mailgun's batch sending, template personalization,
# and metadata tracking capabilities...
#
# Mailgun has two kinds of templates:
# * ESP-stored templates (handlebars syntax), referenced by template name in the
# send API, with substitution data supplied as "custom data" variables.
# Anymail's `template_id` maps to this feature.
# * On-the-fly templating (`%recipient.KEY%` syntax), with template variables
# appearing directly in the message headers and/or body, and data supplied
# as "recipient variables" per-recipient personalizations. Mailgun docs also
# sometimes refer to this data as "template variables," but it's distinct from
# the substitution data used for stored handelbars templates.
#
# Mailgun has two mechanisms for supplying additional data with a message:
# * "Custom data" is supplied via `v:KEY` and/or `h:X-Mailgun-Variables` fields.
# Custom data is passed to tracking webhooks (as 'user-variables') and is
# available for `{{substitutions}}` in ESP-stored handlebars templates.
# Normally, the same custom data is applied to every recipient of a message.
# * "Recipient variables" are supplied via the `recipient-variables` field, and
# provide per-recipient data for batch sending. The recipient specific values
# are available as `%recipient.KEY%` virtually anywhere in the message
# (including header fields and other parameters).
#
# Anymail needs both mechanisms to map its normalized metadata and template merge_data
# features to Mailgun:
# (1) Anymail's `metadata` maps directly to Mailgun's custom data, where it can be
# accessed from webhooks.
# (2) Anymail's `merge_metadata` (per-recipient metadata for batch sends) maps
# *indirectly* through recipient-variables to Mailgun's custom data. To avoid
# conflicts, the recipient-variables mapping prepends 'v:' to merge_metadata keys.
# (E.g., Mailgun's custom-data "user" is set to "%recipient.v:user", which picks
# up its per-recipient value from Mailgun's `recipient-variables[to_email]["v:user"]`.)
# (3) Anymail's `merge_data` (per-recipient template substitutions) maps directly to
# Mailgun's `recipient-variables`, where it can be referenced in on-the-fly templates.
# (4) Anymail's `merge_global_data` (global template substitutions) is copied to
# Mailgun's `recipient-variables` for every recipient, as the default for missing
# `merge_data` keys.
# (5) Only if a stored template is used, `merge_data` and `merge_global_data` are
# *also* mapped *indirectly* through recipient-variables to Mailgun's custom data,
# where they can be referenced in handlebars {{substitutions}}.
# (E.g., Mailgun's custom-data "name" is set to "%recipient.name%", which picks
# up its per-recipient value from Mailgun's `recipient-variables[to_email]["name"]`.)
#
# If Anymail's `merge_data`, `template_id` (stored templates) and `metadata` (or
# `merge_metadata`) are used together, there's a possibility of conflicting keys in
# Mailgun's custom data. Anymail treats that conflict as an unsupported feature error.
def populate_recipient_variables(self):
"""Populate Mailgun recipient-variables from merge data and metadata"""
merge_metadata_keys = set() # all keys used in any recipient's merge_metadata
for recipient_metadata in self.merge_metadata.values():
merge_metadata_keys.update(recipient_metadata.keys())
metadata_vars = {key: "v:%s" % key for key in merge_metadata_keys} # custom-var for key
"""Populate Mailgun recipient-variables and custom data from merge data and metadata"""
# (numbers refer to detailed explanation above)
# Mailgun parameters to construct:
recipient_variables = {}
custom_data = {}
# Set up custom-var substitutions for merge metadata
# data['v:SomeMergeMetadataKey'] = '%recipient.v:SomeMergeMetadataKey%'
for var in metadata_vars.values():
self.data[var] = "%recipient.{var}%".format(var=var)
# (1) metadata --> Mailgun custom_data
custom_data.update(self.metadata)
# Any (toplevel) metadata that is also in (any) merge_metadata must be be moved
# into recipient-variables; and all merge_metadata vars must have defaults
# (else they'll get the '%recipient.v:SomeMergeMetadataKey%' literal string).
base_metadata = {metadata_vars[key]: self.metadata.get(key, '')
# (2) merge_metadata --> Mailgun custom_data via recipient_variables
if self.merge_metadata:
def vkey(key): # 'v:key'
return 'v:{}'.format(key)
merge_metadata_keys = flatset( # all keys used in any recipient's merge_metadata
recipient_data.keys() for recipient_data in self.merge_metadata.values())
custom_data.update({ # custom_data['key'] = '%recipient.v:key%' indirection
key: '%recipient.{}%'.format(vkey(key))
for key in merge_metadata_keys})
base_recipient_data = { # defaults for each recipient must cover all keys
vkey(key): self.metadata.get(key, '')
for key in merge_metadata_keys}
for email in self.to_emails:
this_recipient_data = base_recipient_data.copy()
this_recipient_data.update({
vkey(key): value
for key, value in self.merge_metadata.get(email, {}).items()})
recipient_variables.setdefault(email, {}).update(this_recipient_data)
recipient_vars = {}
for addr in self.to_emails:
# For each recipient, Mailgun recipient-variables[addr] is merger of:
# 1. metadata, for any keys that appear in merge_metadata
recipient_data = base_metadata.copy()
# (3) and (4) merge_data, merge_global_data --> Mailgun recipient_variables
if self.merge_data or self.merge_global_data:
merge_data_keys = flatset( # all keys used in any recipient's merge_data
recipient_data.keys() for recipient_data in self.merge_data.values())
merge_data_keys = merge_data_keys.union(self.merge_global_data.keys())
base_recipient_data = { # defaults for each recipient must cover all keys
key: self.merge_global_data.get(key, '')
for key in merge_data_keys}
for email in self.to_emails:
this_recipient_data = base_recipient_data.copy()
this_recipient_data.update(self.merge_data.get(email, {}))
recipient_variables.setdefault(email, {}).update(this_recipient_data)
# 2. merge_metadata[addr], with keys prefixed with 'v:'
if addr in self.merge_metadata:
recipient_data.update({
metadata_vars[key]: value for key, value in self.merge_metadata[addr].items()
})
# (5) if template, also map Mailgun custom_data to per-recipient_variables
if self.data.get('template') is not None:
conflicts = merge_data_keys.intersection(custom_data.keys())
if conflicts:
self.unsupported_feature(
"conflicting merge_data and metadata keys (%s) when using template_id"
% ', '.join("'%s'" % key for key in conflicts))
custom_data.update({ # custom_data['key'] = '%recipient.key%' indirection
key: '%recipient.{}%'.format(key)
for key in merge_data_keys})
# 3. merge_global_data (because Mailgun doesn't support global variables)
recipient_data.update(self.merge_global_data)
# 4. merge_data[addr]
if addr in self.merge_data:
recipient_data.update(self.merge_data[addr])
if recipient_data:
recipient_vars[addr] = recipient_data
self.data['recipient-variables'] = self.serialize_json(recipient_vars)
# populate Mailgun params
self.data.update({'v:%s' % key: value
for key, value in custom_data.items()})
if recipient_variables or self.is_batch():
self.data['recipient-variables'] = self.serialize_json(recipient_variables)
#
# Payload construction
@@ -285,3 +349,12 @@ def isascii(s):
except UnicodeEncodeError:
return False
return True
def flatset(iterables):
"""Return a set of the items in a single-level flattening of iterables
>>> flatset([1, 2], [2, 3])
set(1, 2, 3)
"""
return set(item for iterable in iterables for item in iterable)

View File

@@ -46,7 +46,7 @@ Email Service Provider |Amazon SES| |Mailgun| |Mailje
.. rubric:: :ref:`templates-and-merge`
---------------------------------------------------------------------------------------------------------------------------------------------------
:attr:`~AnymailMessage.template_id` Yes No Yes Yes Yes Yes Yes Yes
:attr:`~AnymailMessage.template_id` Yes Yes Yes Yes Yes Yes Yes Yes
:attr:`~AnymailMessage.merge_data` Yes Yes Yes Yes No Yes No Yes
:attr:`~AnymailMessage.merge_global_data` Yes (emulated) Yes Yes Yes Yes Yes Yes

View File

@@ -217,6 +217,13 @@ Limitations and quirks
the message to send, so it won't be present in your Mailgun API logs or the metadata
that is sent to tracking webhooks.)
**Additional limitations on merge_data with template_id**
If you are using Mailgun's stored handlebars templates (Anymail's
:attr:`~anymail.message.AnymailMessage.template_id`), :attr:`~anymail.message.AnymailMessage.merge_data`
cannot contain complex types or have any keys that conflict with
:attr:`~anymail.message.AnymailMessage.metadata`. See :ref:`mailgun-template-limitations`
below for more details.
**merge_metadata values default to empty string**
If you use Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata` feature,
and you supply metadata keys for some recipients but not others, Anymail will first
@@ -233,20 +240,43 @@ Limitations and quirks
.. _mailgun-templates:
Batch sending/merge
Batch sending/merge and ESP templates
-------------------------------------
Mailgun supports :ref:`batch sending <batch-send>` with per-recipient
merge data. You can refer to Mailgun "recipient variables" in your
message subject and body, and supply the values with Anymail's
normalized :attr:`~anymail.message.AnymailMessage.merge_data`
and :attr:`~anymail.message.AnymailMessage.merge_global_data`
message attributes:
Mailgun supports :ref:`ESP stored templates <esp-stored-templates>`, on-the-fly
templating, and :ref:`batch sending <batch-send>` with per-recipient merge data.
.. versionchanged:: 6.2
Added support for Mailgun's stored (handlebars) templates.
Mailgun has two different syntaxes for substituting data into templates:
* "Recipient variables" look like ``%recipient.name%``, and are used with on-the-fly
templates. You can refer to a recipient variable inside a message's body, subject,
or other message attributes defined in your Django code. See `Mailgun batch sending`_
for more information. (Note that Mailgun's docs also sometimes refer to recipient
variables as "template *variables*," and there are some additional predefined ones
described in their docs.)
* "Template *substitutions*" look like ``{{ name }}``, and can *only* be used in
handlebars templates that are defined and stored in your Mailgun account (via
the Mailgun dashboard or API). You refer to a stored template using Anymail's
:attr:`~anymail.message.AnymailMessage.template_id` in your Django code.
See `Mailgun templates`_ for more information.
With either type of template, you supply the substitution data using Anymail's
normalized :attr:`~anymail.message.AnymailMessage.merge_data` and
:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes. Anymail
will figure out the correct Mailgun API parameters to use.
Here's an example defining an on-the-fly template that uses Mailgun recipient variables:
.. code-block:: python
message = EmailMessage(
...
from_email="shipping@example.com",
# Use %recipient.___% syntax in subject and body:
subject="Your order %recipient.order_no% has shipped",
body="""Hi %recipient.name%,
We shipped your order %recipient.order_no%
@@ -262,15 +292,97 @@ message attributes:
'ship_date': "May 15" # Anymail maps globals to all recipients
}
Mailgun does not natively support global merge data. Anymail emulates
the capability by copying any `merge_global_data` values to each
recipient's section in Mailgun's "recipient-variables" API parameter.
And here's an example that uses the same data with a stored template, which could refer
to ``{{ name }}``, ``{{ order_no }}``, and ``{{ ship_date }}`` in its definition:
.. code-block:: python
message = EmailMessage(
from_email="shipping@example.com",
# The message body and html_body come from from the stored template.
# (You can still use %recipient.___% fields in the subject:)
subject="Your order %recipient.order_no% has shipped",
to=["alice@example.com", "Bob <bob@example.com>"]
)
message.template_id = 'shipping-notification' # name of template in our account
# The substitution data is exactly the same as in the previous example:
message.merge_data = {
'alice@example.com': {'name': "Alice", 'order_no': "12345"},
'bob@example.com': {'name': "Bob", 'order_no': "54321"},
}
message.merge_global_data = {
'ship_date': "May 15" # Anymail maps globals to all recipients
}
When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`,
Anymail supplies Mailgun's ``recipient-variables`` parameter, which puts Mailgun
in batch sending mode so that each "to" recipient sees only their own email address.
(Any cc's or bcc's will be duplicated for *every* to-recipient.)
If you want to use batch sending with a regular message (without a template), set
merge data to an empty dict: `message.merge_data = {}`.
Mailgun does not natively support global merge data. Anymail emulates
the capability by copying any :attr:`~anymail.message.AnymailMessage.merge_global_data`
values to every recipient.
.. _mailgun-template-limitations:
Limitations with stored handlebars templates
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Although Anymail tries to insulate you from Mailgun's relatively complicated API
parameters for template substitutions in batch sends, there are two cases it can't
handle. These *only* apply to stored handlebars templates (when you've set Anymail's
:attr:`~anymail.message.AnymailMessage.template_id` attribute).
First, metadata and template merge data substitutions use the same underlying
"custom data" API parameters when a handlebars template is used. If you have any
duplicate keys between your tracking metadata
(:attr:`~anymail.message.AnymailMessage.metadata`/:attr:`~anymail.message.AnymailMessage.merge_metadata`)
and your template merge data
(:attr:`~anymail.message.AnymailMessage.merge_data`/:attr:`~anymail.message.AnymailMessage.merge_global_data`),
Anymail will raise an :exc:`~anymail.exceptions.AnymailUnsupportedFeature` error.
Second, Mailgun's API does not allow complex data types like lists or dicts to be
passed as template substitutions for a batch send (confirmed with Mailgun support
8/2019). Your Anymail :attr:`~anymail.message.AnymailMessage.merge_data` and
:attr:`~anymail.message.AnymailMessage.merge_global_data` should only use simple
types like string or number. This means you cannot use the handlebars ``{{#each item}}``
block helper or dotted field notation like ``{{object.field}}`` with data passed
through Anymail's normalized merge data attributes.
Most ESPs do not support complex merge data types, so trying to do that is not recommended
anyway, for portability reasons. But if you *do* want to pass complex types to Mailgun
handlebars templates, and you're only sending to one recipient at a time, here's a
(non-portable!) workaround:
.. code-block:: python
# Using complex substitutions with Mailgun handlebars templates.
# This works only for a single recipient, and is not at all portable between ESPs.
message = EmailMessage(
from_email="shipping@example.com",
to=["alice@example.com"] # single recipient *only* (no batch send)
subject="Your order has shipped", # recipient variables *not* available
)
message.template_id = 'shipping-notification' # name of template in our account
substitutions = {
'items': [ # complex substitution data
{'product': "Anvil", 'quantity': 1},
{'product': "Tacks", 'quantity': 100},
],
'ship_date': "May 15",
}
# Do *not* set Anymail's message.merge_data, merge_global_data, or merge_metadata.
# Instead add Mailgun custom variables directly:
message.extra_headers['X-Mailgun-Variables'] = json.dumps(substitutions)
See the `Mailgun batch sending`_ docs for more information.
.. _Mailgun batch sending:
https://documentation.mailgun.com/user_manual.html#batch-sending
https://documentation.mailgun.com/en/latest/user_manual.html#batch-sending
.. _Mailgun templates:
https://documentation.mailgun.com/en/latest/user_manual.html#templates
.. _mailgun-webhooks:

View File

@@ -490,7 +490,39 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
'bob@example.com': {'test': "value"},
})
def test_merge_data_with_template(self):
# Mailgun *stored* (handlebars) templates get their variable substitutions
# from Mailgun's custom-data (not recipient-variables). To support batch sends
# with stored templates, Anymail sets up custom-data to pull values from
# recipient-variables. (Note this same Mailgun custom-data is also used for
# webhook metadata tracking.)
self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
self.message.template_id = 'welcome_template'
self.message.merge_data = {
'alice@example.com': {'name': "Alice", 'group': "Developers"},
'bob@example.com': {'name': "Bob"}, # and leave group undefined
}
self.message.merge_global_data = {
'group': "Users", # default
'site': "ExampleCo",
}
self.message.send()
data = self.get_api_call_data()
# custom-data variables for merge_data refer to recipient-variables:
self.assertEqual(data['v:name'], '%recipient.name%')
self.assertEqual(data['v:group'], '%recipient.group%')
self.assertEqual(data['v:site'], '%recipient.site%')
# recipient-variables populates them:
self.assertJSONEqual(data['recipient-variables'], {
'alice@example.com': {'name': "Alice", 'group': "Developers", 'site': "ExampleCo"},
'bob@example.com': {'name': "Bob", 'group': "Users", 'site': "ExampleCo"},
})
def test_merge_metadata(self):
# Per-recipient custom-data uses the same recipient-variables mechanism
# as above, but prepends 'v:' to the recipient-data keys for metadata to
# keep them separate.
# (For on-the-fly templates -- not stored handlebars templates.)
self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
self.message.merge_metadata = {
'alice@example.com': {'order_id': 123, 'tier': 'premium'},
@@ -528,10 +560,54 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
self.assertJSONEqual(data['recipient-variables'], {
'alice@example.com': {'name': "Alice", 'group': "Developers",
'v:order_id': 123, 'v:tier': 'premium'},
'bob@example.com': {'name': "Bob", # undefined merge_data --> omitted
'bob@example.com': {'name': "Bob", 'group': '', # undefined merge_data --> empty string
'v:order_id': 678, 'v:tier': ''}, # undefined metadata --> empty string
})
def test_merge_data_with_merge_metadata_and_template(self):
# This case gets tricky, because when a stored template is used, the per-recipient
# merge_metadata and merge_data both end up in the same Mailgun custom-data keys.
self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
self.message.template_id = 'order_notification'
self.message.merge_data = {
'alice@example.com': {'name': "Alice", 'group': "Developers"},
'bob@example.com': {'name': "Bob"}, # and leave group undefined
}
self.message.merge_metadata = {
'alice@example.com': {'order_id': 123, 'tier': 'premium'},
'bob@example.com': {'order_id': 678}, # and leave tier undefined
}
self.message.send()
data = self.get_api_call_data()
# custom-data covers both merge_data and merge_metadata:
self.assertEqual(data['v:name'], '%recipient.name%') # from merge_data
self.assertEqual(data['v:group'], '%recipient.group%') # from merge_data
self.assertEqual(data['v:order_id'], '%recipient.v:order_id%') # from merge_metadata
self.assertEqual(data['v:tier'], '%recipient.v:tier%') # from merge_metadata
self.assertJSONEqual(data['recipient-variables'], {
'alice@example.com': {'name': "Alice", 'group': "Developers",
'v:order_id': 123, 'v:tier': 'premium'},
'bob@example.com': {'name': "Bob", 'group': '', # undefined merge_data --> empty string
'v:order_id': 678, 'v:tier': ''}, # undefined metadata --> empty string
})
def test_conflicting_merge_data_with_merge_metadata_and_template(self):
# When a stored template is used, the same Mailgun custom-data must hold both
# per-recipient merge_data and metadata, so there's potential for conflict.
self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
self.message.template_id = 'order_notification'
self.message.merge_data = {
'alice@example.com': {'name': "Alice", 'group': "Developers"},
'bob@example.com': {'name': "Bob"},
}
self.message.metadata = {'group': "Order processing subsystem"}
with self.assertRaisesMessage(
AnymailUnsupportedFeature,
"conflicting merge_data and metadata keys ('group') when using template_id"
):
self.message.send()
def test_force_batch(self):
# Mailgun uses presence of recipient-variables to indicate batch send
self.message.to = ['alice@example.com', 'Bob <bob@example.com>']

View File

@@ -161,6 +161,27 @@ class MailgunBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
# (We could try fetching the message from event["storage"]["url"]
# to verify content and other headers.)
def test_stored_template(self):
message = AnymailMessage(
template_id='test-template', # name of a real template named in Anymail's Mailgun test account
subject='Your order %recipient.order%', # Mailgun templates don't define subject
from_email='Test From <from@example.com>', # Mailgun templates don't define sender
to=["test+to1@anymail.info"],
# metadata and merge_data must not have any conflicting keys when using template_id
metadata={"meta1": "simple string", "meta2": 2},
merge_data={
'test+to1@anymail.info': {
'name': "Test Recipient",
}
},
merge_global_data={
'order': '12345',
},
)
message.send()
recipient_status = message.anymail_status.recipients
self.assertEqual(recipient_status['test+to1@anymail.info'].status, 'queued')
# As of Anymail 0.10, this test is no longer possible, because
# Anymail now raises AnymailInvalidAddress without even calling Mailgun
# def test_invalid_from(self):