Add merge_metadata for other ESPs

Support merge_metadata in Mailgun, Mailjet, Mandrill, Postmark, 
SparkPost, and Test backends. (SendGrid covered in earlier PR.)

Also:
* Add `merge_metadata` to AnymailMessage, AnymailMessageMixin
* Add `is_batch()` logic to BasePayload, for consistent handling
* Docs

Note: Mailjet implementation switches *all* batch sending from their 
"Recipients" field to to the "Messages" array bulk sending option.
This allows an independent payload for each batch recipient.
In addition to supporting merge_metadata, this also removes the
prior limitation on mixing Cc/Bcc with merge_data.

Closes #141.

CHANGELOG.rst

@@ -39,6 +39,17 @@ Breaking changes
   code is doing something like `message.anymail_status.recipients[email.lower()]`,
   you should remove the `.lower()`
 
+Features
+~~~~~~~~
+
+* Add new `merge_metadata` option for providing per-recipient metadata in batch
+  sends. Available for all supported ESPs *except* Amazon SES and SendinBlue.
+  See `docs <https://anymail.readthedocs.io/en/latest/sending/anymail_additions/#anymail.message.AnymailMessage.merge_metadata>`_.
+  (Thanks `@janneThoft`_ for the idea and SendGrid implementation.)
+
+* **Mailjet:** Remove limitation on using `cc` or `bcc` together with `merge_data`.
+
+
 Fixes
 ~~~~~
 
@@ -908,6 +919,7 @@ Features
 .. _@calvin: https://github.com/calvin
 .. _@costela: https://github.com/costela
 .. _@decibyte: https://github.com/decibyte
+.. _@janneThoft: https://github.com/janneThoft
 .. _@joshkersey: https://github.com/joshkersey
 .. _@Lekensteyn: https://github.com/Lekensteyn
 .. _@lewistaylor: https://github.com/lewistaylor

anymail/backends/base.py

@@ -250,11 +250,16 @@ class BasePayload(object):
     )
     esp_message_attrs = ()  # subclasses can override
 
+    # If any of these attrs are set on a message, treat the message
+    # as a batch send (separate message for each `to` recipient):
+    batch_attrs = ('merge_data', 'merge_metadata')
+
     def __init__(self, message, defaults, backend):
         self.message = message
         self.defaults = defaults
         self.backend = backend
         self.esp_name = backend.esp_name
+        self._batch_attrs_used = {attr: UNSET for attr in self.batch_attrs}
 
         self.init_payload()
 
@@ -287,6 +292,20 @@ class BasePayload(object):
                     # AttributeError here? Your Payload subclass is missing a set_<attr> implementation
                     setter = getattr(self, 'set_%s' % attr)
                 setter(value)
+            if attr in self.batch_attrs:
+                self._batch_attrs_used[attr] = (value is not UNSET)
+
+    def is_batch(self):
+        """
+        Return True if the message should be treated as a batch send.
+
+        Intended to be used inside serialize_data or similar, after all relevant
+        attributes have been processed. Will error if called before that (e.g.,
+        inside a set_<attr> method or during __init__).
+        """
+        batch_attrs_used = self._batch_attrs_used.values()
+        assert UNSET not in batch_attrs_used, "Cannot call is_batch before all attributes processed"
+        return any(batch_attrs_used)
 
     def unsupported_feature(self, feature):
         if not self.backend.ignore_unsupported_features:

anymail/backends/mailgun.py

@@ -68,8 +68,10 @@ class MailgunPayload(RequestsPayload):
         self.all_recipients = []  # used for backend.parse_recipient_status
 
         # late-binding of recipient-variables:
-        self.merge_data = None
+        self.merge_data = {}
-        self.merge_global_data = None
+        self.merge_global_data = {}
+        self.metadata = {}
+        self.merge_metadata = {}
         self.to_emails = []
 
         super(MailgunPayload, self).__init__(message, defaults, backend, auth=auth, *args, **kwargs)
@@ -117,32 +119,51 @@ class MailgunPayload(RequestsPayload):
         return params
 
     def serialize_data(self):
-        self.populate_recipient_variables()
+        if self.is_batch() or self.merge_global_data:
+            self.populate_recipient_variables()
         return self.data
 
     def populate_recipient_variables(self):
-        """Populate Mailgun recipient-variables header from merge data"""
+        """Populate Mailgun recipient-variables from merge data and metadata"""
-        merge_data = self.merge_data
+        merge_metadata_keys = set()  # all keys used in any recipient's merge_metadata
+        for recipient_metadata in self.merge_metadata.values():
+            merge_metadata_keys.update(recipient_metadata.keys())
+        metadata_vars = {key: "v:%s" % key for key in merge_metadata_keys}  # custom-var for key
 
-        if self.merge_global_data is not None:
+        # Set up custom-var substitutions for merge metadata
-            # Mailgun doesn't support global variables.
+        # data['v:SomeMergeMetadataKey'] = '%recipient.v:SomeMergeMetadataKey%'
-            # We emulate them by populating recipient-variables for all recipients.
+        for var in metadata_vars.values():
-            if merge_data is not None:
+            self.data[var] = "%recipient.{var}%".format(var=var)
-                merge_data = merge_data.copy()  # don't modify the original, which doesn't belong to us
-            else:
-                merge_data = {}
-            for email in self.to_emails:
-                try:
-                    recipient_data = merge_data[email]
-                except KeyError:
-                    merge_data[email] = self.merge_global_data
-                else:
-                    # Merge globals (recipient_data wins in conflict)
-                    merge_data[email] = self.merge_global_data.copy()
-                    merge_data[email].update(recipient_data)
 
-        if merge_data is not None:
+        # Any (toplevel) metadata that is also in (any) merge_metadata must be be moved
-            self.data['recipient-variables'] = self.serialize_json(merge_data)
+        # into recipient-variables; and all merge_metadata vars must have defaults
+        # (else they'll get the '%recipient.v:SomeMergeMetadataKey%' literal string).
+        base_metadata = {metadata_vars[key]: self.metadata.get(key, '')
+                         for key in merge_metadata_keys}
+
+        recipient_vars = {}
+        for addr in self.to_emails:
+            # For each recipient, Mailgun recipient-variables[addr] is merger of:
+            # 1. metadata, for any keys that appear in merge_metadata
+            recipient_data = base_metadata.copy()
+
+            # 2. merge_metadata[addr], with keys prefixed with 'v:'
+            if addr in self.merge_metadata:
+                recipient_data.update({
+                    metadata_vars[key]: value for key, value in self.merge_metadata[addr].items()
+                })
+
+            # 3. merge_global_data (because Mailgun doesn't support global variables)
+            recipient_data.update(self.merge_global_data)
+
+            # 4. merge_data[addr]
+            if addr in self.merge_data:
+                recipient_data.update(self.merge_data[addr])
+
+            if recipient_data:
+                recipient_vars[addr] = recipient_data
+
+        self.data['recipient-variables'] = self.serialize_json(recipient_vars)
 
     #
     # Payload construction
@@ -210,6 +231,7 @@ class MailgunPayload(RequestsPayload):
         self.sender_domain = email.domain
 
     def set_metadata(self, metadata):
+        self.metadata = metadata  # save for handling merge_metadata later
         for key, value in metadata.items():
             self.data["v:%s" % key] = value
 
@@ -242,6 +264,10 @@ class MailgunPayload(RequestsPayload):
         # Processed at serialization time (to allow merging global data)
         self.merge_global_data = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # Processed at serialization time (to allow combining with merge_data)
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Allow override of sender_domain via esp_extra

anymail/backends/mailjet.py

@@ -80,8 +80,10 @@ class MailjetPayload(RequestsPayload):
             'Content-Type': 'application/json',
         }
         # Late binding of recipients and their variables
-        self.recipients = {}
+        self.recipients = {'to': []}
-        self.merge_data = None
+        self.metadata = None
+        self.merge_data = {}
+        self.merge_metadata = {}
         super(MailjetPayload, self).__init__(message, defaults, backend,
                                              auth=auth, headers=http_headers, *args, **kwargs)
 
@@ -89,22 +91,37 @@ class MailjetPayload(RequestsPayload):
         return "send"
 
     def serialize_data(self):
-        self._finish_recipients()
         self._populate_sender_from_template()
+        if self.is_batch():
+            self.data = {'Messages': [
+                self._data_for_recipient(to_addr)
+                for to_addr in self.recipients['to']
+            ]}
         return self.serialize_json(self.data)
 
-    #
+    def _data_for_recipient(self, email):
-    # Payload construction
+        # Return send data for single recipient, without modifying self.data
-    #
+        data = self.data.copy()
+        data['To'] = self._format_email_for_mailjet(email)
 
-    def _finish_recipients(self):
+        if email.addr_spec in self.merge_data:
-        # NOTE do not set both To and Recipients, it behaves specially: each
+            recipient_merge_data = self.merge_data[email.addr_spec]
-        # recipient receives a separate mail but the To address receives one
+            if 'Vars' in data:
-        # listing all recipients.
+                data['Vars'] = data['Vars'].copy()  # clone merge_global_data
-        if "cc" in self.recipients or "bcc" in self.recipients:
+                data['Vars'].update(recipient_merge_data)
-            self._finish_recipients_single()
+            else:
-        else:
+                data['Vars'] = recipient_merge_data
-            self._finish_recipients_with_vars()
+
+        if email.addr_spec in self.merge_metadata:
+            recipient_metadata = self.merge_metadata[email.addr_spec]
+            if self.metadata:
+                metadata = self.metadata.copy()  # clone toplevel metadata
+                metadata.update(recipient_metadata)
+            else:
+                metadata = recipient_metadata
+            data["Mj-EventPayLoad"] = self.serialize_json(metadata)
+
+        return data
 
     def _populate_sender_from_template(self):
         # If no From address was given, use the address from the template.
@@ -137,42 +154,21 @@ class MailjetPayload(RequestsPayload):
                                               email_message=self.message, response=response, backend=self.backend)
             self.set_from_email(parsed)
 
-    def _finish_recipients_with_vars(self):
+    def _format_email_for_mailjet(self, email):
-        """Send bulk mail with different variables for each mail."""
+        """Return EmailAddress email converted to a string that Mailjet can parse properly"""
-        assert "Cc" not in self.data and "Bcc" not in self.data
+        # Workaround Mailjet 3.0 bug parsing display-name with commas
-        recipients = []
+        # (see test_comma_in_display_name in test_mailjet_backend for details)
-        merge_data = self.merge_data or {}
+        if "," in email.display_name:
-        for email in self.recipients["to"]:
+            return EmailAddress(email.display_name.encode('utf-8'), email.addr_spec).formataddr('utf-8')
-            recipient = {
+        else:
-                "Email": email.addr_spec,
+            return email.address
-                "Name": email.display_name,
-                "Vars": merge_data.get(email.addr_spec)
-            }
-            # Strip out empty Name and Vars
-            recipient = {k: v for k, v in recipient.items() if v}
-            recipients.append(recipient)
-        self.data["Recipients"] = recipients
 
-    def _finish_recipients_single(self):
+    #
-        """Send a single mail with some To, Cc and Bcc headers."""
+    # Payload construction
-        assert "Recipients" not in self.data
+    #
-        if self.merge_data:
-            # When Cc and Bcc headers are given, then merge data cannot be set.
-            raise NotImplementedError("Cannot set merge data with bcc/cc")
-        for recipient_type, emails in self.recipients.items():
-            # Workaround Mailjet 3.0 bug parsing display-name with commas
-            # (see test_comma_in_display_name in test_mailjet_backend for details)
-            formatted_emails = [
-                email.address if "," not in email.display_name
-                # else name has a comma, so force it into MIME encoded-word utf-8 syntax:
-                else EmailAddress(email.display_name.encode('utf-8'), email.addr_spec).formataddr('utf-8')
-                for email in emails
-            ]
-            self.data[recipient_type.capitalize()] = ", ".join(formatted_emails)
 
     def init_payload(self):
-        self.data = {
+        self.data = {}
-        }
 
     def set_from_email(self, email):
         self.data["FromEmail"] = email.addr_spec
@@ -181,9 +177,10 @@ class MailjetPayload(RequestsPayload):
 
     def set_recipients(self, recipient_type, emails):
         assert recipient_type in ["to", "cc", "bcc"]
-        # Will be handled later in serialize_data
         if emails:
-            self.recipients[recipient_type] = emails
+            self.recipients[recipient_type] = emails  # save for recipient_status processing
+            self.data[recipient_type.capitalize()] = ", ".join(
+                [self._format_email_for_mailjet(email) for email in emails])
 
     def set_subject(self, subject):
         self.data["Subject"] = subject
@@ -225,8 +222,8 @@ class MailjetPayload(RequestsPayload):
         self.data["Sender"] = email.addr_spec  # ??? v3 docs unclear
 
     def set_metadata(self, metadata):
-        # Mailjet expects a single string payload
         self.data["Mj-EventPayLoad"] = self.serialize_json(metadata)
+        self.metadata = metadata  # keep original in case we need to merge with merge_metadata
 
     def set_tags(self, tags):
         # The choices here are CustomID or Campaign, and Campaign seems closer
@@ -257,5 +254,9 @@ class MailjetPayload(RequestsPayload):
     def set_merge_global_data(self, merge_global_data):
         self.data["Vars"] = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # Will be handled later in serialize_data
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.data.update(extra)

anymail/backends/mandrill.py

@@ -79,6 +79,9 @@ class MandrillPayload(RequestsPayload):
 
     def serialize_data(self):
         self.process_esp_extra()
+        if self.is_batch():
+            # hide recipients from each other
+            self.data['message']['preserve_recipients'] = False
         return self.serialize_json(self.data)
 
     #
@@ -163,7 +166,6 @@ class MandrillPayload(RequestsPayload):
         self.data.setdefault("template_content", [])  # Mandrill requires something here
 
     def set_merge_data(self, merge_data):
-        self.data['message']['preserve_recipients'] = False  # if merge, hide recipients from each other
         self.data['message']['merge_vars'] = [
             {'rcpt': rcpt, 'vars': [{'name': key, 'content': rcpt_data[key]}
                                     for key in sorted(rcpt_data.keys())]}  # sort for testing reproducibility
@@ -176,6 +178,13 @@ class MandrillPayload(RequestsPayload):
             for var, value in merge_global_data.items()
         ]
 
+    def set_merge_metadata(self, merge_metadata):
+        # recipient_metadata format is similar to, but not quite the same as, merge_vars:
+        self.data['message']['recipient_metadata'] = [
+            {'rcpt': rcpt, 'values': rcpt_data}
+            for rcpt, rcpt_data in merge_metadata.items()
+        ]
+
     def set_esp_extra(self, extra):
         # late bind in serialize_data, so that obsolete Djrill attrs can contribute
         self.esp_extra = extra

anymail/backends/postmark.py

@@ -156,10 +156,11 @@ class PostmarkPayload(RequestsPayload):
         self.to_emails = []
         self.cc_and_bcc_emails = []  # need to track (separately) for parse_recipient_status
         self.merge_data = None
+        self.merge_metadata = None
         super(PostmarkPayload, self).__init__(message, defaults, backend, headers=headers, *args, **kwargs)
 
     def get_api_endpoint(self):
-        batch_send = self.merge_data is not None and len(self.to_emails) > 1
+        batch_send = self.is_batch() and len(self.to_emails) > 1
         if 'TemplateAlias' in self.data or 'TemplateId' in self.data or 'TemplateModel' in self.data:
             if batch_send:
                 return "email/batchWithTemplates"
@@ -197,6 +198,14 @@ class PostmarkPayload(RequestsPayload):
                 data["TemplateModel"].update(recipient_data)
             else:
                 data["TemplateModel"] = recipient_data
+        if self.merge_metadata and to.addr_spec in self.merge_metadata:
+            recipient_metadata = self.merge_metadata[to.addr_spec]
+            if "Metadata" in data:
+                # merge recipient_metadata into toplevel metadata
+                data["Metadata"] = data["Metadata"].copy()
+                data["Metadata"].update(recipient_metadata)
+            else:
+                data["Metadata"] = recipient_metadata
         return data
 
     #
@@ -298,6 +307,10 @@ class PostmarkPayload(RequestsPayload):
     def set_merge_global_data(self, merge_global_data):
         self.data["TemplateModel"] = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # late-bind
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Special handling for 'server_token':

anymail/backends/sparkpost.py

@@ -95,11 +95,12 @@ class SparkPostPayload(BasePayload):
         self.all_recipients = []
         self.to_emails = []
         self.merge_data = {}
+        self.merge_metadata = {}
 
     def get_api_params(self):
         # Compose recipients param from to_emails and merge_data (if any)
         recipients = []
-        if len(self.merge_data) > 0:
+        if self.is_batch():
             # Build JSON recipient structures
             for email in self.to_emails:
                 rcpt = {'address': {'email': email.addr_spec}}
@@ -109,6 +110,10 @@ class SparkPostPayload(BasePayload):
                     rcpt['substitution_data'] = self.merge_data[email.addr_spec]
                 except KeyError:
                     pass  # no merge_data or none for this recipient
+                try:
+                    rcpt['metadata'] = self.merge_metadata[email.addr_spec]
+                except KeyError:
+                    pass  # no merge_metadata or none for this recipient
                 recipients.append(rcpt)
         else:
             # Just use simple recipients list
@@ -213,6 +218,9 @@ class SparkPostPayload(BasePayload):
     def set_merge_data(self, merge_data):
         self.merge_data = merge_data  # merged into params['recipients'] in get_api_params
 
+    def set_merge_metadata(self, merge_metadata):
+        self.merge_metadata = merge_metadata  # merged into params['recipients'] in get_api_params
+
     def set_merge_global_data(self, merge_global_data):
         self.params['substitution_data'] = merge_global_data
 

anymail/backends/test.py

@@ -22,6 +22,9 @@ class EmailBackend(AnymailBaseBackend):
     esp_name = "Test"
 
     def __init__(self, *args, **kwargs):
+        # Allow replacing the payload, for testing.
+        # (Real backends would generally not implement this option.)
+        self._payload_class = kwargs.pop('payload_class', TestPayload)
         super(EmailBackend, self).__init__(*args, **kwargs)
         if not hasattr(mail, 'outbox'):
             mail.outbox = []  # see django.core.mail.backends.locmem
@@ -32,7 +35,7 @@ class EmailBackend(AnymailBaseBackend):
         return mail.outbox.index(message)
 
     def build_message_payload(self, message, defaults):
-        return TestPayload(backend=self, message=message, defaults=defaults)
+        return self._payload_class(backend=self, message=message, defaults=defaults)
 
     def post_to_esp(self, payload, message):
         # Keep track of the sent messages and params (for test cases)
@@ -130,6 +133,9 @@ class TestPayload(BasePayload):
     def set_merge_data(self, merge_data):
         self.params['merge_data'] = merge_data
 
+    def set_merge_metadata(self, merge_metadata):
+        self.params['merge_metadata'] = merge_metadata
+
     def set_merge_global_data(self, merge_global_data):
         self.params['merge_global_data'] = merge_global_data
 

anymail/message.py

@@ -29,6 +29,7 @@ class AnymailMessageMixin(object):
         self.template_id = kwargs.pop('template_id', UNSET)
         self.merge_data = kwargs.pop('merge_data', UNSET)
         self.merge_global_data = kwargs.pop('merge_global_data', UNSET)
+        self.merge_metadata = kwargs.pop('merge_metadata', UNSET)
         self.anymail_status = AnymailStatus()
 
         # noinspection PyArgumentList

docs/esps/amazon_ses.rst

@@ -68,6 +68,11 @@ Limitations and quirks
   :attr:`~anymail.message.AnymailMessage.tags` feature. See :ref:`amazon-ses-tags`
   below for more information and additional options.
 
+**No merge_metadata**
+  Amazon SES's batch sending API does not support the custom headers Anymail uses
+  for metadata, so Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata`
+  feature is not available. (See :ref:`amazon-ses-tags` below for more information.)
+
 **Open and click tracking overrides**
   Anymail's :attr:`~anymail.message.AnymailMessage.track_opens` and
   :attr:`~anymail.message.AnymailMessage.track_clicks` are not supported.
@@ -164,7 +169,8 @@ For more complex use cases, set the SES `Tags` parameter directly in Anymail's
 :ref:`esp_extra <amazon-ses-esp-extra>`. See the example below. (Because custom headers do not
 work with SES's SendBulkTemplatedEmail call, esp_extra Tags is the only way to attach
 data to SES messages also using Anymail's :attr:`~anymail.message.AnymailMessage.template_id`
-and :attr:`~anymail.message.AnymailMessage.merge_data` features.)
+and :attr:`~anymail.message.AnymailMessage.merge_data` features, and the
+:attr:`~anymail.message.AnymailMessage.merge_metadata` cannot be supported.)
 
 
 .. _Introducing Sending Metrics:

docs/esps/index.rst

@@ -38,6 +38,7 @@ Email Service Provider                        |Amazon SES|  |Mailgun|    |Mailje
 ---------------------------------------------------------------------------------------------------------------------------------------------------
 :attr:`~AnymailMessage.envelope_sender`       Yes           Domain only  Yes         Domain only  No          No          No            Yes
 :attr:`~AnymailMessage.metadata`              Yes           Yes          Yes         Yes          Yes         Yes         Yes           Yes
+:attr:`~AnymailMessage.merge_metadata`        No            Yes          Yes         Yes          Yes         Yes         No            Yes
 :attr:`~AnymailMessage.send_at`               No            Yes          No          Yes          No          Yes         No            Yes
 :attr:`~AnymailMessage.tags`                  Yes           Yes          Max 1 tag   Yes          Max 1 tag   Yes         Max 1 tag     Max 1 tag
 :attr:`~AnymailMessage.track_clicks`          No            Yes          Yes         Yes          Yes         Yes         No            Yes

docs/esps/mailgun.rst

@@ -179,6 +179,29 @@ Limitations and quirks
   obvious reasons, only the domain portion applies. You can use anything before
   the @, and it will be ignored.
 
+**Using merge_metadata with merge_data**
+  If you use both Anymail's :attr:`~anymail.message.AnymailMessage.merge_data`
+  and :attr:`~anymail.message.AnymailMessage.merge_metadata` features, make sure your
+  merge_data keys do not start with ``v:``. (It's a good idea anyway to avoid colons
+  and other special characters in merge_data keys, as this isn't generally portable
+  to other ESPs.)
+
+  The same underlying Mailgun feature ("recipient-variables") is used to implement
+  both Anymail features. To avoid conflicts, Anymail prepends ``v:`` to recipient
+  variables needed for merge_metadata. (This prefix is stripped as Mailgun prepares
+  the message to send, so it won't be present in your Mailgun API logs or the metadata
+  that is sent to tracking webhooks.)
+
+**merge_metadata values default to empty string**
+  If you use Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata` feature,
+  and you supply metadata keys for some recipients but not others, Anymail will first
+  try to resolve the missing keys in :attr:`~anymail.message.AnymailMessage.metadata`,
+  and if they are not found there will default them to an empty string value.
+
+  Your tracking webhooks will receive metadata values (either that you provided or the
+  default empty string) for *every* key used with *any* recipient in the send.
+
+
 .. _undocumented API requirement:
     https://mailgun.uservoice.com/forums/156243-feature-requests/suggestions/35668606
 

docs/esps/mailjet.rst

@@ -11,9 +11,11 @@ Anymail integrates with the `Mailjet`_ email service, using their transactional
 
 .. note::
 
-    Mailjet is developing an improved `v3.1 Send API`_ (in public beta as of mid-2017).
+    Mailjet has released a newer `v3.1 Send API`_, but due to mismatches between its
-    Once the v3.1 API is released, Anymail will switch to it. This change should be
+    documentation and actual behavior, Anymail has been unable to switch to it.
-    largely transparent to your code, unless you are using Anymail's
+    Anymail's maintainers have reported the problems to Mailjet, and if and when they
+    are resolved, Anymail will switch to the v3.1 API. This change should be largely
+    transparent to your code, unless you are using Anymail's
     :ref:`esp_extra <mailjet-esp-extra>` feature to set API-specific options.
 
 
@@ -132,26 +134,26 @@ Limitations and quirks
   special approval from Mailjet support to use custom senders.
 
 **Commas in recipient names**
-  Mailjet's v3 API does not properly handle commas in recipient display-names
+  Mailjet's v3 API does not properly handle commas in recipient display-names.
-  *if* your message also uses the ``cc`` or ``bcc`` fields.
   (Tested July, 2017, and confirmed with Mailjet API support.)
 
   If your message would be affected, Anymail attempts to work around
   the problem by switching to `MIME encoded-word`_ syntax where needed.
 
   Most modern email clients should support this syntax, but if you run
-  into issues either avoid using ``cc`` and ``bcc``, or strip commas from all
+  into issues, you might want to strip commas from all
   recipient names (in ``to``, ``cc``, *and* ``bcc``) before sending.
 
+  (This should be resolved in a future release when
+  Anymail :ref:`switches <mailjet-v31-api>` to Mailjet's upcoming v3.1 API.)
+
 .. _MIME encoded-word: https://en.wikipedia.org/wiki/MIME#Encoded-Word
 
-**Merge data not compatible with cc/bcc**
+.. versionchanged:: 6.0
-  Mailjet's v3 API is not capable of representing both ``cc`` or ``bcc`` fields
-  and :attr:`~anymail.message.AnymailMessage.merge_data` in the same message.
-  If you attempt to combine them, Anymail will raise an error at send time.
 
-(The latter two limitations should be resolved in a future release when
+  Earlier versions of Anymail were unable to mix ``cc`` or ``bcc`` fields
-Anymail :ref:`switches <mailjet-v31-api>` to Mailjet's upcoming v3.1 API.)
+  and :attr:`~anymail.message.AnymailMessage.merge_data` in the same Mailjet message.
+  This limitation was removed in Anymail 6.0.
 
 
 .. _mailjet-templates:

docs/esps/sendinblue.rst

@@ -178,7 +178,8 @@ SendinBlue supports :ref:`ESP stored templates <esp-stored-templates>`
 populated with global merge data for all recipients, but does not
 offer :ref:`batch sending <batch-send>` with per-recipient merge data.
 Anymail's :attr:`~anymail.message.AnymailMessage.merge_data`
-message attribute is not supported with the SendinBlue backend.
+and :attr:`~anymail.message.AnymailMessage.merge_metadata`
+message attributes are not supported with the SendinBlue backend.
 
 To use a SendinBlue template, set the message's
 :attr:`~anymail.message.AnymailMessage.template_id` to the numeric

docs/sending/anymail_additions.rst

@@ -115,6 +115,31 @@ ESP send options (AnymailMessage)
         as metadata. See :ref:`formatting-merge-data`.
 
 
+    .. attribute:: merge_metadata
+
+        Set this to a `dict` of *per-recipient* metadata values the ESP should store
+        with the message, for later search and retrieval. Each key in the dict is a
+        recipient email (address portion only), and its value is a dict of metadata
+        for that recipient:
+
+        .. code-block:: python
+
+            message.to = ["wile@example.com", "Mr. Runner <rr@example.com>"]
+            message.merge_metadata = {
+                "wile@example.com": {"customer": 123, "order": "acme-zxyw"},
+                "rr@example.com": {"customer": 45678, "order": "acme-wblt"},
+            }
+
+        When :attr:`!merge_metadata` is set, Anymail will use the ESP's
+        :ref:`batch sending <batch-send>` option, so that each `to` recipient gets an
+        individual message (and doesn't see the other emails on the `to` list).
+
+        All of the notes on :attr:`metadata` keys and value formatting also apply
+        to :attr:`!merge_metadata`. If there are conflicting keys, the
+        :attr:`!merge_metadata` values will take precedence over :attr:`!metadata`
+        for that recipient.
+
+
     .. attribute:: tags
 
         Set this to a `list` of `str` tags to apply to the message (usually
@@ -131,7 +156,8 @@ ESP send options (AnymailMessage)
 
     .. caution::
 
-        Some ESPs put :attr:`metadata` and :attr:`tags` in email headers,
+        Some ESPs put :attr:`metadata` (and a recipient's :attr:`merge_metadata`)
+        and :attr:`tags` in email headers,
         which are included with the email when it is delivered. Anything you
         put in them **could be exposed to the recipients,** so don't
         include sensitive data.

docs/sending/templates.rst

@@ -125,7 +125,8 @@ To use batch sending with Anymail (for ESPs that support it):
 
 .. caution::
 
-    It's critical to set the :attr:`~AnymailMessage.merge_data` attribute:
+    It's critical to set the :attr:`~AnymailMessage.merge_data`
+    (or :attr:`~AnymailMessage.merge_metadata`) attribute:
     this is how Anymail recognizes the message as a batch send.
 
     When you provide merge_data, Anymail will tell the ESP to send an individual customized

tests/test_general_backend.py

@@ -11,7 +11,7 @@ from django.utils.functional import Promise
 from django.utils.timezone import utc
 from django.utils.translation import ugettext_lazy
 
-from anymail.backends.test import EmailBackend as TestBackend
+from anymail.backends.test import EmailBackend as TestBackend, TestPayload
 from anymail.exceptions import AnymailConfigurationError, AnymailInvalidAddress, AnymailUnsupportedFeature
 from anymail.message import AnymailMessage
 from anymail.utils import get_anymail_setting
@@ -425,3 +425,45 @@ class SpecialHeaderTests(TestBackendTestCase):
         self.message.extra_headers = {"To": "Apparent Recipient <but-not-really@example.com>"}
         with self.assertRaisesMessage(AnymailUnsupportedFeature, "spoofing `To` header"):
             self.message.send()
+
+
+class BatchSendDetectionTestCase(TestBackendTestCase):
+    """Tests shared code to consistently determine whether to use batch send"""
+
+    def setUp(self):
+        super(BatchSendDetectionTestCase, self).setUp()
+        self.backend = TestBackend()
+
+    def test_default_is_not_batch(self):
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertFalse(payload.is_batch())
+
+    def test_merge_data_implies_batch(self):
+        self.message.merge_data = {}  # *anything* (even empty dict) implies batch
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertTrue(payload.is_batch())
+
+    def test_merge_metadata_implies_batch(self):
+        self.message.merge_metadata = {}  # *anything* (even empty dict) implies batch
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertTrue(payload.is_batch())
+
+    def test_merge_global_data_does_not_imply_batch(self):
+        self.message.merge_global_data = {}
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertFalse(payload.is_batch())
+
+    def test_cannot_call_is_batch_during_init(self):
+        # It's tempting to try to warn about unsupported batch features in setters,
+        # but because of the way payload attrs are processed, it won't work...
+        class ImproperlyImplementedPayload(TestPayload):
+            def set_cc(self, emails):
+                if self.is_batch():  # this won't work here!
+                    self.unsupported_feature("cc with batch send")
+                super(ImproperlyImplementedPayload, self).set_cc(emails)
+
+        connection = mail.get_connection('anymail.backends.test.EmailBackend',
+                                         payload_class=ImproperlyImplementedPayload)
+        with self.assertRaisesMessage(AssertionError,
+                                      "Cannot call is_batch before all attributes processed"):
+            connection.send_messages([self.message])

tests/test_mailgun_backend.py

@@ -98,6 +98,7 @@ class MailgunBackendStandardEmailTests(MailgunBackendMockAPITestCase):
         self.assertEqual(data['h:Reply-To'], "another@example.com")
         self.assertEqual(data['h:X-MyHeader'], 'my value')
         self.assertEqual(data['h:Message-ID'], 'mycustommsgid@example.com')
+        self.assertNotIn('recipient-variables', data)  # multiple recipients, but not a batch send
 
     def test_html_message(self):
         text_content = 'This is an important message.'
@@ -387,6 +388,7 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
         data = self.get_api_call_data()
         self.assertEqual(data['v:user_id'], '12345')
         self.assertEqual(data['v:items'], '["mail","gun"]')
+        self.assertNotIn('recipient-variables', data)  # shouldn't be needed for non-batch
 
     def test_send_at(self):
         utc_plus_6 = get_fixed_timezone(6 * 60)
@@ -484,6 +486,56 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
             'bob@example.com': {'test': "value"},
         })
 
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'tier': 'basic', 'notification_batch': 'zx912'}
+        self.message.send()
+
+        data = self.get_api_call_data()
+        # custom-data variables for merge_metadata refer to recipient-variables:
+        self.assertEqual(data['v:order_id'], '%recipient.v:order_id%')
+        self.assertEqual(data['v:tier'], '%recipient.v:tier%')
+        self.assertEqual(data['v:notification_batch'], 'zx912')  # metadata constant doesn't need var
+        # recipient-variables populates them:
+        self.assertJSONEqual(data['recipient-variables'], {
+            'alice@example.com': {'v:order_id': 123, 'v:tier': 'premium'},
+            'bob@example.com': {'v:order_id': 678, 'v:tier': 'basic'},  # tier merged from metadata default
+        })
+
+    def test_merge_data_with_merge_metadata(self):
+        # merge_data and merge_metadata both use recipient-variables
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.body = "Hi %recipient.name%. Welcome to %recipient.group% at %recipient.site%."
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},  # and leave group undefined
+        }
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},  # and leave tier undefined
+        }
+        self.message.send()
+
+        data = self.get_api_call_data()
+        self.assertJSONEqual(data['recipient-variables'], {
+            'alice@example.com': {'name': "Alice", 'group': "Developers",
+                                  'v:order_id': 123, 'v:tier': 'premium'},
+            'bob@example.com': {'name': "Bob",  # undefined merge_data --> omitted
+                                'v:order_id': 678, 'v:tier': ''},  # undefined metadata --> empty string
+        })
+
+    def test_force_batch(self):
+        # Mailgun uses presence of recipient-variables to indicate batch send
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_data = {}
+        self.message.send()
+        data = self.get_api_call_data()
+        self.assertJSONEqual(data['recipient-variables'], {})
+
     def test_sender_domain(self):
         """Mailgun send domain can come from from_email, envelope_sender, or esp_extra"""
         # You could also use MAILGUN_SENDER_DOMAIN in your ANYMAIL settings, as in the next test.

tests/test_mailjet_backend.py

@@ -77,7 +77,7 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
         self.assertEqual(data['Subject'], "Subject here")
         self.assertEqual(data['Text-part'], "Here is the message.")
         self.assertEqual(data['FromEmail'], "from@sender.example.com")
-        self.assertEqual(data['Recipients'], [{"Email": "to@example.com"}])
+        self.assertEqual(data['To'], "to@example.com")
 
     def test_name_addr(self):
         """Make sure RFC2822 name-addr format (with display-name) is allowed
@@ -99,7 +99,11 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
         self.assertEqual(data['Bcc'], 'Blind Copy <bcc1@example.com>, bcc2@example.com')
 
     def test_comma_in_display_name(self):
-        # note there are two paths: with cc/bcc, and without
+        # Mailjet 3.0 API doesn't properly parse RFC-2822 quoted display-names from To/Cc/Bcc:
+        # `To: "Recipient, Ltd." <to@example.com>` tries to send messages to `"Recipient`
+        # and to `Ltd.` (neither of which are actual email addresses).
+        # As a workaround, force MIME "encoded-word" utf-8 encoding, which gets past Mailjet's broken parsing.
+        # (This shouldn't be necessary in Mailjet 3.1, where Name becomes a separate json field for Cc/Bcc.)
         msg = mail.EmailMessage(
             'Subject', 'Message', '"Example, Inc." <from@example.com>',
             ['"Recipient, Ltd." <to@example.com>'])
@@ -107,17 +111,6 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
         data = self.get_api_call_json()
         self.assertEqual(data['FromName'], 'Example, Inc.')
         self.assertEqual(data['FromEmail'], 'from@example.com')
-        self.assertEqual(data['Recipients'][0]["Email"], "to@example.com")
-        self.assertEqual(data['Recipients'][0]["Name"], "Recipient, Ltd.")  # separate Name field works fine
-
-        # Mailjet 3.0 API doesn't properly parse RFC-2822 quoted display-names from To/Cc/Bcc:
-        # `To: "Recipient, Ltd." <to@example.com>` tries to send messages to `"Recipient`
-        # and to `Ltd.` (neither of which are actual email addresses).
-        # As a workaround, force MIME "encoded-word" utf-8 encoding, which gets past Mailjet's broken parsing.
-        # (This shouldn't be necessary in Mailjet 3.1, where Name becomes a separate json field for Cc/Bcc.)
-        msg.cc = ['cc@example.com']
-        msg.send()
-        data = self.get_api_call_json()
         # self.assertEqual(data['To'], '"Recipient, Ltd." <to@example.com>')  # this doesn't work
         self.assertEqual(data['To'], '=?utf-8?q?Recipient=2C_Ltd=2E?= <to@example.com>')  # workaround
 
@@ -492,19 +485,50 @@ class MailjetBackendAnymailFeatureTests(MailjetBackendMockAPITestCase):
             self.message.send()
 
     def test_merge_data(self):
-        self.message.to = ['alice@example.com']
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.cc = ['cc@example.com']
         self.message.template_id = '1234567'
         self.message.merge_data = {
             'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},
         }
+        self.message.merge_global_data = {'group': "Users", 'site': "ExampleCo"}
         self.message.send()
+
         data = self.get_api_call_json()
-        self.assertEqual(data['Mj-TemplateID'], '1234567')
+        messages = data['Messages']
-        self.assertNotIn('Vars', data)
+        self.assertEqual(len(messages), 2)
-        self.assertEqual(data['Recipients'], [{
+        self.assertEqual(messages[0]['To'], 'alice@example.com')
-            'Email': 'alice@example.com',
+        self.assertEqual(messages[0]['Cc'], 'cc@example.com')
-            'Vars': {'name': "Alice", 'group': "Developers"}
+        self.assertEqual(messages[0]['Mj-TemplateID'], '1234567')
-        }])
+        self.assertEqual(messages[0]['Vars'],
+                         {'name': "Alice", 'group': "Developers", 'site': "ExampleCo"})
+
+        self.assertEqual(messages[1]['To'], 'Bob <bob@example.com>')
+        self.assertEqual(messages[1]['Cc'], 'cc@example.com')
+        self.assertEqual(messages[1]['Mj-TemplateID'], '1234567')
+        self.assertEqual(messages[1]['Vars'],
+                         {'name': "Bob", 'group': "Users", 'site': "ExampleCo"})
+
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+
+        data = self.get_api_call_json()
+        messages = data['Messages']
+        self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0]['To'], 'alice@example.com')
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertJSONEqual(messages[0]['Mj-EventPayLoad'],
+                             {'order_id': 123, 'tier': 'premium', 'notification_batch': 'zx912'})
+        self.assertEqual(messages[1]['To'], 'Bob <bob@example.com>')
+        self.assertJSONEqual(messages[1]['Mj-EventPayLoad'],
+                             {'order_id': 678, 'notification_batch': 'zx912'})
 
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.

tests/test_mandrill_backend.py

@@ -379,7 +379,25 @@ class MandrillBackendAnymailFeatureTests(MandrillBackendMockAPITestCase):
             {'name': "group", 'content': "Users"},
             {'name': "site", 'content': "ExampleCo"},
         ])
-        self.assertEqual(data['message']['preserve_recipients'], False)  # we force with merge_data
+        self.assertIs(data['message']['preserve_recipients'], False)  # merge_data implies batch
+
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertCountEqual(data['message']['recipient_metadata'], [{
+            'rcpt': 'alice@example.com',
+            'values': {'order_id': 123, 'tier': 'premium'},
+        }, {
+            'rcpt': 'bob@example.com',
+            'values': {'order_id': 678},
+        }])
+        self.assertIs(data['message']['preserve_recipients'], False)  # merge_metadata implies batch
 
     def test_missing_from(self):
         """Make sure a missing from_email omits from* from API call.

tests/test_postmark_backend.py

@@ -398,8 +398,7 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         data = self.get_api_call_json()
         self.assertEqual(data['TemplateAlias'], 'welcome-message')
 
-    def test_merge_data(self):
+    _mock_batch_response = json.dumps([{
-        self.set_mock_response(raw=json.dumps([{
             "ErrorCode": 0,
             "Message": "OK",
             "To": "alice@example.com",
@@ -411,8 +410,10 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
             "To": "bob@example.com",
             "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
             "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d",
-        }]).encode('utf-8'))
+        }]).encode('utf-8')
 
+    def test_merge_data(self):
+        self.set_mock_response(raw=self._mock_batch_response)
         message = AnymailMessage(
             from_email='from@example.com',
             template_id=1234567,  # Postmark only supports merge_data content in a template
@@ -451,20 +452,7 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
 
     def test_merge_data_no_template(self):
         # merge_data={} can be used to force batch sending without a template
-        self.set_mock_response(raw=json.dumps([{
+        self.set_mock_response(raw=self._mock_batch_response)
-            "ErrorCode": 0,
-            "Message": "OK",
-            "To": "alice@example.com",
-            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
-            "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817",
-        }, {
-            "ErrorCode": 0,
-            "Message": "OK",
-            "To": "bob@example.com",
-            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
-            "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d",
-        }]).encode('utf-8'))
-
         message = AnymailMessage(
             from_email='from@example.com',
             to=['alice@example.com', 'Bob <bob@example.com>'],
@@ -496,6 +484,45 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         self.assertEqual(recipients['bob@example.com'].status, 'sent')
         self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d')
 
+    def test_merge_metadata(self):
+        self.set_mock_response(raw=self._mock_batch_response)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+
+        self.assert_esp_called('/email/batch')
+        data = self.get_api_call_json()
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0]["To"], "alice@example.com")
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertEqual(data[0]["Metadata"], {'order_id': 123, 'tier': 'premium', 'notification_batch': 'zx912'})
+        self.assertEqual(data[1]["To"], "Bob <bob@example.com>")
+        self.assertEqual(data[1]["Metadata"], {'order_id': 678, 'notification_batch': 'zx912'})
+
+    def test_merge_metadata_with_template(self):
+        self.set_mock_response(raw=self._mock_batch_response)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.template_id = 1234567
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+        }
+        self.message.send()
+
+        self.assert_esp_called('/email/batchWithTemplates')
+        data = self.get_api_call_json()
+        messages = data["Messages"]
+        self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0]["To"], "alice@example.com")
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertEqual(messages[0]["Metadata"], {'order_id': 123})
+        self.assertEqual(messages[1]["To"], "Bob <bob@example.com>")
+        self.assertEqual(messages[1]["Metadata"], {'order_id': 678, 'tier': 'premium'})
+
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
 

tests/test_sparkpost_backend.py

@@ -443,6 +443,24 @@ class SparkPostBackendAnymailFeatureTests(SparkPostBackendMockAPITestCase):
         ])
         self.assertEqual(params['substitution_data'], {'group': "Users", 'site': "ExampleCo"})
 
+    def test_merge_metadata(self):
+        self.set_mock_response(accepted=2)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+        params = self.get_send_params()
+        self.assertEqual(params['recipients'], [
+            {'address': {'email': 'alice@example.com'},
+             'metadata': {'order_id': 123}},
+            {'address': {'email': 'bob@example.com', 'name': 'Bob'},
+             'metadata': {'order_id': 678, 'tier': 'premium'}}
+        ])
+        self.assertEqual(params['metadata'], {'notification_batch': 'zx912'})
+
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.