Add ESP templates, batch send and merge

* message.template_id to use ESP stored templates
* message.merge_data and merge_global_data
  to supply per-recipient/global merge variables
  (with or without an ESP stored template)
* When using per-recipient merge_data, tell ESP to use
  batch send: individual message per "to" address.
  (Mailgun does this automatically; SendGrid requires
  using a different "to" field; Mandrill requires
  `preserve_recipients=False`; Postmark doesn't
  support *this type* of batch sending with merge data.)
* Allow message.from_email=None (must be set after
  init) and message.subject=None to suppress those
  fields in API calls (for ESPs that allow "From" and
  "Subject" in their template definitions).

Mailgun:
* Emulate merge_global_data by copying to
  recipient-variables for each recipient.

SendGrid:
* Add delimiters to merge field names via
  esp_extra['merge_field_format'] or
  ANYMAIL_SENDGRID_MERGE_FIELD_FORMAT setting.

Mandrill:
* Remove Djrill versions of these features;
  update migration notes.

Closes #5.

anymail/backends/base.py

@@ -197,6 +197,9 @@ class BasePayload(object):
         ('tags', combine, None),
         ('track_clicks', last, None),
         ('track_opens', last, None),
+        ('template_id', last, None),
+        ('merge_data', combine, None),
+        ('merge_global_data', combine, None),
         ('esp_extra', combine, None),
     )
     esp_message_attrs = ()  # subclasses can override
@@ -356,6 +359,15 @@ class BasePayload(object):
     def set_track_opens(self, track_opens):
         self.unsupported_feature("track_opens")
 
+    def set_template_id(self, template_id):
+        self.unsupported_feature("template_id")
+
+    def set_merge_data(self, merge_data):
+        self.unsupported_feature("merge_data")
+
+    def set_merge_global_data(self, merge_global_data):
+        self.unsupported_feature("merge_global_data")
+
     # ESP-specific payload construction
     def set_esp_extra(self, extra):
         self.unsupported_feature("esp_extra")

anymail/backends/mailgun.py

@@ -56,6 +56,12 @@ class MailgunPayload(RequestsPayload):
         auth = ("api", backend.api_key)
         self.sender_domain = None
         self.all_recipients = []  # used for backend.parse_recipient_status
+
+        # late-binding of recipient-variables:
+        self.merge_data = None
+        self.merge_global_data = None
+        self.to_emails = []
+
         super(MailgunPayload, self).__init__(message, defaults, backend, auth=auth, *args, **kwargs)
 
     def get_api_endpoint(self):
@@ -66,6 +72,34 @@ class MailgunPayload(RequestsPayload):
                                backend=self.backend, email_message=self.message, payload=self)
         return "%s/messages" % self.sender_domain
 
+    def serialize_data(self):
+        self.populate_recipient_variables()
+        return self.data
+
+    def populate_recipient_variables(self):
+        """Populate Mailgun recipient-variables header from merge data"""
+        merge_data = self.merge_data
+
+        if self.merge_global_data is not None:
+            # Mailgun doesn't support global variables.
+            # We emulate them by populating recipient-variables for all recipients.
+            if merge_data is not None:
+                merge_data = merge_data.copy()  # don't modify the original, which doesn't belong to us
+            else:
+                merge_data = {}
+            for email in self.to_emails:
+                try:
+                    recipient_data = merge_data[email]
+                except KeyError:
+                    merge_data[email] = self.merge_global_data
+                else:
+                    # Merge globals (recipient_data wins in conflict)
+                    merge_data[email] = self.merge_global_data.copy()
+                    merge_data[email].update(recipient_data)
+
+        if merge_data is not None:
+            self.data['recipient-variables'] = self.serialize_json(merge_data)
+
     #
     # Payload construction
     #
@@ -87,8 +121,10 @@ class MailgunPayload(RequestsPayload):
     def set_recipients(self, recipient_type, emails):
         assert recipient_type in ["to", "cc", "bcc"]
         if emails:
-            self.data[recipient_type] = [str(email) for email in emails]
+            self.data[recipient_type] = [email.address for email in emails]
             self.all_recipients += emails  # used for backend.parse_recipient_status
+        if recipient_type == 'to':
+            self.to_emails = [email.email for email in emails]  # used for populate_recipient_variables
 
     def set_subject(self, subject):
         self.data["subject"] = subject
@@ -145,6 +181,17 @@ class MailgunPayload(RequestsPayload):
     def set_track_opens(self, track_opens):
         self.data["o:tracking-opens"] = "yes" if track_opens else "no"
 
+    # template_id: Mailgun doesn't offer stored templates.
+    # (The message body and other fields *are* the template content.)
+
+    def set_merge_data(self, merge_data):
+        # Processed at serialization time (to allow merging global data)
+        self.merge_data = merge_data
+
+    def set_merge_global_data(self, merge_global_data):
+        # Processed at serialization time (to allow merging global data)
+        self.merge_global_data = merge_global_data
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Allow override of sender_domain via esp_extra

anymail/backends/mandrill.py

@@ -147,6 +147,24 @@ class MandrillPayload(RequestsPayload):
     def set_track_opens(self, track_opens):
         self.data["message"]["track_opens"] = track_opens
 
+    def set_template_id(self, template_id):
+        self.data["template_name"] = template_id
+        self.data.setdefault("template_content", [])  # Mandrill requires something here
+
+    def set_merge_data(self, merge_data):
+        self.data['message']['preserve_recipients'] = False  # if merge, hide recipients from each other
+        self.data['message']['merge_vars'] = [
+            {'rcpt': rcpt, 'vars': [{'name': key, 'content': rcpt_data[key]}
+                                    for key in sorted(rcpt_data.keys())]}  # sort for testing reproducibility
+            for rcpt, rcpt_data in merge_data.items()
+        ]
+
+    def set_merge_global_data(self, merge_global_data):
+        self.data['message']['global_merge_vars'] = [
+            {'name': var, 'content': value}
+            for var, value in merge_global_data.items()
+        ]
+
     def set_esp_extra(self, extra):
         pass
 
@@ -170,10 +188,7 @@ class MandrillPayload(RequestsPayload):
         ('subaccount', last, None),
         ('google_analytics_domains', last, None),
         ('google_analytics_campaign', last, None),
-        ('global_merge_vars', combine, _expand_merge_vars),
-        ('merge_vars', combine, None),
         ('recipient_metadata', combine, None),
-        ('template_name', last, None),
         ('template_content', combine, _expand_merge_vars),
     )
 
@@ -183,20 +198,9 @@ class MandrillPayload(RequestsPayload):
     def set_ip_pool(self, ip_pool):
         self.data["ip_pool"] = ip_pool
 
-    def set_template_name(self, template_name):
-        self.data["template_name"] = template_name
-        self.data.setdefault("template_content", [])  # Mandrill requires something here
-
     def set_template_content(self, template_content):
         self.data["template_content"] = template_content
 
-    def set_merge_vars(self, merge_vars):
-        # For testing reproducibility, we sort the recipients
-        self.data['message']['merge_vars'] = [
-            {'rcpt': rcpt, 'vars': _expand_merge_vars(merge_vars[rcpt])}
-            for rcpt in sorted(merge_vars.keys())
-        ]
-
     def set_recipient_metadata(self, recipient_metadata):
         # For testing reproducibility, we sort the recipients
         self.data['message']['recipient_metadata'] = [

anymail/backends/postmark.py

@@ -100,7 +100,11 @@ class PostmarkPayload(RequestsPayload):
         super(PostmarkPayload, self).__init__(message, defaults, backend, headers=headers, *args, **kwargs)
 
     def get_api_endpoint(self):
-        return "email"
+        if 'TemplateId' in self.data or 'TemplateModel' in self.data:
+            # This is the one Postmark API documented to have a trailing slash. (Typo?)
+            return "email/withTemplate/"
+        else:
+            return "email"
 
     def get_request_params(self, api_url):
         params = super(PostmarkPayload, self).get_request_params(api_url)
@@ -185,6 +189,14 @@ class PostmarkPayload(RequestsPayload):
     def set_track_opens(self, track_opens):
         self.data["TrackOpens"] = track_opens
 
+    def set_template_id(self, template_id):
+        self.data["TemplateId"] = template_id
+
+    # merge_data: Postmark doesn't support per-recipient substitutions
+
+    def set_merge_global_data(self, merge_global_data):
+        self.data["TemplateModel"] = merge_global_data
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Special handling for 'server_token':

anymail/backends/sendgrid.py

@@ -1,7 +1,9 @@
+import warnings
+
 from django.core.mail import make_msgid
 from requests.structures import CaseInsensitiveDict
 
-from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError
+from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError, AnymailWarning
 from ..message import AnymailRecipientStatus
 from ..utils import get_anymail_setting, timestamp
 
@@ -31,6 +33,8 @@ class SendGridBackend(AnymailRequestsBackend):
 
         self.generate_message_id = get_anymail_setting('generate_message_id', esp_name=esp_name,
                                                        kwargs=kwargs, default=True)
+        self.merge_field_format = get_anymail_setting('merge_field_format', esp_name=esp_name,
+                                                      kwargs=kwargs, default=None)
 
         # This is SendGrid's Web API v2 (because the Web API v3 doesn't support sending)
         api_url = get_anymail_setting('api_url', esp_name=esp_name, kwargs=kwargs,
@@ -65,6 +69,10 @@ class SendGridPayload(RequestsPayload):
         self.generate_message_id = backend.generate_message_id
         self.message_id = None  # Message-ID -- assigned in serialize_data unless provided in headers
         self.smtpapi = {}  # SendGrid x-smtpapi field
+        self.to_list = []  # late-bound 'to' field
+        self.merge_field_format = backend.merge_field_format
+        self.merge_data = None  # late-bound per-recipient data
+        self.merge_global_data = None
 
         http_headers = kwargs.pop('headers', {})
         query_params = kwargs.pop('params', {})
@@ -86,6 +94,15 @@ class SendGridPayload(RequestsPayload):
         if self.generate_message_id:
             self.ensure_message_id()
 
+        self.build_merge_data()
+        if self.merge_data is None:
+            # Standard 'to' and 'toname' headers
+            self.set_recipients('to', self.to_list)
+        else:
+            # Merge-friendly smtpapi 'to' field
+            self.smtpapi['to'] = [email.address for email in self.to_list]
+            self.all_recipients += self.to_list
+
         # Serialize x-smtpapi to json:
         if len(self.smtpapi) > 0:
             # If esp_extra was also used to set x-smtpapi, need to merge it
@@ -132,6 +149,41 @@ class SendGridPayload(RequestsPayload):
             domain = None
         return make_msgid(domain=domain)
 
+    def build_merge_data(self):
+        """Set smtpapi['sub'] and ['section']"""
+        if self.merge_data is not None:
+            # Convert from {to1: {a: A1, b: B1}, to2: {a: A2}}  (merge_data format)
+            # to {a: [A1, A2], b: [B1, ""]}  ({field: [data in to-list order], ...})
+            all_fields = set()
+            for recipient_data in self.merge_data.values():
+                all_fields = all_fields.union(recipient_data.keys())
+            recipients = [email.email for email in self.to_list]
+
+            if self.merge_field_format is None and all(field.isalnum() for field in all_fields):
+                warnings.warn(
+                    "Your SendGrid merge fields don't seem to have delimiters, "
+                    "which can cause unexpected results with Anymail's merge_data. "
+                    "Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
+                    AnymailWarning)
+
+            sub_field_fmt = self.merge_field_format or '{}'
+            sub_fields = {field: sub_field_fmt.format(field) for field in all_fields}
+
+            self.smtpapi['sub'] = {
+                # If field data is missing for recipient, use (formatted) field as the substitution.
+                # (This allows default to resolve from global "section" substitutions.)
+                sub_fields[field]: [self.merge_data.get(recipient, {}).get(field, sub_fields[field])
+                                    for recipient in recipients]
+                for field in all_fields
+            }
+
+        if self.merge_global_data is not None:
+            section_field_fmt = self.merge_field_format or '{}'
+            self.smtpapi['section'] = {
+                section_field_fmt.format(field): data
+                for field, data in self.merge_global_data.items()
+            }
+
     #
     # Payload construction
     #
@@ -146,6 +198,11 @@ class SendGridPayload(RequestsPayload):
         if email.name:
             self.data["fromname"] = email.name
 
+    def set_to(self, emails):
+        # late-bind in self.serialize_data, because whether it goes in smtpapi
+        # depends on whether there is merge_data
+        self.to_list = emails
+
     def set_recipients(self, recipient_type, emails):
         assert recipient_type in ["to", "cc", "bcc"]
         if emails:
@@ -229,5 +286,18 @@ class SendGridPayload(RequestsPayload):
         # (You could add it through esp_extra.)
         self.add_filter('opentrack', 'enable', int(track_opens))
 
+    def set_template_id(self, template_id):
+        self.add_filter('templates', 'enable', 1)
+        self.add_filter('templates', 'template_id', template_id)
+
+    def set_merge_data(self, merge_data):
+        # Becomes smtpapi['sub'] in build_merge_data, after we know recipients and merge_field_format.
+        self.merge_data = merge_data
+
+    def set_merge_global_data(self, merge_global_data):
+        # Becomes smtpapi['section'] in build_merge_data, after we know merge_field_format.
+        self.merge_global_data = merge_global_data
+
     def set_esp_extra(self, extra):
+        self.merge_field_format = extra.pop('merge_field_format', self.merge_field_format)
         self.data.update(extra)

anymail/message.py

@@ -25,6 +25,9 @@ class AnymailMessageMixin(object):
         self.tags = kwargs.pop('tags', UNSET)
         self.track_clicks = kwargs.pop('track_clicks', UNSET)
         self.track_opens = kwargs.pop('track_opens', UNSET)
+        self.template_id = kwargs.pop('template_id', UNSET)
+        self.merge_data = kwargs.pop('merge_data', UNSET)
+        self.merge_global_data = kwargs.pop('merge_global_data', UNSET)
         self.anymail_status = None
 
         # noinspection PyArgumentList