Add merge_metadata for other ESPs

Support merge_metadata in Mailgun, Mailjet, Mandrill, Postmark, 
SparkPost, and Test backends. (SendGrid covered in earlier PR.)

Also:
* Add `merge_metadata` to AnymailMessage, AnymailMessageMixin
* Add `is_batch()` logic to BasePayload, for consistent handling
* Docs

Note: Mailjet implementation switches *all* batch sending from their 
"Recipients" field to to the "Messages" array bulk sending option.
This allows an independent payload for each batch recipient.
In addition to supporting merge_metadata, this also removes the
prior limitation on mixing Cc/Bcc with merge_data.

Closes #141.

tests/test_general_backend.py

@@ -11,7 +11,7 @@ from django.utils.functional import Promise
 from django.utils.timezone import utc
 from django.utils.translation import ugettext_lazy
 
-from anymail.backends.test import EmailBackend as TestBackend
+from anymail.backends.test import EmailBackend as TestBackend, TestPayload
 from anymail.exceptions import AnymailConfigurationError, AnymailInvalidAddress, AnymailUnsupportedFeature
 from anymail.message import AnymailMessage
 from anymail.utils import get_anymail_setting
@@ -425,3 +425,45 @@ class SpecialHeaderTests(TestBackendTestCase):
         self.message.extra_headers = {"To": "Apparent Recipient <but-not-really@example.com>"}
         with self.assertRaisesMessage(AnymailUnsupportedFeature, "spoofing `To` header"):
             self.message.send()
+
+
+class BatchSendDetectionTestCase(TestBackendTestCase):
+    """Tests shared code to consistently determine whether to use batch send"""
+
+    def setUp(self):
+        super(BatchSendDetectionTestCase, self).setUp()
+        self.backend = TestBackend()
+
+    def test_default_is_not_batch(self):
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertFalse(payload.is_batch())
+
+    def test_merge_data_implies_batch(self):
+        self.message.merge_data = {}  # *anything* (even empty dict) implies batch
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertTrue(payload.is_batch())
+
+    def test_merge_metadata_implies_batch(self):
+        self.message.merge_metadata = {}  # *anything* (even empty dict) implies batch
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertTrue(payload.is_batch())
+
+    def test_merge_global_data_does_not_imply_batch(self):
+        self.message.merge_global_data = {}
+        payload = self.backend.build_message_payload(self.message, {})
+        self.assertFalse(payload.is_batch())
+
+    def test_cannot_call_is_batch_during_init(self):
+        # It's tempting to try to warn about unsupported batch features in setters,
+        # but because of the way payload attrs are processed, it won't work...
+        class ImproperlyImplementedPayload(TestPayload):
+            def set_cc(self, emails):
+                if self.is_batch():  # this won't work here!
+                    self.unsupported_feature("cc with batch send")
+                super(ImproperlyImplementedPayload, self).set_cc(emails)
+
+        connection = mail.get_connection('anymail.backends.test.EmailBackend',
+                                         payload_class=ImproperlyImplementedPayload)
+        with self.assertRaisesMessage(AssertionError,
+                                      "Cannot call is_batch before all attributes processed"):
+            connection.send_messages([self.message])

tests/test_mailgun_backend.py

@@ -98,6 +98,7 @@ class MailgunBackendStandardEmailTests(MailgunBackendMockAPITestCase):
         self.assertEqual(data['h:Reply-To'], "another@example.com")
         self.assertEqual(data['h:X-MyHeader'], 'my value')
         self.assertEqual(data['h:Message-ID'], 'mycustommsgid@example.com')
+        self.assertNotIn('recipient-variables', data)  # multiple recipients, but not a batch send
 
     def test_html_message(self):
         text_content = 'This is an important message.'
@@ -387,6 +388,7 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
         data = self.get_api_call_data()
         self.assertEqual(data['v:user_id'], '12345')
         self.assertEqual(data['v:items'], '["mail","gun"]')
+        self.assertNotIn('recipient-variables', data)  # shouldn't be needed for non-batch
 
     def test_send_at(self):
         utc_plus_6 = get_fixed_timezone(6 * 60)
@@ -484,6 +486,56 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
             'bob@example.com': {'test': "value"},
         })
 
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'tier': 'basic', 'notification_batch': 'zx912'}
+        self.message.send()
+
+        data = self.get_api_call_data()
+        # custom-data variables for merge_metadata refer to recipient-variables:
+        self.assertEqual(data['v:order_id'], '%recipient.v:order_id%')
+        self.assertEqual(data['v:tier'], '%recipient.v:tier%')
+        self.assertEqual(data['v:notification_batch'], 'zx912')  # metadata constant doesn't need var
+        # recipient-variables populates them:
+        self.assertJSONEqual(data['recipient-variables'], {
+            'alice@example.com': {'v:order_id': 123, 'v:tier': 'premium'},
+            'bob@example.com': {'v:order_id': 678, 'v:tier': 'basic'},  # tier merged from metadata default
+        })
+
+    def test_merge_data_with_merge_metadata(self):
+        # merge_data and merge_metadata both use recipient-variables
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.body = "Hi %recipient.name%. Welcome to %recipient.group% at %recipient.site%."
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},  # and leave group undefined
+        }
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},  # and leave tier undefined
+        }
+        self.message.send()
+
+        data = self.get_api_call_data()
+        self.assertJSONEqual(data['recipient-variables'], {
+            'alice@example.com': {'name': "Alice", 'group': "Developers",
+                                  'v:order_id': 123, 'v:tier': 'premium'},
+            'bob@example.com': {'name': "Bob",  # undefined merge_data --> omitted
+                                'v:order_id': 678, 'v:tier': ''},  # undefined metadata --> empty string
+        })
+
+    def test_force_batch(self):
+        # Mailgun uses presence of recipient-variables to indicate batch send
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_data = {}
+        self.message.send()
+        data = self.get_api_call_data()
+        self.assertJSONEqual(data['recipient-variables'], {})
+
     def test_sender_domain(self):
         """Mailgun send domain can come from from_email, envelope_sender, or esp_extra"""
         # You could also use MAILGUN_SENDER_DOMAIN in your ANYMAIL settings, as in the next test.

tests/test_mailjet_backend.py

@@ -77,7 +77,7 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
         self.assertEqual(data['Subject'], "Subject here")
         self.assertEqual(data['Text-part'], "Here is the message.")
         self.assertEqual(data['FromEmail'], "from@sender.example.com")
-        self.assertEqual(data['Recipients'], [{"Email": "to@example.com"}])
+        self.assertEqual(data['To'], "to@example.com")
 
     def test_name_addr(self):
         """Make sure RFC2822 name-addr format (with display-name) is allowed
@@ -99,7 +99,11 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
         self.assertEqual(data['Bcc'], 'Blind Copy <bcc1@example.com>, bcc2@example.com')
 
     def test_comma_in_display_name(self):
-        # note there are two paths: with cc/bcc, and without
+        # Mailjet 3.0 API doesn't properly parse RFC-2822 quoted display-names from To/Cc/Bcc:
+        # `To: "Recipient, Ltd." <to@example.com>` tries to send messages to `"Recipient`
+        # and to `Ltd.` (neither of which are actual email addresses).
+        # As a workaround, force MIME "encoded-word" utf-8 encoding, which gets past Mailjet's broken parsing.
+        # (This shouldn't be necessary in Mailjet 3.1, where Name becomes a separate json field for Cc/Bcc.)
         msg = mail.EmailMessage(
             'Subject', 'Message', '"Example, Inc." <from@example.com>',
             ['"Recipient, Ltd." <to@example.com>'])
@@ -107,17 +111,6 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
         data = self.get_api_call_json()
         self.assertEqual(data['FromName'], 'Example, Inc.')
         self.assertEqual(data['FromEmail'], 'from@example.com')
-        self.assertEqual(data['Recipients'][0]["Email"], "to@example.com")
-        self.assertEqual(data['Recipients'][0]["Name"], "Recipient, Ltd.")  # separate Name field works fine
-
-        # Mailjet 3.0 API doesn't properly parse RFC-2822 quoted display-names from To/Cc/Bcc:
-        # `To: "Recipient, Ltd." <to@example.com>` tries to send messages to `"Recipient`
-        # and to `Ltd.` (neither of which are actual email addresses).
-        # As a workaround, force MIME "encoded-word" utf-8 encoding, which gets past Mailjet's broken parsing.
-        # (This shouldn't be necessary in Mailjet 3.1, where Name becomes a separate json field for Cc/Bcc.)
-        msg.cc = ['cc@example.com']
-        msg.send()
-        data = self.get_api_call_json()
         # self.assertEqual(data['To'], '"Recipient, Ltd." <to@example.com>')  # this doesn't work
         self.assertEqual(data['To'], '=?utf-8?q?Recipient=2C_Ltd=2E?= <to@example.com>')  # workaround
 
@@ -492,19 +485,50 @@ class MailjetBackendAnymailFeatureTests(MailjetBackendMockAPITestCase):
             self.message.send()
 
     def test_merge_data(self):
-        self.message.to = ['alice@example.com']
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.cc = ['cc@example.com']
         self.message.template_id = '1234567'
         self.message.merge_data = {
             'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"},
         }
+        self.message.merge_global_data = {'group': "Users", 'site': "ExampleCo"}
         self.message.send()
+
         data = self.get_api_call_json()
-        self.assertEqual(data['Mj-TemplateID'], '1234567')
-        self.assertNotIn('Vars', data)
-        self.assertEqual(data['Recipients'], [{
-            'Email': 'alice@example.com',
-            'Vars': {'name': "Alice", 'group': "Developers"}
-        }])
+        messages = data['Messages']
+        self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0]['To'], 'alice@example.com')
+        self.assertEqual(messages[0]['Cc'], 'cc@example.com')
+        self.assertEqual(messages[0]['Mj-TemplateID'], '1234567')
+        self.assertEqual(messages[0]['Vars'],
+                         {'name': "Alice", 'group': "Developers", 'site': "ExampleCo"})
+
+        self.assertEqual(messages[1]['To'], 'Bob <bob@example.com>')
+        self.assertEqual(messages[1]['Cc'], 'cc@example.com')
+        self.assertEqual(messages[1]['Mj-TemplateID'], '1234567')
+        self.assertEqual(messages[1]['Vars'],
+                         {'name': "Bob", 'group': "Users", 'site': "ExampleCo"})
+
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+
+        data = self.get_api_call_json()
+        messages = data['Messages']
+        self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0]['To'], 'alice@example.com')
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertJSONEqual(messages[0]['Mj-EventPayLoad'],
+                             {'order_id': 123, 'tier': 'premium', 'notification_batch': 'zx912'})
+        self.assertEqual(messages[1]['To'], 'Bob <bob@example.com>')
+        self.assertJSONEqual(messages[1]['Mj-EventPayLoad'],
+                             {'order_id': 678, 'notification_batch': 'zx912'})
 
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.

tests/test_mandrill_backend.py

@@ -379,7 +379,25 @@ class MandrillBackendAnymailFeatureTests(MandrillBackendMockAPITestCase):
             {'name': "group", 'content': "Users"},
             {'name': "site", 'content': "ExampleCo"},
         ])
-        self.assertEqual(data['message']['preserve_recipients'], False)  # we force with merge_data
+        self.assertIs(data['message']['preserve_recipients'], False)  # merge_data implies batch
+
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertCountEqual(data['message']['recipient_metadata'], [{
+            'rcpt': 'alice@example.com',
+            'values': {'order_id': 123, 'tier': 'premium'},
+        }, {
+            'rcpt': 'bob@example.com',
+            'values': {'order_id': 678},
+        }])
+        self.assertIs(data['message']['preserve_recipients'], False)  # merge_metadata implies batch
 
     def test_missing_from(self):
         """Make sure a missing from_email omits from* from API call.

tests/test_postmark_backend.py

@@ -398,8 +398,7 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         data = self.get_api_call_json()
         self.assertEqual(data['TemplateAlias'], 'welcome-message')
 
-    def test_merge_data(self):
-        self.set_mock_response(raw=json.dumps([{
+    _mock_batch_response = json.dumps([{
             "ErrorCode": 0,
             "Message": "OK",
             "To": "alice@example.com",
@@ -411,8 +410,10 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
             "To": "bob@example.com",
             "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
             "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d",
-        }]).encode('utf-8'))
+        }]).encode('utf-8')
 
+    def test_merge_data(self):
+        self.set_mock_response(raw=self._mock_batch_response)
         message = AnymailMessage(
             from_email='from@example.com',
             template_id=1234567,  # Postmark only supports merge_data content in a template
@@ -451,20 +452,7 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
 
     def test_merge_data_no_template(self):
         # merge_data={} can be used to force batch sending without a template
-        self.set_mock_response(raw=json.dumps([{
-            "ErrorCode": 0,
-            "Message": "OK",
-            "To": "alice@example.com",
-            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
-            "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817",
-        }, {
-            "ErrorCode": 0,
-            "Message": "OK",
-            "To": "bob@example.com",
-            "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00",
-            "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d",
-        }]).encode('utf-8'))
-
+        self.set_mock_response(raw=self._mock_batch_response)
         message = AnymailMessage(
             from_email='from@example.com',
             to=['alice@example.com', 'Bob <bob@example.com>'],
@@ -496,6 +484,45 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         self.assertEqual(recipients['bob@example.com'].status, 'sent')
         self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d')
 
+    def test_merge_metadata(self):
+        self.set_mock_response(raw=self._mock_batch_response)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123, 'tier': 'premium'},
+            'bob@example.com': {'order_id': 678},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+
+        self.assert_esp_called('/email/batch')
+        data = self.get_api_call_json()
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0]["To"], "alice@example.com")
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertEqual(data[0]["Metadata"], {'order_id': 123, 'tier': 'premium', 'notification_batch': 'zx912'})
+        self.assertEqual(data[1]["To"], "Bob <bob@example.com>")
+        self.assertEqual(data[1]["Metadata"], {'order_id': 678, 'notification_batch': 'zx912'})
+
+    def test_merge_metadata_with_template(self):
+        self.set_mock_response(raw=self._mock_batch_response)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.template_id = 1234567
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+        }
+        self.message.send()
+
+        self.assert_esp_called('/email/batchWithTemplates')
+        data = self.get_api_call_json()
+        messages = data["Messages"]
+        self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0]["To"], "alice@example.com")
+        # metadata and merge_metadata[recipient] are combined:
+        self.assertEqual(messages[0]["Metadata"], {'order_id': 123})
+        self.assertEqual(messages[1]["To"], "Bob <bob@example.com>")
+        self.assertEqual(messages[1]["Metadata"], {'order_id': 678, 'tier': 'premium'})
+
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
 

tests/test_sparkpost_backend.py

@@ -443,6 +443,24 @@ class SparkPostBackendAnymailFeatureTests(SparkPostBackendMockAPITestCase):
         ])
         self.assertEqual(params['substitution_data'], {'group': "Users", 'site': "ExampleCo"})
 
+    def test_merge_metadata(self):
+        self.set_mock_response(accepted=2)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+        }
+        self.message.metadata = {'notification_batch': 'zx912'}
+        self.message.send()
+        params = self.get_send_params()
+        self.assertEqual(params['recipients'], [
+            {'address': {'email': 'alice@example.com'},
+             'metadata': {'order_id': 123}},
+            {'address': {'email': 'bob@example.com', 'name': 'Bob'},
+             'metadata': {'order_id': 678, 'tier': 'premium'}}
+        ])
+        self.assertEqual(params['metadata'], {'notification_batch': 'zx912'})
+
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.