SendGrid: add merge_metadata

Add support in SendGrid backend for per-recipient metadata.

anymail/backends/base.py

@@ -245,6 +245,7 @@ class BasePayload(object):
         ('template_id', last, force_non_lazy),
         ('merge_data', combine, force_non_lazy_dict),
         ('merge_global_data', combine, force_non_lazy_dict),
+        ('merge_metadata', combine, force_non_lazy_dict),
         ('esp_extra', combine, force_non_lazy_dict),
     )
     esp_message_attrs = ()  # subclasses can override
@@ -495,6 +496,9 @@ class BasePayload(object):
     def set_merge_global_data(self, merge_global_data):
         self.unsupported_feature("merge_global_data")
 
+    def set_merge_metadata(self, merge_metadata):
+        self.unsupported_feature("merge_metadata")
+
     # ESP-specific payload construction
     def set_esp_extra(self, extra):
         self.unsupported_feature("esp_extra")

anymail/backends/sendgrid.py

@@ -77,6 +77,7 @@ class SendGridPayload(RequestsPayload):
         self.merge_field_format = backend.merge_field_format
         self.merge_data = None  # late-bound per-recipient data
         self.merge_global_data = None
+        self.merge_metadata = None
 
         http_headers = kwargs.pop('headers', {})
         http_headers['Authorization'] = 'Bearer %s' % backend.api_key
@@ -101,6 +102,7 @@ class SendGridPayload(RequestsPayload):
         if self.generate_message_id:
             self.set_anymail_id()
         self.build_merge_data()
+        self.build_merge_metadata()
 
         if not self.data["headers"]:
             del self.data["headers"]  # don't send empty headers
@@ -204,6 +206,28 @@ class SendGridPayload(RequestsPayload):
                     "Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
                     AnymailWarning)
 
+    def build_merge_metadata(self):
+        if self.merge_metadata is None:
+            return
+
+        if self.merge_data is None:
+            # Burst apart each to-email in personalizations[0] into a separate
+            # personalization, and add merge_metadata for that recipient
+            assert len(self.data["personalizations"]) == 1
+            base_personalizations = self.data["personalizations"].pop()
+            to_list = base_personalizations.pop("to")  # {email, name?} for each message.to
+            for recipient in to_list:
+                personalization = base_personalizations.copy()  # captures cc, bcc, and any esp_extra
+                personalization["to"] = [recipient]
+                self.data["personalizations"].append(personalization)
+
+        for personalization in self.data["personalizations"]:
+            recipient_email = personalization["to"][0]["email"]
+            recipient_metadata = self.merge_metadata.get(recipient_email)
+            if recipient_metadata:
+                recipient_custom_args = self.transform_metadata(recipient_metadata)
+                personalization["custom_args"] = recipient_custom_args
+
     #
     # Payload construction
     #
@@ -296,11 +320,14 @@ class SendGridPayload(RequestsPayload):
         self.data.setdefault("attachments", []).append(att)
 
     def set_metadata(self, metadata):
+        self.data["custom_args"] = self.transform_metadata(metadata)
+
+    def transform_metadata(self, metadata):
         # SendGrid requires custom_args values to be strings -- not integers.
         # (And issues the cryptic error {"field": null, "message": "Bad Request", "help": null}
         # if they're not.)
         # We'll stringify ints and floats; anything else is the caller's responsibility.
-        self.data["custom_args"] = {
+        return {
             k: str(v) if isinstance(v, BASIC_NUMERIC_TYPES) else v
             for k, v in metadata.items()
         }
@@ -344,6 +371,12 @@ class SendGridPayload(RequestsPayload):
         # template type and merge_field_format.
         self.merge_global_data = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # Becomes personalizations[...]['custom_args'] in
+        # build_merge_data, after we know recipients, template type,
+        # and merge_field_format.
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.merge_field_format = extra.pop("merge_field_format", self.merge_field_format)
         self.use_dynamic_template = extra.pop("use_dynamic_template", self.use_dynamic_template)

tests/test_sendgrid_backend.py

@@ -11,6 +11,7 @@ import six
 from django.core import mail
 from django.test import SimpleTestCase, override_settings, tag
 from django.utils.timezone import get_fixed_timezone, override as override_current_timezone
+from mock import patch
 
 from anymail.exceptions import (AnymailAPIError, AnymailConfigurationError, AnymailSerializationError,
                                 AnymailUnsupportedFeature, AnymailWarning)
@@ -32,6 +33,13 @@ class SendGridBackendMockAPITestCase(RequestsBackendMockAPITestCase):
 
     def setUp(self):
         super(SendGridBackendMockAPITestCase, self).setUp()
+
+        # Patch uuid4 to generate predictable anymail_ids for testing
+        patch_uuid4 = patch('anymail.backends.sendgrid.uuid.uuid4',
+                            side_effect=["mocked-uuid-%d" % n for n in range(1, 5)])
+        patch_uuid4.start()
+        self.addCleanup(patch_uuid4.stop)
+
         # Simple message useful for many tests
         self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', 'from@example.com', ['to@example.com'])
 
@@ -57,7 +65,7 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
             'to': [{'email': "to@example.com"}],
         }])
         # make sure the backend assigned the anymail_id for event tracking and notification
-        self.assertUUIDIsValid(data['custom_args']['anymail_id'])
+        self.assertEqual(data['custom_args']['anymail_id'], 'mocked-uuid-1')
 
     def test_name_addr(self):
         """Make sure RFC2822 name-addr format (with display-name) is allowed
@@ -118,7 +126,7 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
             'Message-ID': "<mycustommsgid@sales.example.com>",
         })
         # make sure custom Message-ID also added to custom_args
-        self.assertUUIDIsValid(data['custom_args']['anymail_id'])
+        self.assertEqual(data['custom_args']['anymail_id'], 'mocked-uuid-1')
 
     def test_html_message(self):
         text_content = 'This is an important message.'
@@ -573,6 +581,122 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         with self.assertWarnsRegex(AnymailWarning, r'SENDGRID_MERGE_FIELD_FORMAT'):
             self.message.send()
 
+    def test_merge_metadata(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+        }
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'alice@example.com'}],
+             'custom_args': {'order_id': '123'}},
+            {'to': [{'email': 'bob@example.com', 'name': '"Bob"'}],
+             'custom_args': {'order_id': '678', 'tier': 'premium'}},
+        ])
+        self.assertEqual(data['custom_args'], {'anymail_id': 'mocked-uuid-1'})
+
+    def test_metadata_with_merge_metadata(self):
+        # Per SendGrid docs: "personalizations[x].custom_args will be merged
+        # with message level custom_args, overriding any conflicting keys."
+        # So there's no need to merge global metadata with per-recipient merge_metadata
+        # (like we have to for template merge_global_data and merge_data).
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>']
+        self.message.metadata = {'tier': 'basic', 'batch': 'ax24'}
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+        }
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'alice@example.com'}],
+             'custom_args': {'order_id': '123'}},
+            {'to': [{'email': 'bob@example.com', 'name': '"Bob"'}],
+             'custom_args': {'order_id': '678', 'tier': 'premium'}},
+        ])
+        self.assertEqual(data['custom_args'],
+                         {'tier': 'basic', 'batch': 'ax24', 'anymail_id': 'mocked-uuid-1'})
+
+    def test_merge_metadata_with_merge_data(self):
+        # (using dynamic templates)
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>', 'celia@example.com']
+        self.message.cc = ['cc@example.com']  # gets applied to *each* recipient in a merge
+        self.message.template_id = "d-5a963add2ec84305813ff860db277d7a"
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"}
+            # and no data for celia@example.com
+        }
+        self.message.merge_global_data = {
+            'group': "Users",
+            'site': "ExampleCo",
+        }
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+            # and no metadata for celia@example.com
+        }
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'alice@example.com'}],
+             'cc': [{'email': 'cc@example.com'}],  # all recipients get the cc
+             'dynamic_template_data': {
+                 'name': "Alice", 'group': "Developers", 'site': "ExampleCo"},
+             'custom_args': {'order_id': '123'}},
+            {'to': [{'email': 'bob@example.com', 'name': '"Bob"'}],
+             'cc': [{'email': 'cc@example.com'}],
+             'dynamic_template_data': {
+                 'name': "Bob", 'group': "Users", 'site': "ExampleCo"},
+             'custom_args': {'order_id': '678', 'tier': 'premium'}},
+            {'to': [{'email': 'celia@example.com'}],
+             'cc': [{'email': 'cc@example.com'}],
+             'dynamic_template_data': {
+                 'group': "Users", 'site': "ExampleCo"}},
+        ])
+
+    def test_merge_metadata_with_legacy_template(self):
+        self.message.to = ['alice@example.com', 'Bob <bob@example.com>', 'celia@example.com']
+        self.message.cc = ['cc@example.com']  # gets applied to *each* recipient in a merge
+        self.message.template_id = "5a963add2ec84305813ff860db277d7a"
+        self.message.esp_extra = {'merge_field_format': ':{}'}
+        self.message.merge_data = {
+            'alice@example.com': {'name': "Alice", 'group': "Developers"},
+            'bob@example.com': {'name': "Bob"}
+            # and no data for celia@example.com
+        }
+        self.message.merge_global_data = {
+            'group': "Users",
+            'site': "ExampleCo",
+        }
+        self.message.merge_metadata = {
+            'alice@example.com': {'order_id': 123},
+            'bob@example.com': {'order_id': 678, 'tier': 'premium'},
+            # and no metadata for celia@example.com
+        }
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['personalizations'], [
+            {'to': [{'email': 'alice@example.com'}],
+             'cc': [{'email': 'cc@example.com'}],  # all recipients get the cc
+             'custom_args': {'order_id': '123'},
+             'substitutions': {':name': "Alice", ':group': "Developers", ':site': ":site"}},
+            {'to': [{'email': 'bob@example.com', 'name': '"Bob"'}],
+             'cc': [{'email': 'cc@example.com'}],
+             'custom_args': {'order_id': '678', 'tier': 'premium'},
+             'substitutions': {':name': "Bob", ':group': ":group", ':site': ":site"}},
+            {'to': [{'email': 'celia@example.com'}],
+             'cc': [{'email': 'cc@example.com'}],
+             # no custom_args
+             'substitutions': {':group': ":group", ':site': ":site"}},
+        ])
+        self.assertEqual(data['sections'], {
+            ':group': "Users",
+            ':site': "ExampleCo",
+        })
+
     @override_settings(ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID=False)  # else we force custom_args
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
@@ -666,7 +790,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         sent = msg.send()
         self.assertEqual(sent, 1)
         self.assertEqual(msg.anymail_status.status, {'queued'})
-        self.assertUUIDIsValid(msg.anymail_status.message_id)  # don't know exactly what it'll be
+        self.assertEqual(msg.anymail_status.message_id, 'mocked-uuid-1')
         self.assertEqual(msg.anymail_status.recipients['to1@example.com'].status, 'queued')
         self.assertEqual(msg.anymail_status.recipients['to1@example.com'].message_id,
                          msg.anymail_status.message_id)