SendGrid UUID message_id cleanup

* Update authors
* Update integration tests
* Add webhook message_id = smtp-id fallback test case
* Test webhooks ignore smtp-id in non-fallback cases
* Update docs

AUTHORS.txt

@@ -7,6 +7,7 @@ Peter Wu
 Charlie DeTar
 Jonathan Baugh
 Noel Rignon
+Josh Kersey
 
 
 Anymail was forked from Djrill, which included contributions from:

docs/esps/sendgrid.rst

@@ -65,10 +65,14 @@ nor ``ANYMAIL_SENDGRID_API_KEY`` is set.
 
 .. rubric:: SENDGRID_GENERATE_MESSAGE_ID
 
-Whether Anymail should generate a Message-ID for messages sent
+Whether Anymail should generate a UUID for each message sent through SendGrid,
-through SendGrid, to facilitate event tracking.
+to facilitate status tracking. The UUID is attached to the message as a
+SendGrid custom arg named "anymail_id" and made available as
+:attr:`anymail_status.message_id <anymail.message.AnymailMessage.anymail_status>`
+on the sent message.
 
-Default ``True``. You can set to ``False`` to disable this behavior.
+Default ``True``. You can set to ``False`` to disable this behavior, in which
+case sent messages will have a `message_id` of ``None``.
 See :ref:`Message-ID quirks <sendgrid-message-id>` below.
 
 
@@ -162,22 +166,26 @@ Limitations and quirks
   Knowing a sent message's ID can be important for later queries about
   the message's status.
 
-  To work around this, Anymail by default generates a new Message-ID for each
+  To work around this, Anymail generates a UUID for each outgoing message,
-  outgoing message, provides it to SendGrid, and includes it in the
+  provides it to SendGrid as a custom arg named "anymail_id" and makes it
-  :attr:`~anymail.message.AnymailMessage.anymail_status`
+  available as the message's
-  attribute after you send the message.
+  :attr:`anymail_status.message_id <anymail.message.AnymailMessage.anymail_status>`
+  attribute after sending. The same UUID will be passed to Anymail's
+  :ref:`tracking webhooks <sendgrid-webhooks>` as
+  :attr:`event.message_id <anymail.signals.AnymailTrackingEvent.message_id>`.
 
-  In later SendGrid API calls, you can match that Message-ID
+  To disable attaching tracking UUIDs to sent messages, set
-  to SendGrid's ``smtp-id`` event field. (Anymail uses an additional
+  :setting:`SENDGRID_GENERATE_MESSAGE_ID <ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID>`
-  workaround to ensure smtp-id is included in all SendGrid events,
+  to False in your Anymail settings.
-  even those that aren't documented to include it.)
 
-  Anymail will use the domain of the message's :attr:`from_email`
+  .. versionchanged:: 3.0
-  to generate the Message-ID. (If this isn't desired, you can supply
-  your own Message-ID in the message's :attr:`extra_headers`.)
 
-  To disable all of these Message-ID workarounds, set
+      Previously, Anymail generated a custom :mailheader:`Message-ID`
-  :setting:`ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID` to False in your settings.
+      header for each sent message. But SendGrid's "smtp-id" event field does
+      not reliably reflect this header, which complicates status tracking.
+      (For compatibility with messages sent in earlier versions, Anymail's
+      webhook :attr:`message_id` will fall back to "smtp-id" when "anymail_id"
+      isn't present.)
 
 **Single Reply-To**
   SendGrid's v3 API only supports a single Reply-To address (and blocks

tests/test_sendgrid_backend.py

@@ -341,7 +341,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         self.message.metadata = {'user_id': "12345", 'items': 6, 'float': 98.6, 'long': longtype(123)}
         self.message.send()
         data = self.get_api_call_json()
-        data['custom_args'].pop('anymail_id', None)  # remove Message-ID we added as tracking workaround
+        data['custom_args'].pop('anymail_id', None)  # remove anymail_id we added for tracking
         self.assertEqual(data['custom_args'], {'user_id': "12345",
                                                'items': "6",  # int converted to a string,
                                                'float': "98.6",  # float converted to a string (watch binary rounding!)
@@ -581,6 +581,13 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
                          msg.anymail_status.message_id)
         self.assertEqual(msg.anymail_status.esp_response.content, self.DEFAULT_RAW_RESPONSE)
 
+    @override_settings(ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID=False)
+    def test_disable_generate_message_id(self):
+        msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['to1@example.com'],)
+        msg.send()
+        self.assertIsNone(msg.anymail_status.message_id)
+        self.assertIsNone(msg.anymail_status.recipients['to1@example.com'].message_id)
+
     # noinspection PyUnresolvedReferences
     def test_send_failed_anymail_status(self):
         """ If the send fails, anymail_status should contain initial values"""

tests/test_sendgrid_integration.py

@@ -54,7 +54,7 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
         message_id = anymail_status.recipients['to@sink.sendgrid.net'].message_id
 
         self.assertEqual(sent_status, 'queued')  # SendGrid always queues
-        self.assertRegex(message_id, r'\<.+@example\.com\>')  # should use from_email's domain
+        self.assertUUIDIsValid(message_id)  # Anymail generates a UUID tracking id
         self.assertEqual(anymail_status.status, {sent_status})  # set of all recipient statuses
         self.assertEqual(anymail_status.message_id, message_id)
 

tests/test_sendgrid_v2_backend.py

@@ -349,7 +349,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
         self.message.metadata = {'user_id': "12345", 'items': 6}
         self.message.send()
         smtpapi = self.get_smtpapi()
-        smtpapi['unique_args'].pop('anymail_id', None)  # remove Message-ID we added as tracking workaround
+        smtpapi['unique_args'].pop('anymail_id', None)  # remove anymail_id we added for tracking
         self.assertEqual(smtpapi['unique_args'], {'user_id': "12345", 'items': 6})
 
     def test_send_at(self):

tests/test_sendgrid_v2_integration.py

@@ -55,7 +55,7 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
         message_id = anymail_status.recipients['to@sink.sendgrid.net'].message_id
 
         self.assertEqual(sent_status, 'queued')  # SendGrid always queues
-        self.assertRegex(message_id, r'\<.+@example\.com\>')  # should use from_email's domain
+        self.assertUUIDIsValid(message_id)  # Anymail generates a UUID tracking id
         self.assertEqual(anymail_status.status, {sent_status})  # set of all recipient statuses
         self.assertEqual(anymail_status.message_id, message_id)
 

tests/test_sendgrid_webhooks.py

@@ -24,6 +24,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
             "email": "recipient@example.com",
             "timestamp": 1461095246,
             "anymail_id": "3c2f4df8-c6dd-4cd2-9b91-6582b81a0349",
+            "smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@ismtpd0006p1sjc2.sendgrid.net>",
             "sg_event_id": "ZyjAM5rnQmuI1KFInHQ3Nw",
             "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
             "event": "processed",
@@ -51,6 +52,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
         raw_events = [{
             "ip": "167.89.17.173",
             "response": "250 2.0.0 OK 1461095248 m143si2210036ioe.159 - gsmtp ",
+            "smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@ismtpd0006p1sjc2.sendgrid.net>",
             "sg_event_id": "nOSv8m0eTQ-vxvwNwt3fZQ",
             "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
             "tls": 1,
@@ -81,6 +83,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
             "email": "invalid@invalid",
             "anymail_id": "c74002d9-7ccb-4f67-8b8c-766cec03c9a6",
             "timestamp": 1461095250,
+            "smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@ismtpd0006p1sjc2.sendgrid.net>",
             "sg_event_id": "3NPOePGOTkeM_U3fgWApfg",
             "sg_message_id": "filter0093p1las1.9128.5717FB8127.0",
             "reason": "Invalid",
@@ -106,6 +109,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
             "email": "unsubscribe@example.com",
             "anymail_id": "a36ec0f9-aabe-45c7-9a84-3e17afb5cb65",
             "timestamp": 1461095250,
+            "smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@ismtpd0006p1sjc2.sendgrid.net>",
             "sg_event_id": "oxy9OLwMTAy5EsuZn1qhIg",
             "sg_message_id": "filter0199p1las1.4745.5717FB6F5.0",
             "reason": "Unsubscribed Address",
@@ -130,6 +134,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
         raw_events = [{
             "ip": "167.89.17.173",
             "status": "5.1.1",
+            "smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@ismtpd0006p1sjc2.sendgrid.net>",
             "sg_event_id": "lC0Rc-FuQmKbnxCWxX1jRQ",
             "reason": "550 5.1.1 The email account that you tried to reach does not exist.",
             "sg_message_id": "Lli-03HcQ5-JLybO9fXsJg.filter0077p1las1.21536.5717FC482.0",
@@ -157,6 +162,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
     def test_deferred_event(self):
         raw_events = [{
             "response": "Email was deferred due to the following reason(s): [IPs were throttled by recipient server]",
+            "smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@ismtpd0006p1sjc2.sendgrid.net>",
             "sg_event_id": "b_syL5UiTvWC_Ky5L6Bs5Q",
             "sg_message_id": "u9Gvi3mzT6iC2poAb58_qQ.filter0465p1mdw1.8054.5718271B40.0",
             "event": "deferred",
@@ -232,3 +238,29 @@ class SendGridDeliveryTestCase(WebhookTestCase):
         self.assertEqual(event.recipient, "recipient@example.com")
         self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36")
         self.assertEqual(event.click_url, "http://www.example.com")
+
+    def test_compatibility_message_id_from_smtp_id(self):
+        # Prior to v3.0, Anymail tried to use a custom Message-ID header as
+        # the `message_id`, and relied on SendGrid passing that to webhooks as
+        # 'smtp-id'. Make sure webhooks extract message_id for messages sent
+        # with earlier Anymail versions. (See issue #108.)
+        raw_events = [{
+            "ip": "167.89.17.173",
+            "response": "250 2.0.0 OK 1461095248 m143si2210036ioe.159 - gsmtp ",
+            "smtp-id": "<152712433591.85282.8340115595767222398@example.com>",
+            "sg_event_id": "nOSv8m0eTQ-vxvwNwt3fZQ",
+            "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
+            "tls": 1,
+            "event": "delivered",
+            "email": "recipient@example.com",
+            "timestamp": 1461095250,
+        }]
+        response = self.client.post('/anymail/sendgrid/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendGridTrackingWebhookView,
+                                                      event=ANY, esp_name='SendGrid')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.message_id, "<152712433591.85282.8340115595767222398@example.com>")
+        self.assertEqual(event.metadata, {})  # smtp-id not left in metadata

tests/utils.py

@@ -166,13 +166,13 @@ class AnymailTestMixin:
         second = rfc822_unfold(second)
         self.assertEqual(first, second, msg)
 
-    def assertUUIDIsValid(self, uuid_str, version=4):
+    def assertUUIDIsValid(self, uuid_str, msg=None, version=4):
         """Assert the uuid_str evaluates to a valid UUID"""
         try:
             uuid.UUID(uuid_str, version=version)
         except (ValueError, AttributeError, TypeError):
-            return False
+            raise self.failureException(
-        return True
+                msg or "%r is not a valid UUID" % uuid_str)
 
 
 # Backported from Python 3.4