diff --git a/AUTHORS.txt b/AUTHORS.txt index c0727d8..285121b 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -7,6 +7,7 @@ Peter Wu Charlie DeTar Jonathan Baugh Noel Rignon +Josh Kersey Anymail was forked from Djrill, which included contributions from: diff --git a/docs/esps/sendgrid.rst b/docs/esps/sendgrid.rst index 533c014..7208bb7 100644 --- a/docs/esps/sendgrid.rst +++ b/docs/esps/sendgrid.rst @@ -65,10 +65,14 @@ nor ``ANYMAIL_SENDGRID_API_KEY`` is set. .. rubric:: SENDGRID_GENERATE_MESSAGE_ID -Whether Anymail should generate a Message-ID for messages sent -through SendGrid, to facilitate event tracking. +Whether Anymail should generate a UUID for each message sent through SendGrid, +to facilitate status tracking. The UUID is attached to the message as a +SendGrid custom arg named "anymail_id" and made available as +:attr:`anymail_status.message_id ` +on the sent message. -Default ``True``. You can set to ``False`` to disable this behavior. +Default ``True``. You can set to ``False`` to disable this behavior, in which +case sent messages will have a `message_id` of ``None``. See :ref:`Message-ID quirks ` below. @@ -162,22 +166,26 @@ Limitations and quirks Knowing a sent message's ID can be important for later queries about the message's status. - To work around this, Anymail by default generates a new Message-ID for each - outgoing message, provides it to SendGrid, and includes it in the - :attr:`~anymail.message.AnymailMessage.anymail_status` - attribute after you send the message. + To work around this, Anymail generates a UUID for each outgoing message, + provides it to SendGrid as a custom arg named "anymail_id" and makes it + available as the message's + :attr:`anymail_status.message_id ` + attribute after sending. The same UUID will be passed to Anymail's + :ref:`tracking webhooks ` as + :attr:`event.message_id `. - In later SendGrid API calls, you can match that Message-ID - to SendGrid's ``smtp-id`` event field. (Anymail uses an additional - workaround to ensure smtp-id is included in all SendGrid events, - even those that aren't documented to include it.) + To disable attaching tracking UUIDs to sent messages, set + :setting:`SENDGRID_GENERATE_MESSAGE_ID ` + to False in your Anymail settings. - Anymail will use the domain of the message's :attr:`from_email` - to generate the Message-ID. (If this isn't desired, you can supply - your own Message-ID in the message's :attr:`extra_headers`.) + .. versionchanged:: 3.0 - To disable all of these Message-ID workarounds, set - :setting:`ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID` to False in your settings. + Previously, Anymail generated a custom :mailheader:`Message-ID` + header for each sent message. But SendGrid's "smtp-id" event field does + not reliably reflect this header, which complicates status tracking. + (For compatibility with messages sent in earlier versions, Anymail's + webhook :attr:`message_id` will fall back to "smtp-id" when "anymail_id" + isn't present.) **Single Reply-To** SendGrid's v3 API only supports a single Reply-To address (and blocks diff --git a/tests/test_sendgrid_backend.py b/tests/test_sendgrid_backend.py index 8c42e6c..dc7ec69 100644 --- a/tests/test_sendgrid_backend.py +++ b/tests/test_sendgrid_backend.py @@ -341,7 +341,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): self.message.metadata = {'user_id': "12345", 'items': 6, 'float': 98.6, 'long': longtype(123)} self.message.send() data = self.get_api_call_json() - data['custom_args'].pop('anymail_id', None) # remove Message-ID we added as tracking workaround + data['custom_args'].pop('anymail_id', None) # remove anymail_id we added for tracking self.assertEqual(data['custom_args'], {'user_id': "12345", 'items': "6", # int converted to a string, 'float': "98.6", # float converted to a string (watch binary rounding!) @@ -581,6 +581,13 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): msg.anymail_status.message_id) self.assertEqual(msg.anymail_status.esp_response.content, self.DEFAULT_RAW_RESPONSE) + @override_settings(ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID=False) + def test_disable_generate_message_id(self): + msg = mail.EmailMessage('Subject', 'Message', 'from@example.com', ['to1@example.com'],) + msg.send() + self.assertIsNone(msg.anymail_status.message_id) + self.assertIsNone(msg.anymail_status.recipients['to1@example.com'].message_id) + # noinspection PyUnresolvedReferences def test_send_failed_anymail_status(self): """ If the send fails, anymail_status should contain initial values""" diff --git a/tests/test_sendgrid_integration.py b/tests/test_sendgrid_integration.py index 9f03fd4..f56fbd1 100644 --- a/tests/test_sendgrid_integration.py +++ b/tests/test_sendgrid_integration.py @@ -54,7 +54,7 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): message_id = anymail_status.recipients['to@sink.sendgrid.net'].message_id self.assertEqual(sent_status, 'queued') # SendGrid always queues - self.assertRegex(message_id, r'\<.+@example\.com\>') # should use from_email's domain + self.assertUUIDIsValid(message_id) # Anymail generates a UUID tracking id self.assertEqual(anymail_status.status, {sent_status}) # set of all recipient statuses self.assertEqual(anymail_status.message_id, message_id) diff --git a/tests/test_sendgrid_v2_backend.py b/tests/test_sendgrid_v2_backend.py index 950435e..385b98e 100644 --- a/tests/test_sendgrid_v2_backend.py +++ b/tests/test_sendgrid_v2_backend.py @@ -349,7 +349,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): self.message.metadata = {'user_id': "12345", 'items': 6} self.message.send() smtpapi = self.get_smtpapi() - smtpapi['unique_args'].pop('anymail_id', None) # remove Message-ID we added as tracking workaround + smtpapi['unique_args'].pop('anymail_id', None) # remove anymail_id we added for tracking self.assertEqual(smtpapi['unique_args'], {'user_id': "12345", 'items': 6}) def test_send_at(self): diff --git a/tests/test_sendgrid_v2_integration.py b/tests/test_sendgrid_v2_integration.py index 1dfb254..2215875 100644 --- a/tests/test_sendgrid_v2_integration.py +++ b/tests/test_sendgrid_v2_integration.py @@ -55,7 +55,7 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): message_id = anymail_status.recipients['to@sink.sendgrid.net'].message_id self.assertEqual(sent_status, 'queued') # SendGrid always queues - self.assertRegex(message_id, r'\<.+@example\.com\>') # should use from_email's domain + self.assertUUIDIsValid(message_id) # Anymail generates a UUID tracking id self.assertEqual(anymail_status.status, {sent_status}) # set of all recipient statuses self.assertEqual(anymail_status.message_id, message_id) diff --git a/tests/test_sendgrid_webhooks.py b/tests/test_sendgrid_webhooks.py index 2978292..215ee24 100644 --- a/tests/test_sendgrid_webhooks.py +++ b/tests/test_sendgrid_webhooks.py @@ -24,6 +24,7 @@ class SendGridDeliveryTestCase(WebhookTestCase): "email": "recipient@example.com", "timestamp": 1461095246, "anymail_id": "3c2f4df8-c6dd-4cd2-9b91-6582b81a0349", + "smtp-id": "", "sg_event_id": "ZyjAM5rnQmuI1KFInHQ3Nw", "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0", "event": "processed", @@ -51,6 +52,7 @@ class SendGridDeliveryTestCase(WebhookTestCase): raw_events = [{ "ip": "167.89.17.173", "response": "250 2.0.0 OK 1461095248 m143si2210036ioe.159 - gsmtp ", + "smtp-id": "", "sg_event_id": "nOSv8m0eTQ-vxvwNwt3fZQ", "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0", "tls": 1, @@ -81,6 +83,7 @@ class SendGridDeliveryTestCase(WebhookTestCase): "email": "invalid@invalid", "anymail_id": "c74002d9-7ccb-4f67-8b8c-766cec03c9a6", "timestamp": 1461095250, + "smtp-id": "", "sg_event_id": "3NPOePGOTkeM_U3fgWApfg", "sg_message_id": "filter0093p1las1.9128.5717FB8127.0", "reason": "Invalid", @@ -106,6 +109,7 @@ class SendGridDeliveryTestCase(WebhookTestCase): "email": "unsubscribe@example.com", "anymail_id": "a36ec0f9-aabe-45c7-9a84-3e17afb5cb65", "timestamp": 1461095250, + "smtp-id": "", "sg_event_id": "oxy9OLwMTAy5EsuZn1qhIg", "sg_message_id": "filter0199p1las1.4745.5717FB6F5.0", "reason": "Unsubscribed Address", @@ -130,6 +134,7 @@ class SendGridDeliveryTestCase(WebhookTestCase): raw_events = [{ "ip": "167.89.17.173", "status": "5.1.1", + "smtp-id": "", "sg_event_id": "lC0Rc-FuQmKbnxCWxX1jRQ", "reason": "550 5.1.1 The email account that you tried to reach does not exist.", "sg_message_id": "Lli-03HcQ5-JLybO9fXsJg.filter0077p1las1.21536.5717FC482.0", @@ -157,6 +162,7 @@ class SendGridDeliveryTestCase(WebhookTestCase): def test_deferred_event(self): raw_events = [{ "response": "Email was deferred due to the following reason(s): [IPs were throttled by recipient server]", + "smtp-id": "", "sg_event_id": "b_syL5UiTvWC_Ky5L6Bs5Q", "sg_message_id": "u9Gvi3mzT6iC2poAb58_qQ.filter0465p1mdw1.8054.5718271B40.0", "event": "deferred", @@ -232,3 +238,29 @@ class SendGridDeliveryTestCase(WebhookTestCase): self.assertEqual(event.recipient, "recipient@example.com") self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36") self.assertEqual(event.click_url, "http://www.example.com") + + def test_compatibility_message_id_from_smtp_id(self): + # Prior to v3.0, Anymail tried to use a custom Message-ID header as + # the `message_id`, and relied on SendGrid passing that to webhooks as + # 'smtp-id'. Make sure webhooks extract message_id for messages sent + # with earlier Anymail versions. (See issue #108.) + raw_events = [{ + "ip": "167.89.17.173", + "response": "250 2.0.0 OK 1461095248 m143si2210036ioe.159 - gsmtp ", + "smtp-id": "<152712433591.85282.8340115595767222398@example.com>", + "sg_event_id": "nOSv8m0eTQ-vxvwNwt3fZQ", + "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0", + "tls": 1, + "event": "delivered", + "email": "recipient@example.com", + "timestamp": 1461095250, + }] + response = self.client.post('/anymail/sendgrid/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendGridTrackingWebhookView, + event=ANY, esp_name='SendGrid') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.message_id, "<152712433591.85282.8340115595767222398@example.com>") + self.assertEqual(event.metadata, {}) # smtp-id not left in metadata diff --git a/tests/utils.py b/tests/utils.py index fddeb46..5ce2ed8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -166,13 +166,13 @@ class AnymailTestMixin: second = rfc822_unfold(second) self.assertEqual(first, second, msg) - def assertUUIDIsValid(self, uuid_str, version=4): + def assertUUIDIsValid(self, uuid_str, msg=None, version=4): """Assert the uuid_str evaluates to a valid UUID""" try: uuid.UUID(uuid_str, version=version) except (ValueError, AttributeError, TypeError): - return False - return True + raise self.failureException( + msg or "%r is not a valid UUID" % uuid_str) # Backported from Python 3.4