From 8e43f299444b5d3118cd125a3d0bd384e2358573 Mon Sep 17 00:00:00 2001 From: medmunds Date: Wed, 20 Apr 2016 19:05:55 -0700 Subject: [PATCH] Workaround missing smtp-id in SendGrid tracking. * Add smtp-id in unique_args (metadata), to ensure it shows up in click and open events. * Add SENDGRID_GENERATE_MESSAGE_ID setting, default True, to control auto-Message-ID behavior. * Document it. --- anymail/backends/sendgrid.py | 29 +++++++++++++++++++++++------ docs/esps/sendgrid.rst | 22 ++++++++++++++++++++-- tests/test_sendgrid_backend.py | 12 ++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/anymail/backends/sendgrid.py b/anymail/backends/sendgrid.py index d7ad6c4..c4fbdec 100644 --- a/anymail/backends/sendgrid.py +++ b/anymail/backends/sendgrid.py @@ -1,3 +1,5 @@ +from email.utils import unquote + from django.core.exceptions import ImproperlyConfigured from django.core.mail import make_msgid from requests.structures import CaseInsensitiveDict @@ -9,7 +11,6 @@ from ..utils import get_anymail_setting, timestamp from .base_requests import AnymailRequestsBackend, RequestsPayload - class SendGridBackend(AnymailRequestsBackend): """ SendGrid API Email Backend @@ -27,6 +28,8 @@ class SendGridBackend(AnymailRequestsBackend): "SENDGRID_PASSWORD in your Django ANYMAIL settings." ) + self.generate_message_id = get_anymail_setting('SENDGRID_GENERATE_MESSAGE_ID', default=True) + # This is SendGrid's Web API v2 (because the Web API v3 doesn't support sending) api_url = get_anymail_setting("SENDGRID_API_URL", "https://api.sendgrid.com/api/") if not api_url.endswith("/"): @@ -56,6 +59,7 @@ class SendGridPayload(RequestsPayload): def __init__(self, message, defaults, backend, *args, **kwargs): self.all_recipients = [] # used for backend.parse_recipient_status + self.generate_message_id = backend.generate_message_id self.message_id = None # Message-ID -- assigned in serialize_data unless provided in headers self.smtpapi = {} # SendGrid x-smtpapi field @@ -76,6 +80,9 @@ class SendGridPayload(RequestsPayload): def serialize_data(self): """Performs any necessary serialization on self.data, and returns the result.""" + if self.generate_message_id: + self.ensure_message_id() + # Serialize x-smtpapi to json: if len(self.smtpapi) > 0: # If esp_extra was also used to set x-smtpapi, need to merge it @@ -86,16 +93,26 @@ class SendGridPayload(RequestsPayload): elif "x-smtpapi" in self.data: self.data["x-smtpapi"] = self.serialize_json(self.data["x-smtpapi"]) - # Add our own message_id, and serialize extra headers to json: + # Serialize extra headers to json: headers = self.data["headers"] - try: - self.message_id = headers["Message-ID"] - except KeyError: - self.message_id = headers["Message-ID"] = self.make_message_id() self.data["headers"] = self.serialize_json(dict(headers.items())) return self.data + def ensure_message_id(self): + """Ensure message has a known Message-ID for later event tracking""" + headers = self.data["headers"] + if "Message-ID" not in headers: + # Only make our own if caller hasn't already provided one + headers["Message-ID"] = self.make_message_id() + self.message_id = headers["Message-ID"] + + # Workaround for missing message ID (smtp-id) in SendGrid engagement events + # (click and open tracking): because unique_args get merged into the raw event + # record, we can supply the 'smtp-id' field for any events missing it. + # Must use the unquoted (no ) version to match other SendGrid APIs. + self.smtpapi.setdefault('unique_args', {})['smtp-id'] = unquote(self.message_id) + def make_message_id(self): """Returns a Message-ID that could be used for this payload diff --git a/docs/esps/sendgrid.rst b/docs/esps/sendgrid.rst index 6dc289a..85350cc 100644 --- a/docs/esps/sendgrid.rst +++ b/docs/esps/sendgrid.rst @@ -78,6 +78,17 @@ nor ``ANYMAIL_SENDGRID_USERNAME`` is set. .. _SendGrid credentials settings: https://app.sendgrid.com/settings/credentials +.. setting:: ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID + +.. rubric:: SENDGRID_GENERATE_MESSAGE_ID + +Whether Anymail should generate a Message-ID for messages sent +through SendGrid, to facilitate event tracking. + +Default ``True``. You can set to ``False`` to disable this behavior. +See :ref:`Message-ID quirks ` below. + + .. setting:: ANYMAIL_SENDGRID_API_URL .. rubric:: SENDGRID_API_URL @@ -132,23 +143,30 @@ Limitations and quirks make sure each one has a unique, non-empty filename. +.. _sendgrid-message-id: + **Message-ID** SendGrid does not return any sort of unique id from its send API call. Knowing a sent message's ID can be important for later queries about the message's status. - To work around this, Anymail generates a new Message-ID for each + To work around this, Anymail by default generates a new Message-ID for each outgoing message, provides it to SendGrid, and includes it in the :attr:`~anymail.message.AnymailMessage.anymail_status` attribute after you send the message. In later SendGrid API calls, you can match that Message-ID - to SendGrid's ``smtp-id`` event field. + to SendGrid's ``smtp-id`` event field. (Anymail uses an additional + workaround to ensure smtp-id is included in all SendGrid events, + even those that aren't documented to include it.) Anymail will use the domain of the message's :attr:`from_email` to generate the Message-ID. (If this isn't desired, you can supply your own Message-ID in the message's :attr:`extra_headers`.) + To disable all of these Message-ID workarounds, set + :setting:`ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID` to False in your settings. + **Invalid Addresses** SendGrid will accept *and send* just about anything as diff --git a/tests/test_sendgrid_backend.py b/tests/test_sendgrid_backend.py index 387500b..955fe7d 100644 --- a/tests/test_sendgrid_backend.py +++ b/tests/test_sendgrid_backend.py @@ -64,6 +64,9 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase): # make sure backend assigned a Message-ID for event tracking email_headers = json.loads(data['headers']) self.assertRegex(email_headers['Message-ID'], r'\<.+@sender\.example\.com\>') # id uses from_email's domain + # make sure we added the unquoted Message-ID to unique_args for event notification + smtpapi = self.get_smtpapi() + self.assertEqual(email_headers['Message-ID'], '<{}>'.format(smtpapi['unique_args']['smtp-id'])) @override_settings(ANYMAIL={'SENDGRID_USERNAME': 'sg_username', 'SENDGRID_PASSWORD': 'sg_password'}) def test_user_pass_auth(self): @@ -124,6 +127,10 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase): 'Reply-To': 'another@example.com', 'X-MyHeader': 'my value', }) + # make sure custom Message-ID also added to unique_args + self.assertJSONEqual(data['x-smtpapi'], { + 'unique_args': {'smtp-id': 'mycustommsgid@sales.example.com'} + }) def test_html_message(self): text_content = 'This is an important message.' @@ -347,6 +354,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): self.message.metadata = {'user_id': "12345", 'items': 6} self.message.send() smtpapi = self.get_smtpapi() + smtpapi['unique_args'].pop('smtp-id', None) # remove Message-ID we added as tracking workaround self.assertEqual(smtpapi['unique_args'], {'user_id': "12345", 'items': 6}) def test_send_at(self): @@ -401,6 +409,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase): self.assertEqual(smtpapi['filters']['clicktrack'], {'settings': {'enable': 1}}) self.assertEqual(smtpapi['filters']['opentrack'], {'settings': {'enable': 0}}) + @override_settings(ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID=False) # else we force unique_args def test_default_omits_options(self): """Make sure by default we don't send any ESP-specific options. @@ -508,6 +517,7 @@ class SendGridBackendSendDefaultsTests(SendGridBackendMockAPITestCase): data = self.get_api_call_data() smtpapi = self.get_smtpapi() # All these values came from ANYMAIL_SEND_DEFAULTS: + smtpapi['unique_args'].pop('smtp-id', None) # remove Message-ID we added as tracking workaround self.assertEqual(smtpapi['unique_args'], {'global': 'globalvalue', 'other': 'othervalue'}) self.assertEqual(smtpapi['category'], ['globaltag']) self.assertEqual(smtpapi['filters']['clicktrack']['settings']['enable'], 1) @@ -525,6 +535,7 @@ class SendGridBackendSendDefaultsTests(SendGridBackendMockAPITestCase): data = self.get_api_call_data() smtpapi = self.get_smtpapi() # All these values came from ANYMAIL_SEND_DEFAULTS + message.*: + smtpapi['unique_args'].pop('smtp-id', None) # remove Message-ID we added as tracking workaround self.assertEqual(smtpapi['unique_args'], { 'global': 'globalvalue', 'message': 'messagevalue', # additional metadata @@ -547,6 +558,7 @@ class SendGridBackendSendDefaultsTests(SendGridBackendMockAPITestCase): data = self.get_api_call_data() smtpapi = self.get_smtpapi() # All these values came from ANYMAIL_SEND_DEFAULTS plus ANYMAIL_SENDGRID_SEND_DEFAULTS: + smtpapi['unique_args'].pop('smtp-id', None) # remove Message-ID we added as tracking workaround self.assertEqual(smtpapi['unique_args'], {'esp': 'espvalue'}) # entire metadata overridden self.assertCountEqual(smtpapi['category'], ['esptag']) # entire tags overridden self.assertEqual(smtpapi['filters']['clicktrack']['settings']['enable'], 1) # no override