pacnpal/django-anymail
1
0
Fork 0
mirror of https://github.com/pacnpal/django-anymail.git synced 2025-12-20 11:51:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

SendGrid: change message_id from Message-ID/smtp-id to UUID anymail_id

Browse Source 
SendGrid does not always correctly provide the sent Message-ID header value 
to a tracking webhook's smtp-id field, making it unreliable to use for Anymail's 
`message_id`.

Instead, generate a UUID `message_id` for Anymail tracking, and pass it from 
send to webhooks in SendGrid custom args as anymail_id.

Webhooks will fall back to smtp-id for compatibility with previously-sent 
messages that didn't have an anymail_id custom arg.

Fixes #108
This commit is contained in:
Josh Kersey
2018-05-30 13:52:36 -05:00
committed by Mike Edmunds
parent 51d2a404c0
commit d8d1407c61
7 changed files with 52 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
anymail/backends/sendgrid.py
View File

@@ -1,7 +1,7 @@
import uuid
from email.utils import quote as rfc822_quote
import warnings
from django.core.mail import make_msgid
from requests.structures import CaseInsensitiveDict
from .base_requests import AnymailRequestsBackend, RequestsPayload
@@ -99,7 +99,7 @@ class SendGridPayload(RequestsPayload):
"""Performs any necessary serialization on self.data, and returns the result."""
if self.generate_message_id:
self.ensure_message_id()
self.set_anymail_id()
self.build_merge_data()
if not self.data["headers"]:
@@ -107,28 +107,11 @@ class SendGridPayload(RequestsPayload):
return self.serialize_json(self.data)
def ensure_message_id(self):
"""Ensure message has a known Message-ID for later event tracking"""
if "Message-ID" not in self.data["headers"]:
# Only make our own if caller hasn't already provided one
self.data["headers"]["Message-ID"] = self.make_message_id()
self.message_id = self.data["headers"]["Message-ID"]
def set_anymail_id(self):
"""Ensure message has a known anymail_id for later event tracking"""
# Workaround for missing message ID (smtp-id) in SendGrid engagement events
# (click and open tracking): because unique_args get merged into the raw event
# record, we can supply the 'smtp-id' field for any events missing it.
self.data.setdefault("custom_args", {})["smtp-id"] = self.message_id
def make_message_id(self):
"""Returns a Message-ID that could be used for this payload
Tries to use the from_email's domain as the Message-ID's domain
"""
try:
_, domain = self.data["from"]["email"].split("@")
except (AttributeError, KeyError, TypeError, ValueError):
domain = None
return make_msgid(domain=domain)
self.message_id = str(uuid.uuid4())
self.data.setdefault("custom_args", {})["anymail_id"] = self.message_id
def build_merge_data(self):
"""Set personalizations[...]['substitutions'] and data['sections']"""

30
anymail/backends/sendgrid_v2.py
View File

@@ -1,6 +1,6 @@
import uuid
import warnings
from django.core.mail import make_msgid
from requests.structures import CaseInsensitiveDict
from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError, AnymailWarning
@@ -99,7 +99,7 @@ class SendGridPayload(RequestsPayload):
"""Performs any necessary serialization on self.data, and returns the result."""
if self.generate_message_id:
self.ensure_message_id()
self.set_anymail_id()
self.build_merge_data()
if self.merge_data is not None:
@@ -136,29 +136,11 @@ class SendGridPayload(RequestsPayload):
return self.data
def ensure_message_id(self):
"""Ensure message has a known Message-ID for later event tracking"""
headers = self.data["headers"]
if "Message-ID" not in headers:
# Only make our own if caller hasn't already provided one
headers["Message-ID"] = self.make_message_id()
self.message_id = headers["Message-ID"]
def set_anymail_id(self):
"""Ensure message has a known anymail_id for later event tracking"""
# Workaround for missing message ID (smtp-id) in SendGrid engagement events
# (click and open tracking): because unique_args get merged into the raw event
# record, we can supply the 'smtp-id' field for any events missing it.
self.smtpapi.setdefault('unique_args', {})['smtp-id'] = self.message_id
def make_message_id(self):
"""Returns a Message-ID that could be used for this payload
Tries to use the from_email's domain as the Message-ID's domain
"""
try:
_, domain = self.data["from"].split("@")
except (AttributeError, KeyError, TypeError, ValueError):
domain = None
return make_msgid(domain=domain)
self.message_id = str(uuid.uuid4())
self.smtpapi.setdefault('unique_args', {})["anymail_id"] = self.message_id
def build_merge_data(self):
"""Set smtpapi['sub'] and ['section']"""

3
anymail/webhooks/sendgrid.py
View File

@@ -72,7 +72,7 @@ class SendGridTrackingWebhookView(AnymailBaseWebhookView):
return AnymailTrackingEvent(
event_type=event_type,
timestamp=timestamp,
message_id=esp_event.get('smtp-id', None),
message_id=esp_event.get('anymail_id', esp_event.get('smtp-id')), # backwards compatibility
event_id=esp_event.get('sg_event_id', None),
recipient=esp_event.get('email', None),
reject_reason=reject_reason,
@@ -86,6 +86,7 @@ class SendGridTrackingWebhookView(AnymailBaseWebhookView):
# Known keys in SendGrid events (used to recover metadata above)
sendgrid_event_keys = {
'anymail_id',
'asm_group_id',
'attempt', # MTA deferred count
'category',

14
tests/test_sendgrid_backend.py
View File

@@ -55,10 +55,8 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
self.assertEqual(data['personalizations'], [{
'to': [{'email': "to@example.com"}],
}])
# make sure backend assigned a Message-ID for event tracking
self.assertRegex(data['headers']['Message-ID'], r'\<.+@sender\.example\.com\>') # id uses from_email's domain
# make sure we added the Message-ID to custom_args for event notification
self.assertEqual(data['headers']['Message-ID'], data['custom_args']['smtp-id'])
# make sure the backend assigned the anymail_id for event tracking and notification
self.assertUUIDIsValid(data['custom_args']['anymail_id'])
def test_name_addr(self):
"""Make sure RFC2822 name-addr format (with display-name) is allowed
@@ -119,9 +117,7 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
'Message-ID': "<mycustommsgid@sales.example.com>",
})
# make sure custom Message-ID also added to custom_args
self.assertEqual(data['custom_args'], {
'smtp-id': "<mycustommsgid@sales.example.com>",
})
self.assertUUIDIsValid(data['custom_args']['anymail_id'])
def test_html_message(self):
text_content = 'This is an important message.'
@@ -345,7 +341,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
self.message.metadata = {'user_id': "12345", 'items': 6, 'float': 98.6, 'long': longtype(123)}
self.message.send()
data = self.get_api_call_json()
data['custom_args'].pop('smtp-id', None) # remove Message-ID we added as tracking workaround
data['custom_args'].pop('anymail_id', None) # remove Message-ID we added as tracking workaround
self.assertEqual(data['custom_args'], {'user_id': "12345",
'items': "6", # int converted to a string,
'float': "98.6", # float converted to a string (watch binary rounding!)
@@ -579,7 +575,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
sent = msg.send()
self.assertEqual(sent, 1)
self.assertEqual(msg.anymail_status.status, {'queued'})
self.assertRegex(msg.anymail_status.message_id, r'\<.+@example\.com\>') # don't know exactly what it'll be
self.assertUUIDIsValid(msg.anymail_status.message_id) # don't know exactly what it'll be
self.assertEqual(msg.anymail_status.recipients['to1@example.com'].status, 'queued')
self.assertEqual(msg.anymail_status.recipients['to1@example.com'].message_id,
msg.anymail_status.message_id)

21
tests/test_sendgrid_v2_backend.py
View File

@@ -63,12 +63,9 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
self.assertEqual(data['text'], "Here is the message.")
self.assertEqual(data['from'], "from@sender.example.com")
self.assertEqual(data['to'], ["to@example.com"])
# make sure backend assigned a Message-ID for event tracking
email_headers = json.loads(data['headers'])
self.assertRegex(email_headers['Message-ID'], r'\<.+@sender\.example\.com\>') # id uses from_email's domain
# make sure we added the Message-ID to unique_args for event notification
# make sure the backend assigned the anymail_id to unique_args for event tracking and notification
smtpapi = self.get_smtpapi()
self.assertEqual(email_headers['Message-ID'], smtpapi['unique_args']['smtp-id'])
self.assertUUIDIsValid(smtpapi['unique_args']['anymail_id'])
@override_settings(ANYMAIL={'SENDGRID_USERNAME': 'sg_username', 'SENDGRID_PASSWORD': 'sg_password'})
def test_user_pass_auth(self):
@@ -129,10 +126,9 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
'Reply-To': 'another@example.com',
'X-MyHeader': 'my value',
})
# make sure custom Message-ID also added to unique_args
self.assertJSONEqual(data['x-smtpapi'], {
'unique_args': {'smtp-id': '<mycustommsgid@sales.example.com>'}
})
# make sure anymail_id also added to unique_args
smtpapi_json = json.loads(data['x-smtpapi'])
self.assertUUIDIsValid(smtpapi_json['unique_args']['anymail_id'])
def test_html_message(self):
text_content = 'This is an important message.'
@@ -293,8 +289,7 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
self.assertNotIn('ccname', data)
self.assertNotIn('bcc', data)
self.assertNotIn('bccname', data)
headers = json.loads(data['headers'])
self.assertNotIn('Reply-To', headers)
self.assertNotIn('headers', data)
# Test empty `to` -- but send requires at least one recipient somewhere (like cc)
self.message.to = []
@@ -354,7 +349,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
self.message.metadata = {'user_id': "12345", 'items': 6}
self.message.send()
smtpapi = self.get_smtpapi()
smtpapi['unique_args'].pop('smtp-id', None) # remove Message-ID we added as tracking workaround
smtpapi['unique_args'].pop('anymail_id', None) # remove Message-ID we added as tracking workaround
self.assertEqual(smtpapi['unique_args'], {'user_id': "12345", 'items': 6})
def test_send_at(self):
@@ -565,7 +560,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
sent = msg.send()
self.assertEqual(sent, 1)
self.assertEqual(msg.anymail_status.status, {'queued'})
self.assertRegex(msg.anymail_status.message_id, r'\<.+@example\.com\>') # don't know exactly what it'll be
self.assertUUIDIsValid(msg.anymail_status.message_id)
self.assertEqual(msg.anymail_status.recipients['to1@example.com'].status, 'queued')
self.assertEqual(msg.anymail_status.recipients['to1@example.com'].message_id,
msg.anymail_status.message_id)

32
tests/test_sendgrid_webhooks.py
View File

@@ -23,7 +23,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
raw_events = [{
"email": "recipient@example.com",
"timestamp": 1461095246,
"smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@example.com>",
"anymail_id": "3c2f4df8-c6dd-4cd2-9b91-6582b81a0349",
"sg_event_id": "ZyjAM5rnQmuI1KFInHQ3Nw",
"sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
"event": "processed",
@@ -41,7 +41,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertEqual(event.event_type, "queued")
self.assertEqual(event.timestamp, datetime(2016, 4, 19, 19, 47, 26, tzinfo=utc))
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<wrfRRvF7Q0GgwUo2CvDmEA@example.com>")
self.assertEqual(event.message_id, "3c2f4df8-c6dd-4cd2-9b91-6582b81a0349")
self.assertEqual(event.event_id, "ZyjAM5rnQmuI1KFInHQ3Nw")
self.assertEqual(event.recipient, "recipient@example.com")
self.assertEqual(event.tags, ["tag1", "tag2"])
@@ -57,7 +57,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
"event": "delivered",
"email": "recipient@example.com",
"timestamp": 1461095250,
"smtp-id": "<wrfRRvF7Q0GgwUo2CvDmEA@example.com>"
"anymail_id": "4ab185c2-0171-492f-9ce0-27de258efc99"
}]
response = self.client.post('/anymail/sendgrid/tracking/',
content_type='application/json', data=json.dumps(raw_events))
@@ -69,7 +69,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertEqual(event.event_type, "delivered")
self.assertEqual(event.timestamp, datetime(2016, 4, 19, 19, 47, 30, tzinfo=utc))
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<wrfRRvF7Q0GgwUo2CvDmEA@example.com>")
self.assertEqual(event.message_id, "4ab185c2-0171-492f-9ce0-27de258efc99")
self.assertEqual(event.event_id, "nOSv8m0eTQ-vxvwNwt3fZQ")
self.assertEqual(event.recipient, "recipient@example.com")
self.assertEqual(event.mta_response, "250 2.0.0 OK 1461095248 m143si2210036ioe.159 - gsmtp ")
@@ -79,7 +79,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
def test_dropped_invalid_event(self):
raw_events = [{
"email": "invalid@invalid",
"smtp-id": "<YZkwwo_vQUidhSh7sCzkvQ@example.com>",
"anymail_id": "c74002d9-7ccb-4f67-8b8c-766cec03c9a6",
"timestamp": 1461095250,
"sg_event_id": "3NPOePGOTkeM_U3fgWApfg",
"sg_message_id": "filter0093p1las1.9128.5717FB8127.0",
@@ -95,7 +95,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertIsInstance(event, AnymailTrackingEvent)
self.assertEqual(event.event_type, "rejected")
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<YZkwwo_vQUidhSh7sCzkvQ@example.com>")
self.assertEqual(event.message_id, "c74002d9-7ccb-4f67-8b8c-766cec03c9a6")
self.assertEqual(event.event_id, "3NPOePGOTkeM_U3fgWApfg")
self.assertEqual(event.recipient, "invalid@invalid")
self.assertEqual(event.reject_reason, "invalid")
@@ -104,7 +104,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
def test_dropped_unsubscribed_event(self):
raw_events = [{
"email": "unsubscribe@example.com",
"smtp-id": "<Kwx3gAIKQOG7Nd5XEO7guQ@example.com>",
"anymail_id": "a36ec0f9-aabe-45c7-9a84-3e17afb5cb65",
"timestamp": 1461095250,
"sg_event_id": "oxy9OLwMTAy5EsuZn1qhIg",
"sg_message_id": "filter0199p1las1.4745.5717FB6F5.0",
@@ -120,7 +120,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertIsInstance(event, AnymailTrackingEvent)
self.assertEqual(event.event_type, "rejected")
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<Kwx3gAIKQOG7Nd5XEO7guQ@example.com>")
self.assertEqual(event.message_id, "a36ec0f9-aabe-45c7-9a84-3e17afb5cb65")
self.assertEqual(event.event_id, "oxy9OLwMTAy5EsuZn1qhIg")
self.assertEqual(event.recipient, "unsubscribe@example.com")
self.assertEqual(event.reject_reason, "unsubscribed")
@@ -137,7 +137,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
"event": "bounce",
"email": "noreply@example.com",
"timestamp": 1461095250,
"smtp-id": "<Lli-03HcQ5-JLybO9fXsJg@example.com>",
"anymail_id": "de212213-bb66-4302-8f3f-20acdb7a104e",
"type": "bounce"
}]
response = self.client.post('/anymail/sendgrid/tracking/',
@@ -149,7 +149,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertIsInstance(event, AnymailTrackingEvent)
self.assertEqual(event.event_type, "bounced")
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<Lli-03HcQ5-JLybO9fXsJg@example.com>")
self.assertEqual(event.message_id, "de212213-bb66-4302-8f3f-20acdb7a104e")
self.assertEqual(event.event_id, "lC0Rc-FuQmKbnxCWxX1jRQ")
self.assertEqual(event.recipient, "noreply@example.com")
self.assertEqual(event.mta_response, "550 5.1.1 The email account that you tried to reach does not exist.")
@@ -163,7 +163,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
"email": "recipient@example.com",
"attempt": "1",
"timestamp": 1461200990,
"smtp-id": "<20160421010427.2847.6797@example.com>",
"anymail_id": "ccf83222-0d7e-4542-8beb-893122afa757",
}]
response = self.client.post('/anymail/sendgrid/tracking/',
content_type='application/json', data=json.dumps(raw_events))
@@ -174,7 +174,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertIsInstance(event, AnymailTrackingEvent)
self.assertEqual(event.event_type, "deferred")
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<20160421010427.2847.6797@example.com>")
self.assertEqual(event.message_id, "ccf83222-0d7e-4542-8beb-893122afa757")
self.assertEqual(event.event_id, "b_syL5UiTvWC_Ky5L6Bs5Q")
self.assertEqual(event.recipient, "recipient@example.com")
self.assertEqual(event.mta_response,
@@ -187,7 +187,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
"ip": "66.102.6.229",
"sg_event_id": "MjIwNDg5NTgtZGE3OC00NDI1LWFiMmMtMDUyZTU2ZmFkOTFm",
"sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
"smtp-id": "<20160421010427.2847.6797@example.com>",
"anymail_id": "44920b35-3e31-478b-bb67-b4f5e0c85ebc",
"useragent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0",
"event": "open"
}]
@@ -200,7 +200,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertIsInstance(event, AnymailTrackingEvent)
self.assertEqual(event.event_type, "opened")
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<20160421010427.2847.6797@example.com>")
self.assertEqual(event.message_id, "44920b35-3e31-478b-bb67-b4f5e0c85ebc")
self.assertEqual(event.event_id, "MjIwNDg5NTgtZGE3OC00NDI1LWFiMmMtMDUyZTU2ZmFkOTFm")
self.assertEqual(event.recipient, "recipient@example.com")
self.assertEqual(event.user_agent, "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0")
@@ -211,7 +211,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
"sg_event_id": "OTdlOGUzYjctYjc5Zi00OWE4LWE4YWUtNjIxNjk2ZTJlNGVi",
"sg_message_id": "_fjPjuJfRW-IPs5SuvYotg.filter0590p1mdw1.2098.57168CFC4B.0",
"useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36",
"smtp-id": "<20160421010427.2847.6797@example.com>",
"anymail_id": "75de5af9-a090-4325-87f9-8c599ad66f60",
"event": "click",
"url_offset": {"index": 0, "type": "html"},
"email": "recipient@example.com",
@@ -227,7 +227,7 @@ class SendGridDeliveryTestCase(WebhookTestCase):
self.assertIsInstance(event, AnymailTrackingEvent)
self.assertEqual(event.event_type, "clicked")
self.assertEqual(event.esp_event, raw_events[0])
self.assertEqual(event.message_id, "<20160421010427.2847.6797@example.com>")
self.assertEqual(event.message_id, "75de5af9-a090-4325-87f9-8c599ad66f60")
self.assertEqual(event.event_id, "OTdlOGUzYjctYjc5Zi00OWE4LWE4YWUtNjIxNjk2ZTJlNGVi")
self.assertEqual(event.recipient, "recipient@example.com")
self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36")

9
tests/utils.py
View File

@@ -4,6 +4,7 @@ import logging
import os
import re
import sys
import uuid
import warnings
from base64 import b64decode
from contextlib import contextmanager
@@ -165,6 +166,14 @@ class AnymailTestMixin:
second = rfc822_unfold(second)
self.assertEqual(first, second, msg)
def assertUUIDIsValid(self, uuid_str, version=4):
"""Assert the uuid_str evaluates to a valid UUID"""
try:
uuid.UUID(uuid_str, version=version)
except (ValueError, AttributeError, TypeError):
return False
return True
# Backported from Python 3.4
class _AssertLogsContext(object):