From 9e7814ad65a419ff82f11bece1c8b40cab1e120f Mon Sep 17 00:00:00 2001 From: medmunds Date: Tue, 14 Aug 2018 11:53:30 -0700 Subject: [PATCH] Mailgun: Support new (non-legacy) webhooks Extend existing Mailgun tracking webhook handler to support both original (legacy) and new (June, 2018) Mailgun webhooks. Closes #117 --- CHANGELOG.rst | 3 + anymail/webhooks/mailgun.py | 166 +++++++++++--- docs/esps/mailgun.rst | 76 +++++-- tests/test_mailgun_inbound.py | 8 +- tests/test_mailgun_webhooks.py | 386 ++++++++++++++++++++++++++++++--- 5 files changed, 563 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 011af94..49eec25 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -43,6 +43,9 @@ Breaking changes Features ~~~~~~~~ +* **Mailgun:** Add support for new Mailgun webhooks. (Mailgun's original "legacy + webhook" format is also still supported. See + `docs `__.) * **Mailgun:** Document how to use new European region. (This works in earlier Anymail versions, too.) * **Postmark:** Add support for Anymail's normalized `metadata` in sending diff --git a/anymail/webhooks/mailgun.py b/anymail/webhooks/mailgun.py index 7515119..86ce825 100644 --- a/anymail/webhooks/mailgun.py +++ b/anymail/webhooks/mailgun.py @@ -7,10 +7,10 @@ from django.utils.crypto import constant_time_compare from django.utils.timezone import utc from .base import AnymailBaseWebhookView -from ..exceptions import AnymailWebhookValidationFailure +from ..exceptions import AnymailWebhookValidationFailure, AnymailInvalidAddress from ..inbound import AnymailInboundMessage from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason -from ..utils import get_anymail_setting, combine, querydict_getfirst +from ..utils import get_anymail_setting, combine, querydict_getfirst, parse_single_address class MailgunBaseWebhookView(AnymailBaseWebhookView): @@ -29,14 +29,29 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView): def validate_request(self, request): super(MailgunBaseWebhookView, self).validate_request(request) # first check basic auth if enabled - try: - # Must use the *last* value of these fields if there are conflicting merged user-variables. - # (Fortunately, Django QueryDict is specced to return the last value.) - token = request.POST['token'] - timestamp = request.POST['timestamp'] - signature = str(request.POST['signature']) # force to same type as hexdigest() (for python2) - except KeyError: - raise AnymailWebhookValidationFailure("Mailgun webhook called without required security fields") + if request.content_type == "application/json": + # New-style webhook: json payload with separate signature block + try: + event = json.loads(request.body.decode('utf-8')) + signature_block = event['signature'] + token = signature_block['token'] + timestamp = signature_block['timestamp'] + signature = signature_block['signature'] + except (KeyError, ValueError, UnicodeDecodeError) as err: + raise AnymailWebhookValidationFailure( + "Mailgun webhook called with invalid payload format", + raised_from=err) + else: + # Legacy webhook: signature fields are interspersed with other POST data + try: + # Must use the *last* value of these fields if there are conflicting merged user-variables. + # (Fortunately, Django QueryDict is specced to return the last value.) + token = request.POST['token'] + timestamp = request.POST['timestamp'] + signature = str(request.POST['signature']) # force to same type as hexdigest() (for python2) + except KeyError: + raise AnymailWebhookValidationFailure("Mailgun webhook called without required security fields") + expected_signature = hmac.new(key=self.api_key, msg='{}{}'.format(timestamp, token).encode('ascii'), digestmod=hashlib.sha256).hexdigest() if not constant_time_compare(signature, expected_signature): @@ -48,7 +63,109 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): signal = tracking + def parse_events(self, request): + if request.content_type == "application/json": + esp_event = json.loads(request.body.decode('utf-8')) + return [self.esp_to_anymail_event(esp_event)] + else: + return [self.mailgun_legacy_to_anymail_event(request.POST)] + event_types = { + # Map Mailgun event: Anymail normalized type + 'accepted': EventType.QUEUED, # not delivered to webhooks (8/2018) + 'rejected': EventType.REJECTED, + 'delivered': EventType.DELIVERED, + 'failed': EventType.BOUNCED, + 'opened': EventType.OPENED, + 'clicked': EventType.CLICKED, + 'unsubscribed': EventType.UNSUBSCRIBED, + 'complained': EventType.COMPLAINED, + } + + reject_reasons = { + # Map Mailgun event_data.reason: Anymail normalized RejectReason + # (these appear in webhook doc examples, but aren't actually documented anywhere) + "bounce": RejectReason.BOUNCED, + "suppress-bounce": RejectReason.BOUNCED, + "generic": RejectReason.BOUNCED, # ??? appears to be used for any temporary failure? + } + + def esp_to_anymail_event(self, esp_event): + event_data = esp_event.get('event-data', {}) + + try: + event_type = self.event_types[event_data['event']] + except KeyError: + event_type = EventType.UNKNOWN + + # Use signature.token for event_id, rather than event_data.id, + # because the latter is only "guaranteed to be unique within a day". + event_id = esp_event.get('signature', {}).get('token') + + recipient = event_data.get('recipient') + + try: + timestamp = datetime.fromtimestamp(float(event_data['timestamp']), tz=utc) + except KeyError: + timestamp = None + + try: + message_id = event_data['message']['headers']['message-id'] + except KeyError: + message_id = None + if message_id and not message_id.startswith('<'): + message_id = "<{}>".format(message_id) + + metadata = event_data.get('user-variables', {}) + tags = event_data.get('tags', []) + + try: + delivery_status = event_data['delivery-status'] + except KeyError: + description = None + mta_response = None + else: + description = delivery_status.get('description') + mta_response = delivery_status.get('message') + + if 'reason' in event_data: + reject_reason = self.reject_reasons.get(event_data['reason'], RejectReason.OTHER) + else: + reject_reason = None + + if event_type == EventType.REJECTED: + # This event has a somewhat different structure than the others... + description = description or event_data.get("reject", {}).get("reason") + reject_reason = reject_reason or RejectReason.OTHER + if not recipient: + try: + to_email = parse_single_address( + event_data["message"]["headers"]["to"]) + except (AnymailInvalidAddress, KeyError): + pass + else: + recipient = to_email.addr_spec + + return AnymailTrackingEvent( + event_type=event_type, + timestamp=timestamp, + message_id=message_id, + event_id=event_id, + recipient=recipient, + reject_reason=reject_reason, + description=description, + mta_response=mta_response, + tags=tags, + metadata=metadata, + click_url=event_data.get('url'), + user_agent=event_data.get('client-info', {}).get('user-agent'), + esp_event=esp_event, + ) + + # Legacy event handling + # (Prior to 2018-06-29, these were the only Mailgun events.) + + legacy_event_types = { # Map Mailgun event: Anymail normalized type 'delivered': EventType.DELIVERED, 'dropped': EventType.REJECTED, @@ -60,7 +177,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): # Mailgun does not send events corresponding to QUEUED or DEFERRED } - reject_reasons = { + legacy_reject_reasons = { # Map Mailgun (SMTP) error codes to Anymail normalized reject_reason. # By default, we will treat anything 400-599 as REJECT_BOUNCED # so only exceptions are listed here. @@ -71,10 +188,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): 607: RejectReason.SPAM, # previous spam complaint } - def parse_events(self, request): - return [self.esp_to_anymail_event(request.POST)] - - def esp_to_anymail_event(self, esp_event): + def mailgun_legacy_to_anymail_event(self, esp_event): # esp_event is a Django QueryDict (from request.POST), # which has multi-valued fields, but is *not* case-insensitive. # Because of the way Mailgun merges user-variables into the event, @@ -82,7 +196,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): # to avoid potential conflicting user-data. esp_event.getfirst = querydict_getfirst.__get__(esp_event) - event_type = self.event_types.get(esp_event.getfirst('event'), EventType.UNKNOWN) + event_type = self.legacy_event_types.get(esp_event.getfirst('event'), EventType.UNKNOWN) timestamp = datetime.fromtimestamp(int(esp_event['timestamp']), tz=utc) # use *last* value of timestamp # Message-Id is not documented for every event, but seems to always be included. # (It's sometimes spelled as 'message-id', lowercase, and missing the .) @@ -107,12 +221,12 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): else: reject_reason = RejectReason.BOUNCED if status_class in ("4", "5") else RejectReason.OTHER else: - reject_reason = self.reject_reasons.get( + reject_reason = self.legacy_reject_reasons.get( mta_status, RejectReason.BOUNCED if 400 <= mta_status < 600 else RejectReason.OTHER) - metadata = self._extract_metadata(esp_event) + metadata = self._extract_legacy_metadata(esp_event) # tags are supposed to be in 'tag' fields, but are sometimes in undocumented X-Mailgun-Tag tags = esp_event.getlist('tag', None) or esp_event.getlist('X-Mailgun-Tag', []) @@ -133,7 +247,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): esp_event=esp_event, ) - def _extract_metadata(self, esp_event): + def _extract_legacy_metadata(self, esp_event): # Mailgun merges user-variables into the POST fields. If you know which user variable # you want to retrieve--and it doesn't conflict with a Mailgun event field--that's fine. # But if you want to extract all user-variables (like we do), it's more complicated... @@ -149,10 +263,10 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): # Each X-Mailgun-Variables value is JSON. Parse and merge them all into single dict: metadata = combine(*[json.loads(value) for value in variables]) - elif event_type in self._known_event_fields: + elif event_type in self._known_legacy_event_fields: # For other events, we must extract from the POST fields, ignoring known Mailgun # event parameters, and treating all other values as user-variables. - known_fields = self._known_event_fields[event_type] + known_fields = self._known_legacy_event_fields[event_type] for field, values in esp_event.lists(): if field not in known_fields: # Unknown fields are assumed to be user-variables. (There should really only be @@ -177,7 +291,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): return metadata - _common_event_fields = { + _common_legacy_event_fields = { # These fields are documented to appear in all Mailgun opened, clicked and unsubscribed events: 'event', 'recipient', 'domain', 'ip', 'country', 'region', 'city', 'user-agent', 'device-type', 'client-type', 'client-name', 'client-os', 'campaign-id', 'campaign-name', 'tag', 'mailing-list', @@ -185,13 +299,13 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView): # Undocumented, but observed in actual events: 'body-plain', 'h', 'message-id', } - _known_event_fields = { + _known_legacy_event_fields = { # For all Mailgun event types that *don't* include message-headers, # map Mailgun (not normalized) event type to set of expected event fields. # Used for metadata extraction. - 'clicked': _common_event_fields | {'url'}, - 'opened': _common_event_fields, - 'unsubscribed': _common_event_fields, + 'clicked': _common_legacy_event_fields | {'url'}, + 'opened': _common_legacy_event_fields, + 'unsubscribed': _common_legacy_event_fields, } diff --git a/docs/esps/mailgun.rst b/docs/esps/mailgun.rst index db53728..b3fb727 100644 --- a/docs/esps/mailgun.rst +++ b/docs/esps/mailgun.rst @@ -147,14 +147,6 @@ values directly to Mailgun. You can use any of the (non-file) parameters listed Limitations and quirks ---------------------- -**Metadata keys and tracking webhooks** - Because of the way Mailgun supplies custom data (user-variables) to webhooks, - there are a few metadata keys that Anymail cannot reliably retrieve in some - tracking events. You should avoid using "body-plain", "h", "message-headers", - "message-id" or "tag" as :attr:`~anymail.message.AnymailMessage.metadata` keys - if you need to access that metadata from an opened, clicked, or unsubscribed - :ref:`tracking event ` handler. - **Envelope sender uses only domain** Anymail's :attr:`~anymail.message.AnymailMessage.envelope_sender` is used to select your Mailgun :ref:`sender domain `. For @@ -212,31 +204,73 @@ See the `Mailgun batch sending`_ docs for more information. Status tracking webhooks ------------------------ +.. versionchanged:: 4.0 + + Added support for Mailgun's June, 2018 (non-"legacy") webhook format. + If you are using Anymail's normalized :ref:`status tracking `, enter -the url in your `Mailgun dashboard`_ on the "Webhooks" tab. Mailgun allows you to enter -a different URL for each event type: just enter this same Anymail tracking URL -for all events you want to receive: +the url in the `Mailgun webhooks dashboard`_. (Be sure to select the correct sending +domain---Mailgun's sandbox and production domains have separate webhook settings.) + +Mailgun allows you to enter a different URL for each event type: just enter this same +Anymail tracking URL for all events you want to receive: :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/mailgun/tracking/` * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret * *yoursite.example.com* is your Django site -If you use multiple Mailgun sending domains, you'll need to enter the webhook -URLs for each of them, using the selector on the left side of Mailgun's dashboard. - Mailgun implements a limited form of webhook signing, and Anymail will verify these signatures (based on your :setting:`MAILGUN_API_KEY ` -Anymail setting). +Anymail setting). By default, Mailgun's webhook signature provides similar security +to Anymail's shared webhook secret, so it's acceptable to omit the +:setting:`ANYMAIL_WEBHOOK_SECRET` setting (and "{random}:{random}@" portion of the +webhook url) with Mailgun webhooks. Mailgun will report these Anymail :attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s: delivered, rejected, bounced, complained, unsubscribed, opened, clicked. The event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` field will be -a Django :class:`~django.http.QueryDict` object of `Mailgun event fields`_. +the parsed `Mailgun webhook payload`_ as a Python `dict` with ``"signature"`` and +``"event-data"`` keys. -.. _Mailgun dashboard: https://mailgun.com/app/dashboard -.. _Mailgun event fields: https://documentation.mailgun.com/user_manual.html#webhooks +Anymail uses Mailgun's webhook `token` as its normalized +:attr:`~anymail.signals.AnymailTrackingEvent.event_id`, rather than Mailgun's +event-data `id` (which is only guaranteed to be unique during a single day). +If you need the event-data id, it can be accessed in your webhook handler as +``event.esp_event["event-data"]["id"]``. (This can be helpful for working with +Mailgun's other event APIs.) + +.. note:: **Mailgun legacy webhooks** + + In late June, 2018, Mailgun introduced a new set of webhooks with an improved + payload design, and at the same time renamed their original webhooks to "Legacy + Webhooks." + + Anymail v4.0 and later supports both new and legacy Mailgun webhooks, and the same + Anymail webhook url works as either. Earlier Anymail versions can only be used + as legacy webhook urls. + + The new (non-legacy) webhooks are preferred, particularly with Anymail's + :attr:`~anymail.message.AnymailMessage.metadata` and + :attr:`~anymail.message.AnymailMessage.tags` features. But if you have already + configured the legacy webhooks, there is no need to change. + + If you are using Mailgun's legacy webhooks: + + * The :attr:`event.esp_event ` field + will be a Django :class:`~django.http.QueryDict` of Mailgun event fields (the + raw POST data provided by legacy webhooks). + + * You should avoid using "body-plain," "h," "message-headers," "message-id" or "tag" + as :attr:`~anymail.message.AnymailMessage.metadata` keys. A design limitation in + Mailgun's legacy webhooks prevents Anymail from reliably retrieving this metadata + from opened, clicked, and unsubscribed events. (This is not an issue with the + newer, non-legacy webhooks.) + + +.. _Mailgun webhooks dashboard: https://mailgun.com/app/webhooks +.. _Mailgun webhook payload: https://documentation.mailgun.com/en/latest/user_manual.html#webhooks .. _mailgun-inbound: @@ -247,7 +281,7 @@ Inbound webhook If you want to receive email from Mailgun through Anymail's normalized :ref:`inbound ` handling, follow Mailgun's `Receiving, Storing and Fowarding Messages`_ guide to set up an inbound route that forwards to Anymail's inbound webhook. (You can configure routes -using Mailgun's API, or simply using the "Routes" tab in your `Mailgun dashboard`_.) +using Mailgun's API, or simply using the `Mailgun routes dashboard`_.) The *action* for your route will be either: @@ -266,7 +300,9 @@ received email (including complex forms like multi-message mailing list digests) If you want to use Anymail's normalized :attr:`~anymail.inbound.AnymailInboundMessage.spam_detected` and :attr:`~anymail.inbound.AnymailInboundMessage.spam_score` attributes, you'll need to set your Mailgun domain's inbound spam filter to "Deliver spam, but add X-Mailgun-SFlag and X-Mailgun-SScore headers" -(in the `Mailgun dashboard`_ on the "Domains" tab). +(in the `Mailgun domains dashboard`_). .. _Receiving, Storing and Fowarding Messages: https://documentation.mailgun.com/en/latest/user_manual.html#receiving-forwarding-and-storing-messages +.. _Mailgun routes dashboard: https://app.mailgun.com/app/routes +.. _Mailgun domains dashboard: https://app.mailgun.com/app/domains diff --git a/tests/test_mailgun_inbound.py b/tests/test_mailgun_inbound.py index 192e6ee..5540a06 100644 --- a/tests/test_mailgun_inbound.py +++ b/tests/test_mailgun_inbound.py @@ -11,7 +11,7 @@ from anymail.inbound import AnymailInboundMessage from anymail.signals import AnymailInboundEvent from anymail.webhooks.mailgun import MailgunInboundWebhookView -from .test_mailgun_webhooks import TEST_API_KEY, mailgun_sign, querydict_to_postdict +from .test_mailgun_webhooks import TEST_API_KEY, mailgun_sign_legacy_payload, querydict_to_postdict from .utils import sample_image_content, sample_email_content from .webhook_cases import WebhookTestCase @@ -19,7 +19,7 @@ from .webhook_cases import WebhookTestCase @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY) class MailgunInboundTestCase(WebhookTestCase): def test_inbound_basics(self): - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'token': '06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0', 'timestamp': '1461261330', 'recipient': 'test@inbound.example.com', @@ -101,7 +101,7 @@ class MailgunInboundTestCase(WebhookTestCase): email_content = sample_email_content() att3 = six.BytesIO(email_content) att3.content_type = 'message/rfc822; charset="us-ascii"' - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'message-headers': '[]', 'attachment-count': '3', 'content-id-map': """{"": "attachment-2"}""", @@ -133,7 +133,7 @@ class MailgunInboundTestCase(WebhookTestCase): def test_inbound_mime(self): # Mailgun provides the full, raw MIME message if the webhook url ends in 'mime' - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'token': '06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0', 'timestamp': '1461261330', 'recipient': 'test@inbound.example.com', diff --git a/tests/test_mailgun_webhooks.py b/tests/test_mailgun_webhooks.py index 00977fb..828ff98 100644 --- a/tests/test_mailgun_webhooks.py +++ b/tests/test_mailgun_webhooks.py @@ -16,14 +16,33 @@ from .webhook_cases import WebhookTestCase, WebhookBasicAuthTestsMixin TEST_API_KEY = 'TEST_API_KEY' -def mailgun_sign(data, api_key=TEST_API_KEY): +def mailgun_signature(timestamp, token, api_key): + """Generates a Mailgun webhook signature""" + # https://documentation.mailgun.com/en/latest/user_manual.html#securing-webhooks + return hmac.new( + key=api_key.encode('ascii'), + msg='{timestamp}{token}'.format(timestamp=timestamp, token=token).encode('ascii'), + digestmod=hashlib.sha256).hexdigest() + + +def mailgun_sign_payload(data, api_key=TEST_API_KEY): + """Add or complete Mailgun webhook signature block in data dict""" + # Modifies the dict in place + event_data = data.get('event-data', {}) + signature = data.setdefault('signature', {}) + token = signature.setdefault('token', '1234567890abcdef1234567890abcdef') + timestamp = signature.setdefault('timestamp', + str(int(float(event_data.get('timestamp', '1234567890.123'))))) + signature['signature'] = mailgun_signature(timestamp, token, api_key=api_key) + return data + + +def mailgun_sign_legacy_payload(data, api_key=TEST_API_KEY): """Add a Mailgun webhook signature to data dict""" # Modifies the dict in place data.setdefault('timestamp', '1234567890') data.setdefault('token', '1234567890abcdef1234567890abcdef') - data['signature'] = hmac.new(key=api_key.encode('ascii'), - msg='{timestamp}{token}'.format(**data).encode('ascii'), - digestmod=hashlib.sha256).hexdigest() + data['signature'] = mailgun_signature(data['timestamp'], data['token'], api_key=api_key) return data @@ -42,8 +61,8 @@ def querydict_to_postdict(qd): class MailgunWebhookSettingsTestCase(WebhookTestCase): def test_requires_api_key(self): with self.assertRaises(ImproperlyConfigured): - self.client.post('/anymail/mailgun/tracking/', - data=mailgun_sign({'event': 'delivered'})) + self.client.post('/anymail/mailgun/tracking/', content_type="application/json", + data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}}))) @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY) @@ -51,32 +70,347 @@ class MailgunWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin should_warn_if_no_auth = False # because we check webhook signature def call_webhook(self): - return self.client.post('/anymail/mailgun/tracking/', - data=mailgun_sign({'event': 'delivered'})) + return self.client.post('/anymail/mailgun/tracking/', content_type="application/json", + data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}}))) # Additional tests are in WebhookBasicAuthTestsMixin def test_verifies_correct_signature(self): - response = self.client.post('/anymail/mailgun/tracking/', - data=mailgun_sign({'event': 'delivered'})) + response = self.client.post('/anymail/mailgun/tracking/', content_type="application/json", + data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}}))) self.assertEqual(response.status_code, 200) def test_verifies_missing_signature(self): - response = self.client.post('/anymail/mailgun/tracking/', - data={'event': 'delivered'}) + response = self.client.post('/anymail/mailgun/tracking/', content_type="application/json", + data=json.dumps({'event-data': {'event': 'delivered'}})) self.assertEqual(response.status_code, 400) def test_verifies_bad_signature(self): - data = mailgun_sign({'event': 'delivered'}, api_key="wrong API key") - response = self.client.post('/anymail/mailgun/tracking/', data=data) + data = mailgun_sign_payload({'event-data': {'event': 'delivered'}}, + api_key="wrong API key") + response = self.client.post('/anymail/mailgun/tracking/', content_type="application/json", + data=json.dumps(data)) self.assertEqual(response.status_code, 400) @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY) -class MailgunDeliveryTestCase(WebhookTestCase): +class MailgunTestCase(WebhookTestCase): + # Tests for Mailgun's new webhooks (announced 2018-06-29) def test_delivered_event(self): - raw_event = mailgun_sign({ + # This is an actual, complete (sanitized) "delivered" event as received from Mailgun. + # (For brevity, later tests omit several payload fields that aren't used by Anymail.) + raw_event = mailgun_sign_payload({ + "signature": { + "timestamp": "1534108637", + "token": "651869375b9df3c98fc15c4889b102119add1235c38fc92824", + "signature": "...", + }, + "event-data": { + "tags": [], + "timestamp": 1534108637.153125, + "storage": { + "url": "https://sw.api.mailgun.net/v3/domains/example.org/messages/eyJwI...", + "key": "eyJwI...", + }, + "recipient-domain": "example.com", + "id": "hTWCTD81RtiDN-...", + "campaigns": [], + "user-variables": {}, + "flags": { + "is-routed": False, + "is-authenticated": True, + "is-system-test": False, + "is-test-mode": False, + }, + "log-level": "info", + "envelope": { + "sending-ip": "333.123.123.200", + "sender": "test@example.org", + "transport": "smtp", + "targets": "recipient@example.com", + }, + "message": { + "headers": { + "to": "recipient@example.com", + "message-id": "20180812211713.1.DF5966851B4BAA99@example.org", + "from": "test@example.org", + "subject": "Testing", + }, + "attachments": [], + "size": 809, + }, + "recipient": "recipient@example.com", + "event": "delivered", + "delivery-status": { + "tls": True, + "mx-host": "smtp-in.example.com", + "attempt-no": 1, + "description": "", + "session-seconds": 3.5700838565826416, + "utf8": True, + "code": 250, + "message": "OK", + "certificate-verified": True, + }, + }, + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "delivered") + self.assertEqual(event.timestamp, datetime(2018, 8, 12, 21, 17, 17, microsecond=153125, tzinfo=utc)) + self.assertEqual(event.message_id, "<20180812211713.1.DF5966851B4BAA99@example.org>") + # Note that Anymail uses the "token" as its normalized event_id: + self.assertEqual(event.event_id, "651869375b9df3c98fc15c4889b102119add1235c38fc92824") + # ... if you want the Mailgun "event id", that's available through the raw esp_event: + self.assertEqual(event.esp_event["event-data"]["id"], "hTWCTD81RtiDN-...") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.esp_event, raw_event) + self.assertEqual(event.tags, []) + self.assertEqual(event.metadata, {}) + + def test_failed_permanent_event(self): + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "failed", + "severity": "permanent", + "reason": "bounce", + "recipient": "invalid@example.com", + "timestamp": 1534110422.389832, + "log-level": "error", + "message": { + "headers": { + "to": "invalid@example.com", + "message-id": "20180812214658.1.0DF563D0B3597700@example.org", + "from": "Test Sender ", + }, + }, + "delivery-status": { + "tls": True, + "mx-host": "aspmx.l.example.org", + "attempt-no": 1, + "description": "", + "session-seconds": 2.952177047729492, + "utf8": True, + "code": 550, + "message": "5.1.1 The email account that you tried to reach does not exist. Please try\n" + "5.1.1 double-checking the recipient's email address for typos", + "certificate-verified": True + } + }, + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.recipient, "invalid@example.com") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.description, "") + self.assertEqual(event.mta_response, + "5.1.1 The email account that you tried to reach does not exist. Please try\n" + "5.1.1 double-checking the recipient's email address for typos") + + def test_failed_temporary_event(self): + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "failed", + "severity": "temporary", + "reason": "generic", + "timestamp": 1534111899.659519, + "log-level": "warn", + "message": { + "headers": { + "to": "undeliverable@nomx.example.com", + "message-id": "20180812214638.1.4A7D468E9BC18C5D@example.org", + "from": "Test Sender ", + "subject": "Testing" + }, + }, + "recipient": "undeliverable@nomx.example.com", + "delivery-status": { + "attempt-no": 3, + "description": "No MX for nomx.example.com", + "session-seconds": 0.0, + "retry-seconds": 1800, + "code": 498, + "message": "No MX for nomx.example.com" + } + }, + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.recipient, "undeliverable@nomx.example.com") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.description, "No MX for nomx.example.com") + self.assertEqual(event.mta_response, "No MX for nomx.example.com") + + def test_rejected_event(self): + # (The "rejected" event is documented and appears in Mailgun dashboard logs, + # but it doesn't appear to be delivered through webhooks as of 8/2018.) + # Note that this payload lacks the recipient field present in all other events. + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "rejected", + "timestamp": 1529704976.104692, + "log-level": "warn", + "reject": { + "reason": "Sandbox subdomains are for test purposes only.", + "description": "", + }, + "message": { + "headers": { + "to": "Recipient Name ", + "message-id": "20180622220256.1.B31A451A2E5422BB@sandbox55887.mailgun.org", + "from": "test@sandbox55887.mailgun.org", + "subject": "Test Subject" + }, + }, + }, + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "rejected") + self.assertEqual(event.reject_reason, "other") + self.assertEqual(event.description, "Sandbox subdomains are for test purposes only.") + self.assertEqual(event.recipient, "recipient@example.org") + + def test_complained_event(self): + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "complained", + "id": "ncV2XwymRUKbPek_MIM-Gw", + "timestamp": 1377214260.049634, + "log-level": "warn", + "recipient": "recipient@example.com", + "message": { + "headers": { + "to": "foo@recipient.com", + "message-id": "20130718032413.263EE2E0926@example.org", + "from": "Sender Name ", + "subject": "We are not spammer", + }, + }, + }, + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "complained") + self.assertEqual(event.recipient, "recipient@example.com") + + def test_unsubscribed_event(self): + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "unsubscribed", + "id": "W3X4JOhFT-OZidZGKKr9iA", + "timestamp": 1377213791.421473, + "log-level": "info", + "recipient": "recipient@example.com", + "message": { + "headers": { + "message-id": "20130822232216.13966.79700@samples.mailgun.org" + } + }, + }, + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "unsubscribed") + self.assertEqual(event.recipient, "recipient@example.com") + + def test_opened_event(self): + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "opened", + "timestamp": 1534109600.089676, + "recipient": "recipient@example.com", + "tags": ["welcome", "variation-A"], + "user-variables": { + "cohort": "2018-08-B", + "user_id": "123456" + }, + "message": { + # Mailgun *only* includes the message-id header for opened, clicked events... + "headers": { + "message-id": "20180812213139.1.BC6694A917BB7E6A@example.org" + } + }, + "geolocation": { + "country": "US", + "region": "CA", + "city": "San Francisco" + }, + "ip": "888.222.444.111", + "client-info": { + "client-type": "browser", + "client-os": "OS X", + "device-type": "desktop", + "client-name": "Chrome", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6)..." + }, + } + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "opened") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.tags, ["welcome", "variation-A"]) + self.assertEqual(event.metadata, {"cohort": "2018-08-B", "user_id": "123456"}) + + def test_clicked_event(self): + raw_event = mailgun_sign_payload({ + "event-data": { + "event": "clicked", + "timestamp": 1534109600.089676, + "recipient": "recipient@example.com", + "url": "https://example.com/test" + } + }) + response = self.client.post('/anymail/mailgun/tracking/', + data=json.dumps(raw_event), content_type='application/json') + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView, + event=ANY, esp_name='Mailgun') + event = kwargs['event'] + self.assertEqual(event.event_type, "clicked") + self.assertEqual(event.click_url, "https://example.com/test") + + +@override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY) +class MailgunLegacyTestCase(WebhookTestCase): + # Tests for Mailgun's "legacy" webhooks + # (which were the only webhooks available prior to Anymail 4.0) + + def test_delivered_event(self): + raw_event = mailgun_sign_legacy_payload({ 'domain': 'example.com', 'message-headers': json.dumps([ ["Sender", "from=example.com"], @@ -113,7 +447,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): self.assertEqual(event.metadata, {}) def test_dropped_bounce(self): - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'code': '605', 'domain': 'example.com', 'description': 'Not delivering to previously bounced address', @@ -152,7 +486,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): self.assertEqual(querydict_to_postdict(event.esp_event), raw_event) def test_dropped_spam(self): - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'code': '607', 'description': 'Not delivering to a user who marked your messages as spam', 'reason': 'hardfail', @@ -170,7 +504,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): self.assertEqual(event.description, 'Not delivering to a user who marked your messages as spam') def test_dropped_timed_out(self): - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'code': '499', 'description': 'Unable to connect to MX servers: [example.com]', 'reason': 'old', @@ -188,7 +522,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): self.assertEqual(event.description, 'Unable to connect to MX servers: [example.com]') def test_invalid_mailbox(self): - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'code': '550', 'error': "550 5.1.1 The email account that you tried to reach does not exist. Please try " " 5.1.1 double-checking the recipient's email address for typos or " @@ -209,7 +543,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): def test_alt_smtp_code(self): # In some cases, Mailgun uses RFC-3463 extended SMTP status codes (x.y.z, rather than nnn). # See issue #62. - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'code': '5.1.1', 'error': 'smtp;550 5.1.1 RESOLVER.ADR.RecipNotFound; not found', 'event': 'bounced', @@ -228,7 +562,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): def test_metadata_message_headers(self): # Metadata fields are interspersed with other data, but also in message-headers # for delivered, bounced and dropped events - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'event': 'delivered', 'message-headers': json.dumps([ ["X-Mailgun-Variables", "{\"custom1\": \"value1\", \"custom2\": \"{\\\"key\\\":\\\"value\\\"}\"}"], @@ -244,7 +578,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): def test_metadata_post_fields(self): # Metadata fields are only interspersed with other event params # for opened, clicked, unsubscribed events - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'event': 'clicked', 'custom1': 'value1', 'custom2': '{"key":"value"}', # you can store JSON, but you'll need to unpack it yourself @@ -267,7 +601,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): "ordinary field": "ordinary metadata value", } - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'event': 'clicked', 'recipient': 'actual-recipient@example.com', 'token': 'actual-event-token', @@ -305,7 +639,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): def test_tags(self): # Most events include multiple 'tag' fields for message's tags - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'tag': ['tag1', 'tag2'], # Django TestClient encodes list as multiple field values 'event': 'opened', }) @@ -316,7 +650,7 @@ class MailgunDeliveryTestCase(WebhookTestCase): def test_x_tags(self): # Delivery events don't include 'tag', but do include 'X-Mailgun-Tag' fields - raw_event = mailgun_sign({ + raw_event = mailgun_sign_legacy_payload({ 'X-Mailgun-Tag': ['tag1', 'tag2'], 'event': 'delivered', })