Mailgun: Support new (non-legacy) webhooks

Extend existing Mailgun tracking webhook handler to support both
original (legacy) and new (June, 2018) Mailgun webhooks.

Closes #117

CHANGELOG.rst

@@ -43,6 +43,9 @@ Breaking changes
 Features
 ~~~~~~~~
 
+* **Mailgun:** Add support for new Mailgun webhooks. (Mailgun's original "legacy
+  webhook" format is also still supported. See
+  `docs <https://anymail.readthedocs.io/en/stable/esps/mailgun/#mailgun-webhooks>`__.)
 * **Mailgun:** Document how to use new European region. (This works in earlier
   Anymail versions, too.)
 * **Postmark:** Add support for Anymail's normalized `metadata` in sending

anymail/webhooks/mailgun.py

@@ -7,10 +7,10 @@ from django.utils.crypto import constant_time_compare
 from django.utils.timezone import utc
 
 from .base import AnymailBaseWebhookView
-from ..exceptions import AnymailWebhookValidationFailure
+from ..exceptions import AnymailWebhookValidationFailure, AnymailInvalidAddress
 from ..inbound import AnymailInboundMessage
 from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
-from ..utils import get_anymail_setting, combine, querydict_getfirst
+from ..utils import get_anymail_setting, combine, querydict_getfirst, parse_single_address
 
 
 class MailgunBaseWebhookView(AnymailBaseWebhookView):
@@ -29,6 +29,20 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView):
 
     def validate_request(self, request):
         super(MailgunBaseWebhookView, self).validate_request(request)  # first check basic auth if enabled
+        if request.content_type == "application/json":
+            # New-style webhook: json payload with separate signature block
+            try:
+                event = json.loads(request.body.decode('utf-8'))
+                signature_block = event['signature']
+                token = signature_block['token']
+                timestamp = signature_block['timestamp']
+                signature = signature_block['signature']
+            except (KeyError, ValueError, UnicodeDecodeError) as err:
+                raise AnymailWebhookValidationFailure(
+                    "Mailgun webhook called with invalid payload format",
+                    raised_from=err)
+        else:
+            # Legacy webhook: signature fields are interspersed with other POST data
             try:
                 # Must use the *last* value of these fields if there are conflicting merged user-variables.
                 # (Fortunately, Django QueryDict is specced to return the last value.)
@@ -37,6 +51,7 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView):
                 signature = str(request.POST['signature'])  # force to same type as hexdigest() (for python2)
             except KeyError:
                 raise AnymailWebhookValidationFailure("Mailgun webhook called without required security fields")
+
         expected_signature = hmac.new(key=self.api_key, msg='{}{}'.format(timestamp, token).encode('ascii'),
                                       digestmod=hashlib.sha256).hexdigest()
         if not constant_time_compare(signature, expected_signature):
@@ -48,7 +63,109 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
 
     signal = tracking
 
+    def parse_events(self, request):
+        if request.content_type == "application/json":
+            esp_event = json.loads(request.body.decode('utf-8'))
+            return [self.esp_to_anymail_event(esp_event)]
+        else:
+            return [self.mailgun_legacy_to_anymail_event(request.POST)]
+
     event_types = {
+        # Map Mailgun event: Anymail normalized type
+        'accepted': EventType.QUEUED,  # not delivered to webhooks (8/2018)
+        'rejected': EventType.REJECTED,
+        'delivered': EventType.DELIVERED,
+        'failed': EventType.BOUNCED,
+        'opened': EventType.OPENED,
+        'clicked': EventType.CLICKED,
+        'unsubscribed': EventType.UNSUBSCRIBED,
+        'complained': EventType.COMPLAINED,
+    }
+
+    reject_reasons = {
+        # Map Mailgun event_data.reason: Anymail normalized RejectReason
+        # (these appear in webhook doc examples, but aren't actually documented anywhere)
+        "bounce": RejectReason.BOUNCED,
+        "suppress-bounce": RejectReason.BOUNCED,
+        "generic": RejectReason.BOUNCED,  # ??? appears to be used for any temporary failure?
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        event_data = esp_event.get('event-data', {})
+
+        try:
+            event_type = self.event_types[event_data['event']]
+        except KeyError:
+            event_type = EventType.UNKNOWN
+
+        # Use signature.token for event_id, rather than event_data.id,
+        # because the latter is only "guaranteed to be unique within a day".
+        event_id = esp_event.get('signature', {}).get('token')
+
+        recipient = event_data.get('recipient')
+
+        try:
+            timestamp = datetime.fromtimestamp(float(event_data['timestamp']), tz=utc)
+        except KeyError:
+            timestamp = None
+
+        try:
+            message_id = event_data['message']['headers']['message-id']
+        except KeyError:
+            message_id = None
+        if message_id and not message_id.startswith('<'):
+            message_id = "<{}>".format(message_id)
+
+        metadata = event_data.get('user-variables', {})
+        tags = event_data.get('tags', [])
+
+        try:
+            delivery_status = event_data['delivery-status']
+        except KeyError:
+            description = None
+            mta_response = None
+        else:
+            description = delivery_status.get('description')
+            mta_response = delivery_status.get('message')
+
+        if 'reason' in event_data:
+            reject_reason = self.reject_reasons.get(event_data['reason'], RejectReason.OTHER)
+        else:
+            reject_reason = None
+
+        if event_type == EventType.REJECTED:
+            # This event has a somewhat different structure than the others...
+            description = description or event_data.get("reject", {}).get("reason")
+            reject_reason = reject_reason or RejectReason.OTHER
+            if not recipient:
+                try:
+                    to_email = parse_single_address(
+                        event_data["message"]["headers"]["to"])
+                except (AnymailInvalidAddress, KeyError):
+                    pass
+                else:
+                    recipient = to_email.addr_spec
+
+        return AnymailTrackingEvent(
+            event_type=event_type,
+            timestamp=timestamp,
+            message_id=message_id,
+            event_id=event_id,
+            recipient=recipient,
+            reject_reason=reject_reason,
+            description=description,
+            mta_response=mta_response,
+            tags=tags,
+            metadata=metadata,
+            click_url=event_data.get('url'),
+            user_agent=event_data.get('client-info', {}).get('user-agent'),
+            esp_event=esp_event,
+        )
+
+    # Legacy event handling
+    # (Prior to 2018-06-29, these were the only Mailgun events.)
+
+    legacy_event_types = {
         # Map Mailgun event: Anymail normalized type
         'delivered': EventType.DELIVERED,
         'dropped': EventType.REJECTED,
@@ -60,7 +177,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
         # Mailgun does not send events corresponding to QUEUED or DEFERRED
     }
 
-    reject_reasons = {
+    legacy_reject_reasons = {
         # Map Mailgun (SMTP) error codes to Anymail normalized reject_reason.
         # By default, we will treat anything 400-599 as REJECT_BOUNCED
         # so only exceptions are listed here.
@@ -71,10 +188,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
         607: RejectReason.SPAM,  # previous spam complaint
     }
 
-    def parse_events(self, request):
+    def mailgun_legacy_to_anymail_event(self, esp_event):
-        return [self.esp_to_anymail_event(request.POST)]
-
-    def esp_to_anymail_event(self, esp_event):
         # esp_event is a Django QueryDict (from request.POST),
         # which has multi-valued fields, but is *not* case-insensitive.
         # Because of the way Mailgun merges user-variables into the event,
@@ -82,7 +196,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
         # to avoid potential conflicting user-data.
         esp_event.getfirst = querydict_getfirst.__get__(esp_event)
 
-        event_type = self.event_types.get(esp_event.getfirst('event'), EventType.UNKNOWN)
+        event_type = self.legacy_event_types.get(esp_event.getfirst('event'), EventType.UNKNOWN)
         timestamp = datetime.fromtimestamp(int(esp_event['timestamp']), tz=utc)  # use *last* value of timestamp
         # Message-Id is not documented for every event, but seems to always be included.
         # (It's sometimes spelled as 'message-id', lowercase, and missing the <angle-brackets>.)
@@ -107,12 +221,12 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
             else:
                 reject_reason = RejectReason.BOUNCED if status_class in ("4", "5") else RejectReason.OTHER
         else:
-            reject_reason = self.reject_reasons.get(
+            reject_reason = self.legacy_reject_reasons.get(
                 mta_status,
                 RejectReason.BOUNCED if 400 <= mta_status < 600
                 else RejectReason.OTHER)
 
-        metadata = self._extract_metadata(esp_event)
+        metadata = self._extract_legacy_metadata(esp_event)
 
         # tags are supposed to be in 'tag' fields, but are sometimes in undocumented X-Mailgun-Tag
         tags = esp_event.getlist('tag', None) or esp_event.getlist('X-Mailgun-Tag', [])
@@ -133,7 +247,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
             esp_event=esp_event,
         )
 
-    def _extract_metadata(self, esp_event):
+    def _extract_legacy_metadata(self, esp_event):
         # Mailgun merges user-variables into the POST fields. If you know which user variable
         # you want to retrieve--and it doesn't conflict with a Mailgun event field--that's fine.
         # But if you want to extract all user-variables (like we do), it's more complicated...
@@ -149,10 +263,10 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
                 # Each X-Mailgun-Variables value is JSON. Parse and merge them all into single dict:
                 metadata = combine(*[json.loads(value) for value in variables])
 
-        elif event_type in self._known_event_fields:
+        elif event_type in self._known_legacy_event_fields:
             # For other events, we must extract from the POST fields, ignoring known Mailgun
             # event parameters, and treating all other values as user-variables.
-            known_fields = self._known_event_fields[event_type]
+            known_fields = self._known_legacy_event_fields[event_type]
             for field, values in esp_event.lists():
                 if field not in known_fields:
                     # Unknown fields are assumed to be user-variables. (There should really only be
@@ -177,7 +291,7 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
 
         return metadata
 
-    _common_event_fields = {
+    _common_legacy_event_fields = {
         # These fields are documented to appear in all Mailgun opened, clicked and unsubscribed events:
         'event', 'recipient', 'domain', 'ip', 'country', 'region', 'city', 'user-agent', 'device-type',
         'client-type', 'client-name', 'client-os', 'campaign-id', 'campaign-name', 'tag', 'mailing-list',
@@ -185,13 +299,13 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
         # Undocumented, but observed in actual events:
         'body-plain', 'h', 'message-id',
     }
-    _known_event_fields = {
+    _known_legacy_event_fields = {
         # For all Mailgun event types that *don't* include message-headers,
         # map Mailgun (not normalized) event type to set of expected event fields.
         # Used for metadata extraction.
-        'clicked': _common_event_fields | {'url'},
+        'clicked': _common_legacy_event_fields | {'url'},
-        'opened': _common_event_fields,
+        'opened': _common_legacy_event_fields,
-        'unsubscribed': _common_event_fields,
+        'unsubscribed': _common_legacy_event_fields,
     }
 
 

docs/esps/mailgun.rst

@@ -147,14 +147,6 @@ values directly to Mailgun. You can use any of the (non-file) parameters listed
 Limitations and quirks
 ----------------------
 
-**Metadata keys and tracking webhooks**
-  Because of the way Mailgun supplies custom data (user-variables) to webhooks,
-  there are a few metadata keys that Anymail cannot reliably retrieve in some
-  tracking events. You should avoid using "body-plain", "h", "message-headers",
-  "message-id" or "tag" as :attr:`~anymail.message.AnymailMessage.metadata` keys
-  if you need to access that metadata from an opened, clicked, or unsubscribed
-  :ref:`tracking event <event-tracking>` handler.
-
 **Envelope sender uses only domain**
   Anymail's :attr:`~anymail.message.AnymailMessage.envelope_sender` is used to
   select your Mailgun :ref:`sender domain <mailgun-sender-domain>`. For
@@ -212,31 +204,73 @@ See the `Mailgun batch sending`_ docs for more information.
 Status tracking webhooks
 ------------------------
 
+.. versionchanged:: 4.0
+
+    Added support for Mailgun's June, 2018 (non-"legacy") webhook format.
+
 If you are using Anymail's normalized :ref:`status tracking <event-tracking>`, enter
-the url in your `Mailgun dashboard`_ on the "Webhooks" tab. Mailgun allows you to enter
+the url in the `Mailgun webhooks dashboard`_. (Be sure to select the correct sending
-a different URL for each event type: just enter this same Anymail tracking URL
+domain---Mailgun's sandbox and production domains have separate webhook settings.)
-for all events you want to receive:
+
+Mailgun allows you to enter a different URL for each event type: just enter this same
+Anymail tracking URL for all events you want to receive:
 
    :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/mailgun/tracking/`
 
      * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret
      * *yoursite.example.com* is your Django site
 
-If you use multiple Mailgun sending domains, you'll need to enter the webhook
-URLs for each of them, using the selector on the left side of Mailgun's dashboard.
-
 Mailgun implements a limited form of webhook signing, and Anymail will verify
 these signatures (based on your :setting:`MAILGUN_API_KEY <ANYMAIL_MAILGUN_API_KEY>`
-Anymail setting).
+Anymail setting). By default, Mailgun's webhook signature provides similar security
+to Anymail's shared webhook secret, so it's acceptable to omit the
+:setting:`ANYMAIL_WEBHOOK_SECRET` setting (and "{random}:{random}@" portion of the
+webhook url) with Mailgun webhooks.
 
 Mailgun will report these Anymail :attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s:
 delivered, rejected, bounced, complained, unsubscribed, opened, clicked.
 
 The event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` field will be
-a Django :class:`~django.http.QueryDict` object of `Mailgun event fields`_.
+the parsed `Mailgun webhook payload`_ as a Python `dict` with ``"signature"`` and
+``"event-data"`` keys.
 
-.. _Mailgun dashboard: https://mailgun.com/app/dashboard
+Anymail uses Mailgun's webhook `token` as its normalized
-.. _Mailgun event fields: https://documentation.mailgun.com/user_manual.html#webhooks
+:attr:`~anymail.signals.AnymailTrackingEvent.event_id`, rather than Mailgun's
+event-data `id` (which is only guaranteed to be unique during a single day).
+If you need the event-data id, it can be accessed in your webhook handler as
+``event.esp_event["event-data"]["id"]``. (This can be helpful for working with
+Mailgun's other event APIs.)
+
+.. note:: **Mailgun legacy webhooks**
+
+    In late June, 2018, Mailgun introduced a new set of webhooks with an improved
+    payload design, and at the same time renamed their original webhooks to "Legacy
+    Webhooks."
+
+    Anymail v4.0 and later supports both new and legacy Mailgun webhooks, and the same
+    Anymail webhook url works as either. Earlier Anymail versions can only be used
+    as legacy webhook urls.
+
+    The new (non-legacy) webhooks are preferred, particularly with Anymail's
+    :attr:`~anymail.message.AnymailMessage.metadata` and
+    :attr:`~anymail.message.AnymailMessage.tags` features. But if you have already
+    configured the legacy webhooks, there is no need to change.
+
+    If you are using Mailgun's legacy webhooks:
+
+    * The :attr:`event.esp_event <anymail.signals.AnymailTrackingEvent.esp_event>` field
+      will be a Django :class:`~django.http.QueryDict` of Mailgun event fields (the
+      raw POST data provided by legacy webhooks).
+
+    * You should avoid using "body-plain," "h," "message-headers," "message-id" or "tag"
+      as :attr:`~anymail.message.AnymailMessage.metadata` keys. A design limitation in
+      Mailgun's legacy webhooks prevents Anymail from reliably retrieving this metadata
+      from opened, clicked, and unsubscribed events. (This is not an issue with the
+      newer, non-legacy webhooks.)
+
+
+.. _Mailgun webhooks dashboard: https://mailgun.com/app/webhooks
+.. _Mailgun webhook payload: https://documentation.mailgun.com/en/latest/user_manual.html#webhooks
 
 
 .. _mailgun-inbound:
@@ -247,7 +281,7 @@ Inbound webhook
 If you want to receive email from Mailgun through Anymail's normalized :ref:`inbound <inbound>`
 handling, follow Mailgun's `Receiving, Storing and Fowarding Messages`_ guide to set up
 an inbound route that forwards to Anymail's inbound webhook. (You can configure routes
-using Mailgun's API, or simply using the "Routes" tab in your `Mailgun dashboard`_.)
+using Mailgun's API, or simply using the `Mailgun routes dashboard`_.)
 
 The *action* for your route will be either:
 
@@ -266,7 +300,9 @@ received email (including complex forms like multi-message mailing list digests)
 If you want to use Anymail's normalized :attr:`~anymail.inbound.AnymailInboundMessage.spam_detected` and
 :attr:`~anymail.inbound.AnymailInboundMessage.spam_score` attributes, you'll need to set your Mailgun
 domain's inbound spam filter to "Deliver spam, but add X-Mailgun-SFlag and X-Mailgun-SScore headers"
-(in the `Mailgun dashboard`_ on the "Domains" tab).
+(in the `Mailgun domains dashboard`_).
 
 .. _Receiving, Storing and Fowarding Messages:
    https://documentation.mailgun.com/en/latest/user_manual.html#receiving-forwarding-and-storing-messages
+.. _Mailgun routes dashboard: https://app.mailgun.com/app/routes
+.. _Mailgun domains dashboard: https://app.mailgun.com/app/domains

tests/test_mailgun_inbound.py

@@ -11,7 +11,7 @@ from anymail.inbound import AnymailInboundMessage
 from anymail.signals import AnymailInboundEvent
 from anymail.webhooks.mailgun import MailgunInboundWebhookView
 
-from .test_mailgun_webhooks import TEST_API_KEY, mailgun_sign, querydict_to_postdict
+from .test_mailgun_webhooks import TEST_API_KEY, mailgun_sign_legacy_payload, querydict_to_postdict
 from .utils import sample_image_content, sample_email_content
 from .webhook_cases import WebhookTestCase
 
@@ -19,7 +19,7 @@ from .webhook_cases import WebhookTestCase
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
 class MailgunInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'token': '06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0',
             'timestamp': '1461261330',
             'recipient': 'test@inbound.example.com',
@@ -101,7 +101,7 @@ class MailgunInboundTestCase(WebhookTestCase):
         email_content = sample_email_content()
         att3 = six.BytesIO(email_content)
         att3.content_type = 'message/rfc822; charset="us-ascii"'
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'message-headers': '[]',
             'attachment-count': '3',
             'content-id-map': """{"<abc123>": "attachment-2"}""",
@@ -133,7 +133,7 @@ class MailgunInboundTestCase(WebhookTestCase):
 
     def test_inbound_mime(self):
         # Mailgun provides the full, raw MIME message if the webhook url ends in 'mime'
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'token': '06c96bafc3f42a66b9edd546347a2fe18dc23461fe80dc52f0',
             'timestamp': '1461261330',
             'recipient': 'test@inbound.example.com',

tests/test_mailgun_webhooks.py

@@ -16,14 +16,33 @@ from .webhook_cases import WebhookTestCase, WebhookBasicAuthTestsMixin
 TEST_API_KEY = 'TEST_API_KEY'
 
 
-def mailgun_sign(data, api_key=TEST_API_KEY):
+def mailgun_signature(timestamp, token, api_key):
+    """Generates a Mailgun webhook signature"""
+    # https://documentation.mailgun.com/en/latest/user_manual.html#securing-webhooks
+    return hmac.new(
+        key=api_key.encode('ascii'),
+        msg='{timestamp}{token}'.format(timestamp=timestamp, token=token).encode('ascii'),
+        digestmod=hashlib.sha256).hexdigest()
+
+
+def mailgun_sign_payload(data, api_key=TEST_API_KEY):
+    """Add or complete Mailgun webhook signature block in data dict"""
+    # Modifies the dict in place
+    event_data = data.get('event-data', {})
+    signature = data.setdefault('signature', {})
+    token = signature.setdefault('token', '1234567890abcdef1234567890abcdef')
+    timestamp = signature.setdefault('timestamp',
+                                     str(int(float(event_data.get('timestamp', '1234567890.123')))))
+    signature['signature'] = mailgun_signature(timestamp, token, api_key=api_key)
+    return data
+
+
+def mailgun_sign_legacy_payload(data, api_key=TEST_API_KEY):
     """Add a Mailgun webhook signature to data dict"""
     # Modifies the dict in place
     data.setdefault('timestamp', '1234567890')
     data.setdefault('token', '1234567890abcdef1234567890abcdef')
-    data['signature'] = hmac.new(key=api_key.encode('ascii'),
+    data['signature'] = mailgun_signature(data['timestamp'], data['token'], api_key=api_key)
-                                 msg='{timestamp}{token}'.format(**data).encode('ascii'),
-                                 digestmod=hashlib.sha256).hexdigest()
     return data
 
 
@@ -42,8 +61,8 @@ def querydict_to_postdict(qd):
 class MailgunWebhookSettingsTestCase(WebhookTestCase):
     def test_requires_api_key(self):
         with self.assertRaises(ImproperlyConfigured):
-            self.client.post('/anymail/mailgun/tracking/',
+            self.client.post('/anymail/mailgun/tracking/', content_type="application/json",
-                             data=mailgun_sign({'event': 'delivered'}))
+                             data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}})))
 
 
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
@@ -51,32 +70,347 @@ class MailgunWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin
     should_warn_if_no_auth = False  # because we check webhook signature
 
     def call_webhook(self):
-        return self.client.post('/anymail/mailgun/tracking/',
+        return self.client.post('/anymail/mailgun/tracking/', content_type="application/json",
-                                data=mailgun_sign({'event': 'delivered'}))
+                                data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}})))
 
     # Additional tests are in WebhookBasicAuthTestsMixin
 
     def test_verifies_correct_signature(self):
-        response = self.client.post('/anymail/mailgun/tracking/',
+        response = self.client.post('/anymail/mailgun/tracking/', content_type="application/json",
-                                    data=mailgun_sign({'event': 'delivered'}))
+                                    data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}})))
         self.assertEqual(response.status_code, 200)
 
     def test_verifies_missing_signature(self):
-        response = self.client.post('/anymail/mailgun/tracking/',
+        response = self.client.post('/anymail/mailgun/tracking/', content_type="application/json",
-                                    data={'event': 'delivered'})
+                                    data=json.dumps({'event-data': {'event': 'delivered'}}))
         self.assertEqual(response.status_code, 400)
 
     def test_verifies_bad_signature(self):
-        data = mailgun_sign({'event': 'delivered'}, api_key="wrong API key")
+        data = mailgun_sign_payload({'event-data': {'event': 'delivered'}},
-        response = self.client.post('/anymail/mailgun/tracking/', data=data)
+                                    api_key="wrong API key")
+        response = self.client.post('/anymail/mailgun/tracking/', content_type="application/json",
+                                    data=json.dumps(data))
         self.assertEqual(response.status_code, 400)
 
 
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
-class MailgunDeliveryTestCase(WebhookTestCase):
+class MailgunTestCase(WebhookTestCase):
+    # Tests for Mailgun's new webhooks (announced 2018-06-29)
 
     def test_delivered_event(self):
-        raw_event = mailgun_sign({
+        # This is an actual, complete (sanitized) "delivered" event as received from Mailgun.
+        # (For brevity, later tests omit several payload fields that aren't used by Anymail.)
+        raw_event = mailgun_sign_payload({
+            "signature": {
+                "timestamp": "1534108637",
+                "token": "651869375b9df3c98fc15c4889b102119add1235c38fc92824",
+                "signature": "...",
+            },
+            "event-data": {
+                "tags": [],
+                "timestamp": 1534108637.153125,
+                "storage": {
+                    "url": "https://sw.api.mailgun.net/v3/domains/example.org/messages/eyJwI...",
+                    "key": "eyJwI...",
+                },
+                "recipient-domain": "example.com",
+                "id": "hTWCTD81RtiDN-...",
+                "campaigns": [],
+                "user-variables": {},
+                "flags": {
+                    "is-routed": False,
+                    "is-authenticated": True,
+                    "is-system-test": False,
+                    "is-test-mode": False,
+                },
+                "log-level": "info",
+                "envelope": {
+                    "sending-ip": "333.123.123.200",
+                    "sender": "test@example.org",
+                    "transport": "smtp",
+                    "targets": "recipient@example.com",
+                },
+                "message": {
+                    "headers": {
+                        "to": "recipient@example.com",
+                        "message-id": "20180812211713.1.DF5966851B4BAA99@example.org",
+                        "from": "test@example.org",
+                        "subject": "Testing",
+                    },
+                    "attachments": [],
+                    "size": 809,
+                },
+                "recipient": "recipient@example.com",
+                "event": "delivered",
+                "delivery-status": {
+                    "tls": True,
+                    "mx-host": "smtp-in.example.com",
+                    "attempt-no": 1,
+                    "description": "",
+                    "session-seconds": 3.5700838565826416,
+                    "utf8": True,
+                    "code": 250,
+                    "message": "OK",
+                    "certificate-verified": True,
+                },
+            },
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.event_type, "delivered")
+        self.assertEqual(event.timestamp, datetime(2018, 8, 12, 21, 17, 17, microsecond=153125, tzinfo=utc))
+        self.assertEqual(event.message_id, "<20180812211713.1.DF5966851B4BAA99@example.org>")
+        # Note that Anymail uses the "token" as its normalized event_id:
+        self.assertEqual(event.event_id, "651869375b9df3c98fc15c4889b102119add1235c38fc92824")
+        # ... if you want the Mailgun "event id", that's available through the raw esp_event:
+        self.assertEqual(event.esp_event["event-data"]["id"], "hTWCTD81RtiDN-...")
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.esp_event, raw_event)
+        self.assertEqual(event.tags, [])
+        self.assertEqual(event.metadata, {})
+
+    def test_failed_permanent_event(self):
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "failed",
+                "severity": "permanent",
+                "reason": "bounce",
+                "recipient": "invalid@example.com",
+                "timestamp": 1534110422.389832,
+                "log-level": "error",
+                "message": {
+                    "headers": {
+                        "to": "invalid@example.com",
+                        "message-id": "20180812214658.1.0DF563D0B3597700@example.org",
+                        "from": "Test Sender ",
+                    },
+                },
+                "delivery-status": {
+                    "tls": True,
+                    "mx-host": "aspmx.l.example.org",
+                    "attempt-no": 1,
+                    "description": "",
+                    "session-seconds": 2.952177047729492,
+                    "utf8": True,
+                    "code": 550,
+                    "message": "5.1.1 The email account that you tried to reach does not exist. Please try\n"
+                               "5.1.1 double-checking the recipient's email address for typos",
+                    "certificate-verified": True
+                }
+            },
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "bounced")
+        self.assertEqual(event.recipient, "invalid@example.com")
+        self.assertEqual(event.reject_reason, "bounced")
+        self.assertEqual(event.description, "")
+        self.assertEqual(event.mta_response,
+                         "5.1.1 The email account that you tried to reach does not exist. Please try\n"
+                         "5.1.1 double-checking the recipient's email address for typos")
+
+    def test_failed_temporary_event(self):
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "failed",
+                "severity": "temporary",
+                "reason": "generic",
+                "timestamp": 1534111899.659519,
+                "log-level": "warn",
+                "message": {
+                    "headers": {
+                        "to": "undeliverable@nomx.example.com",
+                        "message-id": "20180812214638.1.4A7D468E9BC18C5D@example.org",
+                        "from": "Test Sender ",
+                        "subject": "Testing"
+                    },
+                },
+                "recipient": "undeliverable@nomx.example.com",
+                "delivery-status": {
+                    "attempt-no": 3,
+                    "description": "No MX for nomx.example.com",
+                    "session-seconds": 0.0,
+                    "retry-seconds": 1800,
+                    "code": 498,
+                    "message": "No MX for nomx.example.com"
+                }
+            },
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "bounced")
+        self.assertEqual(event.recipient, "undeliverable@nomx.example.com")
+        self.assertEqual(event.reject_reason, "bounced")
+        self.assertEqual(event.description, "No MX for nomx.example.com")
+        self.assertEqual(event.mta_response, "No MX for nomx.example.com")
+
+    def test_rejected_event(self):
+        # (The "rejected" event is documented and appears in Mailgun dashboard logs,
+        # but it doesn't appear to be delivered through webhooks as of 8/2018.)
+        # Note that this payload lacks the recipient field present in all other events.
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "rejected",
+                "timestamp": 1529704976.104692,
+                "log-level": "warn",
+                "reject": {
+                    "reason": "Sandbox subdomains are for test purposes only.",
+                    "description": "",
+                },
+                "message": {
+                    "headers": {
+                        "to": "Recipient Name <recipient@example.org>",
+                        "message-id": "20180622220256.1.B31A451A2E5422BB@sandbox55887.mailgun.org",
+                        "from": "test@sandbox55887.mailgun.org",
+                        "subject": "Test Subject"
+                    },
+                },
+            },
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "rejected")
+        self.assertEqual(event.reject_reason, "other")
+        self.assertEqual(event.description, "Sandbox subdomains are for test purposes only.")
+        self.assertEqual(event.recipient, "recipient@example.org")
+
+    def test_complained_event(self):
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "complained",
+                "id": "ncV2XwymRUKbPek_MIM-Gw",
+                "timestamp": 1377214260.049634,
+                "log-level": "warn",
+                "recipient": "recipient@example.com",
+                "message": {
+                    "headers": {
+                        "to": "foo@recipient.com",
+                        "message-id": "20130718032413.263EE2E0926@example.org",
+                        "from": "Sender Name <sender@example.org>",
+                        "subject": "We are not spammer",
+                    },
+                },
+            },
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "complained")
+        self.assertEqual(event.recipient, "recipient@example.com")
+
+    def test_unsubscribed_event(self):
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "unsubscribed",
+                "id": "W3X4JOhFT-OZidZGKKr9iA",
+                "timestamp": 1377213791.421473,
+                "log-level": "info",
+                "recipient": "recipient@example.com",
+                "message": {
+                    "headers": {
+                        "message-id": "20130822232216.13966.79700@samples.mailgun.org"
+                    }
+                },
+            },
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "unsubscribed")
+        self.assertEqual(event.recipient, "recipient@example.com")
+
+    def test_opened_event(self):
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "opened",
+                "timestamp": 1534109600.089676,
+                "recipient": "recipient@example.com",
+                "tags": ["welcome", "variation-A"],
+                "user-variables": {
+                    "cohort": "2018-08-B",
+                    "user_id": "123456"
+                },
+                "message": {
+                    # Mailgun *only* includes the message-id header for opened, clicked events...
+                    "headers": {
+                        "message-id": "20180812213139.1.BC6694A917BB7E6A@example.org"
+                    }
+                },
+                "geolocation": {
+                    "country": "US",
+                    "region": "CA",
+                    "city": "San Francisco"
+                },
+                "ip": "888.222.444.111",
+                "client-info": {
+                    "client-type": "browser",
+                    "client-os": "OS X",
+                    "device-type": "desktop",
+                    "client-name": "Chrome",
+                    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6)..."
+                },
+            }
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "opened")
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.tags, ["welcome", "variation-A"])
+        self.assertEqual(event.metadata, {"cohort": "2018-08-B", "user_id": "123456"})
+
+    def test_clicked_event(self):
+        raw_event = mailgun_sign_payload({
+            "event-data": {
+                "event": "clicked",
+                "timestamp": 1534109600.089676,
+                "recipient": "recipient@example.com",
+                "url": "https://example.com/test"
+            }
+        })
+        response = self.client.post('/anymail/mailgun/tracking/',
+                                    data=json.dumps(raw_event), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailgunTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailgun')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "clicked")
+        self.assertEqual(event.click_url, "https://example.com/test")
+
+
+@override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
+class MailgunLegacyTestCase(WebhookTestCase):
+    # Tests for Mailgun's "legacy" webhooks
+    # (which were the only webhooks available prior to Anymail 4.0)
+
+    def test_delivered_event(self):
+        raw_event = mailgun_sign_legacy_payload({
             'domain': 'example.com',
             'message-headers': json.dumps([
                 ["Sender", "from=example.com"],
@@ -113,7 +447,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
         self.assertEqual(event.metadata, {})
 
     def test_dropped_bounce(self):
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'code': '605',
             'domain': 'example.com',
             'description': 'Not delivering to previously bounced address',
@@ -152,7 +486,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
         self.assertEqual(querydict_to_postdict(event.esp_event), raw_event)
 
     def test_dropped_spam(self):
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'code': '607',
             'description': 'Not delivering to a user who marked your messages as spam',
             'reason': 'hardfail',
@@ -170,7 +504,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
         self.assertEqual(event.description, 'Not delivering to a user who marked your messages as spam')
 
     def test_dropped_timed_out(self):
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'code': '499',
             'description': 'Unable to connect to MX servers: [example.com]',
             'reason': 'old',
@@ -188,7 +522,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
         self.assertEqual(event.description, 'Unable to connect to MX servers: [example.com]')
 
     def test_invalid_mailbox(self):
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'code': '550',
             'error': "550 5.1.1 The email account that you tried to reach does not exist. Please try "
                      "    5.1.1 double-checking the recipient's email address for typos or "
@@ -209,7 +543,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
     def test_alt_smtp_code(self):
         # In some cases, Mailgun uses RFC-3463 extended SMTP status codes (x.y.z, rather than nnn).
         # See issue #62.
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'code': '5.1.1',
             'error': 'smtp;550 5.1.1 RESOLVER.ADR.RecipNotFound; not found',
             'event': 'bounced',
@@ -228,7 +562,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
     def test_metadata_message_headers(self):
         # Metadata fields are interspersed with other data, but also in message-headers
         # for delivered, bounced and dropped events
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'event': 'delivered',
             'message-headers': json.dumps([
                 ["X-Mailgun-Variables", "{\"custom1\": \"value1\", \"custom2\": \"{\\\"key\\\":\\\"value\\\"}\"}"],
@@ -244,7 +578,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
     def test_metadata_post_fields(self):
         # Metadata fields are only interspersed with other event params
         # for opened, clicked, unsubscribed events
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'event': 'clicked',
             'custom1': 'value1',
             'custom2': '{"key":"value"}',  # you can store JSON, but you'll need to unpack it yourself
@@ -267,7 +601,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
             "ordinary field": "ordinary metadata value",
         }
 
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'event': 'clicked',
             'recipient': 'actual-recipient@example.com',
             'token': 'actual-event-token',
@@ -305,7 +639,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
 
     def test_tags(self):
         # Most events include multiple 'tag' fields for message's tags
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'tag': ['tag1', 'tag2'],  # Django TestClient encodes list as multiple field values
             'event': 'opened',
         })
@@ -316,7 +650,7 @@ class MailgunDeliveryTestCase(WebhookTestCase):
 
     def test_x_tags(self):
         # Delivery events don't include 'tag', but do include 'X-Mailgun-Tag' fields
-        raw_event = mailgun_sign({
+        raw_event = mailgun_sign_legacy_payload({
             'X-Mailgun-Tag': ['tag1', 'tag2'],
             'event': 'delivered',
         })