Mailjet: support tracking webhooks

anymail/urls.py

@@ -1,6 +1,7 @@
 from django.conf.urls import url
 
 from .webhooks.mailgun import MailgunTrackingWebhookView
+from .webhooks.mailjet import MailjetTrackingWebhookView
 from .webhooks.mandrill import MandrillTrackingWebhookView
 from .webhooks.postmark import PostmarkTrackingWebhookView
 from .webhooks.sendgrid import SendGridTrackingWebhookView
@@ -10,6 +11,7 @@ from .webhooks.sparkpost import SparkPostTrackingWebhookView
 app_name = 'anymail'
 urlpatterns = [
     url(r'^mailgun/tracking/$', MailgunTrackingWebhookView.as_view(), name='mailgun_tracking_webhook'),
+    url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'),
     url(r'^mandrill/tracking/$', MandrillTrackingWebhookView.as_view(), name='mandrill_tracking_webhook'),
     url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'),
     url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'),

anymail/webhooks/mailjet.py

@@ -0,0 +1,97 @@
+import json
+from datetime import datetime
+
+from django.utils.timezone import utc
+
+from .base import AnymailBaseWebhookView
+from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+
+
+class MailjetTrackingWebhookView(AnymailBaseWebhookView):
+    """Handler for Mailjet delivery and engagement tracking webhooks"""
+
+    signal = tracking
+
+    def parse_events(self, request):
+        esp_events = json.loads(request.body.decode('utf-8'))
+        return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
+
+    # https://dev.mailjet.com/guides/#events
+    event_types = {
+        # Map Mailjet event: Anymail normalized type
+        'sent': EventType.DELIVERED,  # accepted by receiving MTA
+        'open': EventType.OPENED,
+        'click': EventType.CLICKED,
+        'bounce': EventType.BOUNCED,
+        'blocked': EventType.REJECTED,
+        'spam': EventType.COMPLAINED,
+        'unsub': EventType.UNSUBSCRIBED,
+    }
+
+    reject_reasons = {
+        # Map Mailjet error strings to Anymail normalized reject_reason
+        # error_related_to: recipient
+        'user unknown': RejectReason.BOUNCED,
+        'mailbox inactive': RejectReason.BOUNCED,
+        'quota exceeded': RejectReason.BOUNCED,
+        'blacklisted': RejectReason.BLOCKED,  # might also be previous unsubscribe
+        'spam reporter': RejectReason.SPAM,
+        # error_related_to: domain
+        'invalid domain': RejectReason.BOUNCED,
+        'no mail host': RejectReason.BOUNCED,
+        'relay/access denied': RejectReason.BOUNCED,
+        'greylisted': RejectReason.OTHER,  # see special handling below
+        'typofix': RejectReason.INVALID,
+        # error_related_to: spam (all Mailjet policy/filtering; see above for spam complaints)
+        'sender blocked': RejectReason.BLOCKED,
+        'content blocked': RejectReason.BLOCKED,
+        'policy issue': RejectReason.BLOCKED,
+        # error_related_to: mailjet
+        'preblocked': RejectReason.BLOCKED,
+        'duplicate in campaign': RejectReason.OTHER,
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        event_type = self.event_types.get(esp_event['event'], EventType.UNKNOWN)
+        if esp_event.get('error', None) == 'greylisted' and not esp_event.get('hard_bounce', False):
+            # "This is a temporary error due to possible unrecognised senders. Delivery will be re-attempted."
+            event_type = EventType.DEFERRED
+
+        try:
+            timestamp = datetime.fromtimestamp(esp_event['time'], tz=utc)
+        except (KeyError, ValueError):
+            timestamp = None
+
+        try:
+            # convert bigint MessageID to str to match backend AnymailRecipientStatus
+            message_id = str(esp_event['MessageID'])
+        except (KeyError, TypeError):
+            message_id = None
+
+        if 'error' in esp_event:
+            reject_reason = self.reject_reasons.get(esp_event['error'], RejectReason.OTHER)
+        else:
+            reject_reason = None
+
+        tag = esp_event.get('customcampaign', None)
+        tags = [tag] if tag else []
+
+        try:
+            metadata = json.loads(esp_event['Payload'])
+        except (KeyError, ValueError):
+            metadata = {}
+
+        return AnymailTrackingEvent(
+            event_type=event_type,
+            timestamp=timestamp,
+            message_id=message_id,
+            event_id=None,
+            recipient=esp_event.get('email', None),
+            reject_reason=reject_reason,
+            mta_response=esp_event.get('smtp_reply', None),
+            tags=tags,
+            metadata=metadata,
+            click_url=esp_event.get('url', None),
+            user_agent=esp_event.get('agent', None),
+            esp_event=esp_event,
+        )

tests/test_mailjet_webhooks.py

@@ -0,0 +1,226 @@
+import json
+from datetime import datetime
+
+from django.utils.timezone import utc
+from mock import ANY
+
+from anymail.signals import AnymailTrackingEvent
+from anymail.webhooks.mailjet import MailjetTrackingWebhookView
+from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
+
+
+class MailjetWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
+    def call_webhook(self):
+        return self.client.post('/anymail/mailjet/tracking/',
+                                content_type='application/json', data=json.dumps([]))
+
+    # Actual tests are in WebhookBasicAuthTestsMixin
+
+
+class MailjetDeliveryTestCase(WebhookTestCase):
+
+    def test_sent_event(self):
+        # Mailjet's "sent" event indicates receiving MTA has accepted message; Anymail calls this "delivered"
+        raw_events = [{
+            "event": "sent",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "recipient@example.com",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "tag1",
+            "mj_message_id": "12345678901234567",
+            "smtp_reply": "sent (250 2.0.0 OK 1498093527 a67bc12345def.22 - gsmtp)",
+            "Payload": "{\"meta1\": \"simple string\", \"meta2\": 2}",
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.event_type, "delivered")
+        self.assertEqual(event.timestamp, datetime(2017, 6, 22, 1, 5, 27, tzinfo=utc))
+        self.assertEqual(event.esp_event, raw_events[0])
+        self.assertEqual(event.mta_response, "sent (250 2.0.0 OK 1498093527 a67bc12345def.22 - gsmtp)")
+        self.assertEqual(event.message_id, "12345678901234567")  # converted to str (matching backend status)
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.tags, ["tag1"])
+        self.assertEqual(event.metadata, {"meta1": "simple string", "meta2": 2})
+
+    def test_open_event(self):
+        raw_events = [{
+            "event": "open",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "recipient@example.com",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "ip": "192.168.100.100",
+            "geo": "US",
+            "agent": "Mozilla/5.0 (via ggpht.com GoogleImageProxy)",
+            "Payload": "",
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "opened")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.user_agent, "Mozilla/5.0 (via ggpht.com GoogleImageProxy)")
+        self.assertEqual(event.tags, [])
+        self.assertEqual(event.metadata, {})
+
+    def test_click_event(self):
+        raw_events = [{
+            "event": "open",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "recipient@example.com",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "url": "http://example.com",
+            "ip": "192.168.100.100",
+            "geo": "US",
+            "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) Chrome/58.0.3029.110",
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "opened")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) Chrome/58.0.3029.110")
+        self.assertEqual(event.click_url, "http://example.com")
+        self.assertEqual(event.tags, [])
+        self.assertEqual(event.metadata, {})
+
+    def test_bounce_event(self):
+        raw_events = [{
+            "event": "bounce",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "invalid@invalid",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "blocked": True,
+            "hard_bounce": True,
+            "error_related_to": "domain",
+            "error": "invalid domain"
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "bounced")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "invalid@invalid")
+        self.assertEqual(event.reject_reason, "bounced")
+        self.assertEqual(event.mta_response, None)
+
+    def test_blocked_event(self):
+        raw_events = [{
+            "event": "blocked",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "bad@example.com",
+            "mj_campaign_id": 0,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "error_related_to": "domain",
+            "error": "typofix",
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "rejected")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "bad@example.com")
+        self.assertEqual(event.reject_reason, "invalid")
+        self.assertEqual(event.mta_response, None)
+
+    def test_spam_event(self):
+        raw_events = [{
+            "event": "spam",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "spam@example.com",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "source": "greylisted"
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "complained")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "spam@example.com")
+
+    def test_unsub_event(self):
+        raw_events = [{
+            "event": "unsub",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "recipient@example.com",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "mj_list_id": 0,
+            "ip": "127.0.0.4",
+            "geo": "",
+            "agent": "List-Unsubscribe"
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "unsubscribed")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "recipient@example.com")
+
+    def test_bounced_greylist_event(self):
+        # greylist "bounce" should be reported as "deferred" (will be retried later)
+        raw_events = [{
+            "event": "bounce",
+            "time": 1498093527,
+            "MessageID": 12345678901234567,
+            "email": "protected@example.com",
+            "mj_campaign_id": 1234567890,
+            "mj_contact_id": 9876543210,
+            "customcampaign": "",
+            "blocked": True,
+            "hard_bounce": False,
+            "error_related_to": "domain",
+            "error": "greylisted"
+        }]
+        response = self.client.post('/anymail/mailjet/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView,
+                                                      event=ANY, esp_name='Mailjet')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "deferred")
+        self.assertEqual(event.message_id, "12345678901234567")
+        self.assertEqual(event.recipient, "protected@example.com")
+        self.assertEqual(event.reject_reason, "other")