From d6d80440663589ba2e098be3bcb949cc9107fcfc Mon Sep 17 00:00:00 2001 From: medmunds Date: Thu, 13 Jul 2017 11:27:00 -0700 Subject: [PATCH] Mailjet: support tracking webhooks --- anymail/urls.py | 2 + anymail/webhooks/mailjet.py | 97 ++++++++++++++ tests/test_mailjet_webhooks.py | 226 +++++++++++++++++++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 anymail/webhooks/mailjet.py create mode 100644 tests/test_mailjet_webhooks.py diff --git a/anymail/urls.py b/anymail/urls.py index 5a22094..e3e6cb5 100644 --- a/anymail/urls.py +++ b/anymail/urls.py @@ -1,6 +1,7 @@ from django.conf.urls import url from .webhooks.mailgun import MailgunTrackingWebhookView +from .webhooks.mailjet import MailjetTrackingWebhookView from .webhooks.mandrill import MandrillTrackingWebhookView from .webhooks.postmark import PostmarkTrackingWebhookView from .webhooks.sendgrid import SendGridTrackingWebhookView @@ -10,6 +11,7 @@ from .webhooks.sparkpost import SparkPostTrackingWebhookView app_name = 'anymail' urlpatterns = [ url(r'^mailgun/tracking/$', MailgunTrackingWebhookView.as_view(), name='mailgun_tracking_webhook'), + url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'), url(r'^mandrill/tracking/$', MandrillTrackingWebhookView.as_view(), name='mandrill_tracking_webhook'), url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'), url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'), diff --git a/anymail/webhooks/mailjet.py b/anymail/webhooks/mailjet.py new file mode 100644 index 0000000..bb8c5da --- /dev/null +++ b/anymail/webhooks/mailjet.py @@ -0,0 +1,97 @@ +import json +from datetime import datetime + +from django.utils.timezone import utc + +from .base import AnymailBaseWebhookView +from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason + + +class MailjetTrackingWebhookView(AnymailBaseWebhookView): + """Handler for Mailjet delivery and engagement tracking webhooks""" + + signal = tracking + + def parse_events(self, request): + esp_events = json.loads(request.body.decode('utf-8')) + return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events] + + # https://dev.mailjet.com/guides/#events + event_types = { + # Map Mailjet event: Anymail normalized type + 'sent': EventType.DELIVERED, # accepted by receiving MTA + 'open': EventType.OPENED, + 'click': EventType.CLICKED, + 'bounce': EventType.BOUNCED, + 'blocked': EventType.REJECTED, + 'spam': EventType.COMPLAINED, + 'unsub': EventType.UNSUBSCRIBED, + } + + reject_reasons = { + # Map Mailjet error strings to Anymail normalized reject_reason + # error_related_to: recipient + 'user unknown': RejectReason.BOUNCED, + 'mailbox inactive': RejectReason.BOUNCED, + 'quota exceeded': RejectReason.BOUNCED, + 'blacklisted': RejectReason.BLOCKED, # might also be previous unsubscribe + 'spam reporter': RejectReason.SPAM, + # error_related_to: domain + 'invalid domain': RejectReason.BOUNCED, + 'no mail host': RejectReason.BOUNCED, + 'relay/access denied': RejectReason.BOUNCED, + 'greylisted': RejectReason.OTHER, # see special handling below + 'typofix': RejectReason.INVALID, + # error_related_to: spam (all Mailjet policy/filtering; see above for spam complaints) + 'sender blocked': RejectReason.BLOCKED, + 'content blocked': RejectReason.BLOCKED, + 'policy issue': RejectReason.BLOCKED, + # error_related_to: mailjet + 'preblocked': RejectReason.BLOCKED, + 'duplicate in campaign': RejectReason.OTHER, + } + + def esp_to_anymail_event(self, esp_event): + event_type = self.event_types.get(esp_event['event'], EventType.UNKNOWN) + if esp_event.get('error', None) == 'greylisted' and not esp_event.get('hard_bounce', False): + # "This is a temporary error due to possible unrecognised senders. Delivery will be re-attempted." + event_type = EventType.DEFERRED + + try: + timestamp = datetime.fromtimestamp(esp_event['time'], tz=utc) + except (KeyError, ValueError): + timestamp = None + + try: + # convert bigint MessageID to str to match backend AnymailRecipientStatus + message_id = str(esp_event['MessageID']) + except (KeyError, TypeError): + message_id = None + + if 'error' in esp_event: + reject_reason = self.reject_reasons.get(esp_event['error'], RejectReason.OTHER) + else: + reject_reason = None + + tag = esp_event.get('customcampaign', None) + tags = [tag] if tag else [] + + try: + metadata = json.loads(esp_event['Payload']) + except (KeyError, ValueError): + metadata = {} + + return AnymailTrackingEvent( + event_type=event_type, + timestamp=timestamp, + message_id=message_id, + event_id=None, + recipient=esp_event.get('email', None), + reject_reason=reject_reason, + mta_response=esp_event.get('smtp_reply', None), + tags=tags, + metadata=metadata, + click_url=esp_event.get('url', None), + user_agent=esp_event.get('agent', None), + esp_event=esp_event, + ) diff --git a/tests/test_mailjet_webhooks.py b/tests/test_mailjet_webhooks.py new file mode 100644 index 0000000..70ada43 --- /dev/null +++ b/tests/test_mailjet_webhooks.py @@ -0,0 +1,226 @@ +import json +from datetime import datetime + +from django.utils.timezone import utc +from mock import ANY + +from anymail.signals import AnymailTrackingEvent +from anymail.webhooks.mailjet import MailjetTrackingWebhookView +from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase + + +class MailjetWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin): + def call_webhook(self): + return self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps([])) + + # Actual tests are in WebhookBasicAuthTestsMixin + + +class MailjetDeliveryTestCase(WebhookTestCase): + + def test_sent_event(self): + # Mailjet's "sent" event indicates receiving MTA has accepted message; Anymail calls this "delivered" + raw_events = [{ + "event": "sent", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "recipient@example.com", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "tag1", + "mj_message_id": "12345678901234567", + "smtp_reply": "sent (250 2.0.0 OK 1498093527 a67bc12345def.22 - gsmtp)", + "Payload": "{\"meta1\": \"simple string\", \"meta2\": 2}", + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "delivered") + self.assertEqual(event.timestamp, datetime(2017, 6, 22, 1, 5, 27, tzinfo=utc)) + self.assertEqual(event.esp_event, raw_events[0]) + self.assertEqual(event.mta_response, "sent (250 2.0.0 OK 1498093527 a67bc12345def.22 - gsmtp)") + self.assertEqual(event.message_id, "12345678901234567") # converted to str (matching backend status) + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.tags, ["tag1"]) + self.assertEqual(event.metadata, {"meta1": "simple string", "meta2": 2}) + + def test_open_event(self): + raw_events = [{ + "event": "open", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "recipient@example.com", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "", + "ip": "192.168.100.100", + "geo": "US", + "agent": "Mozilla/5.0 (via ggpht.com GoogleImageProxy)", + "Payload": "", + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "opened") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.user_agent, "Mozilla/5.0 (via ggpht.com GoogleImageProxy)") + self.assertEqual(event.tags, []) + self.assertEqual(event.metadata, {}) + + def test_click_event(self): + raw_events = [{ + "event": "open", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "recipient@example.com", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "", + "url": "http://example.com", + "ip": "192.168.100.100", + "geo": "US", + "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) Chrome/58.0.3029.110", + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "opened") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) Chrome/58.0.3029.110") + self.assertEqual(event.click_url, "http://example.com") + self.assertEqual(event.tags, []) + self.assertEqual(event.metadata, {}) + + def test_bounce_event(self): + raw_events = [{ + "event": "bounce", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "invalid@invalid", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "", + "blocked": True, + "hard_bounce": True, + "error_related_to": "domain", + "error": "invalid domain" + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "invalid@invalid") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.mta_response, None) + + def test_blocked_event(self): + raw_events = [{ + "event": "blocked", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "bad@example.com", + "mj_campaign_id": 0, + "mj_contact_id": 9876543210, + "customcampaign": "", + "error_related_to": "domain", + "error": "typofix", + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "rejected") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "bad@example.com") + self.assertEqual(event.reject_reason, "invalid") + self.assertEqual(event.mta_response, None) + + def test_spam_event(self): + raw_events = [{ + "event": "spam", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "spam@example.com", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "", + "source": "greylisted" + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "complained") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "spam@example.com") + + def test_unsub_event(self): + raw_events = [{ + "event": "unsub", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "recipient@example.com", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "", + "mj_list_id": 0, + "ip": "127.0.0.4", + "geo": "", + "agent": "List-Unsubscribe" + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "unsubscribed") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "recipient@example.com") + + def test_bounced_greylist_event(self): + # greylist "bounce" should be reported as "deferred" (will be retried later) + raw_events = [{ + "event": "bounce", + "time": 1498093527, + "MessageID": 12345678901234567, + "email": "protected@example.com", + "mj_campaign_id": 1234567890, + "mj_contact_id": 9876543210, + "customcampaign": "", + "blocked": True, + "hard_bounce": False, + "error_related_to": "domain", + "error": "greylisted" + }] + response = self.client.post('/anymail/mailjet/tracking/', + content_type='application/json', data=json.dumps(raw_events)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=MailjetTrackingWebhookView, + event=ANY, esp_name='Mailjet') + event = kwargs['event'] + self.assertEqual(event.event_type, "deferred") + self.assertEqual(event.message_id, "12345678901234567") + self.assertEqual(event.recipient, "protected@example.com") + self.assertEqual(event.reject_reason, "other")