diff --git a/anymail/urls.py b/anymail/urls.py index 75a3f77..9c8ae9f 100644 --- a/anymail/urls.py +++ b/anymail/urls.py @@ -5,6 +5,7 @@ from .webhooks.mailjet import MailjetInboundWebhookView, MailjetTrackingWebhookV from .webhooks.mandrill import MandrillCombinedWebhookView from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView from .webhooks.sendgrid import SendGridInboundWebhookView, SendGridTrackingWebhookView +from .webhooks.sendinblue import SendinBlueTrackingWebhookView from .webhooks.sparkpost import SparkPostInboundWebhookView, SparkPostTrackingWebhookView @@ -20,6 +21,7 @@ urlpatterns = [ url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'), url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'), url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'), + url(r'^sendinblue/tracking/$', SendinBlueTrackingWebhookView.as_view(), name='sendinblue_tracking_webhook'), url(r'^sparkpost/tracking/$', SparkPostTrackingWebhookView.as_view(), name='sparkpost_tracking_webhook'), # Anymail uses a combined Mandrill webhook endpoint, to simplify Mandrill's key-validation scheme: diff --git a/anymail/webhooks/sendinblue.py b/anymail/webhooks/sendinblue.py new file mode 100644 index 0000000..08b2dc3 --- /dev/null +++ b/anymail/webhooks/sendinblue.py @@ -0,0 +1,76 @@ +import json +from datetime import datetime + +from django.utils.timezone import utc + +from .base import AnymailBaseWebhookView +from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking + + +class SendinBlueTrackingWebhookView(AnymailBaseWebhookView): + """Handler for SendinBlue delivery and engagement tracking webhooks""" + + esp_name = "SendinBlue" + signal = tracking + + def parse_events(self, request): + esp_event = json.loads(request.body.decode('utf-8')) + return [self.esp_to_anymail_event(esp_event)] + + # SendinBlue's webhook payload data doesn't seem to be documented anywhere. + # There's a list of webhook events at https://apidocs.sendinblue.com/webhooks/#3. + event_types = { + # Map SendinBlue event type: Anymail normalized (event type, reject reason) + "request": (EventType.QUEUED, None), # received even if message won't be sent (e.g., before "blocked") + "delivered": (EventType.DELIVERED, None), + "hard_bounce": (EventType.BOUNCED, RejectReason.BOUNCED), + "soft_bounce": (EventType.BOUNCED, RejectReason.BOUNCED), + "blocked": (EventType.REJECTED, RejectReason.BLOCKED), + "spam": (EventType.COMPLAINED, RejectReason.SPAM), + "invalid_email": (EventType.BOUNCED, RejectReason.INVALID), + "deferred": (EventType.DEFERRED, None), + "opened": (EventType.OPENED, None), # see also unique_opened below + "click": (EventType.CLICKED, None), + "unsubscribe": (EventType.UNSUBSCRIBED, None), + "list_addition": (EventType.SUBSCRIBED, None), # shouldn't occur for transactional messages + "unique_opened": (EventType.OPENED, None), # you'll *also* receive an "opened" + } + + def esp_to_anymail_event(self, esp_event): + esp_type = esp_event.get("event") + event_type, reject_reason = self.event_types.get(esp_type, (EventType.UNKNOWN, None)) + recipient = esp_event.get("email") + + try: + # SendinBlue supplies "ts", "ts_event" and "date" fields, which seem to be based on the + # timezone set in the account preferences (and possibly with inconsistent DST adjustment). + # "ts_epoch" is the only field that seems to be consistently UTC; it's in milliseconds + timestamp = datetime.fromtimestamp(esp_event["ts_epoch"] / 1000.0, tz=utc) + except (KeyError, ValueError): + timestamp = None + + try: + tags = [esp_event["tag"]] + except KeyError: + tags = [] + + try: + metadata = json.loads(esp_event["X-Mailin-custom"]) + except (KeyError, TypeError): + metadata = {} + + return AnymailTrackingEvent( + description=None, + esp_event=esp_event, + event_id=None, # SendinBlue doesn't provide a unique event id + event_type=event_type, + message_id=esp_event.get("message-id"), + metadata=metadata, + mta_response=esp_event.get("reason"), + recipient=recipient, + reject_reason=reject_reason, + tags=tags, + timestamp=timestamp, + user_agent=None, + click_url=esp_event.get("link"), + ) diff --git a/docs/esps/index.rst b/docs/esps/index.rst index c0efb51..3e979ce 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -49,7 +49,7 @@ Email Service Provider |Mailgun| |Mailjet| |Mandrill .. rubric:: :ref:`Status ` and :ref:`event tracking ` ------------------------------------------------------------------------------------------------------------------------------------- :attr:`~AnymailMessage.anymail_status` Yes Yes Yes Yes Yes Yes Yes -|AnymailTrackingEvent| from webhooks Yes Yes Yes Yes Yes *Coming* Yes +|AnymailTrackingEvent| from webhooks Yes Yes Yes Yes Yes Yes Yes .. rubric:: :ref:`Inbound handling ` ------------------------------------------------------------------------------------------------------------------------------------- diff --git a/docs/esps/sendinblue.rst b/docs/esps/sendinblue.rst index 9a8de85..044d1ac 100644 --- a/docs/esps/sendinblue.rst +++ b/docs/esps/sendinblue.rst @@ -222,8 +222,37 @@ only in *non*-template sends.) Status tracking webhooks ------------------------ -SendinBlue supports status tracking webhooks. Integration with Anymail's normalized -:ref:`status tracking ` is planned for a future release. +If you are using Anymail's normalized :ref:`status tracking `, add +the url at SendinBlue's site under `Transactional > Settings > Webhook`_. + +The "URL to call" is: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/sendinblue/tracking/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret + * *yoursite.example.com* is your Django site + +Be sure to select the checkboxes for all the event types you want to receive. (Also make +sure you are in the "Transactional" section of their site; SendinBlue has a separate set +of "Campaign" webhooks, which don't apply to messages sent through Anymail.) + +If you are interested in tracking opens, note that SendinBlue has both a "First opening" +and an "Opened" event type, and will generate both the first time a message is opened. +Anymail normalizes both of these events to "opened." To avoid double counting, you should +only enable one of the two. + +SendinBlue will report these Anymail :attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s: +queued, rejected, bounced, deferred, delivered, opened (see note above), clicked, complained, +unsubscribed, subscribed (though this should never occur for transactional email). + +For events that occur in rapid succession, SendinBlue frequently delivers them out of order. +For example, it's not uncommon to receive a "delivered" event before the corresponding "queued." + +The event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` field will be +a `dict` of raw webhook data received from SendinBlue. + + +.. _Transactional > Settings > Webhook: https://app-smtp.sendinblue.com/webhook .. _sendinblue-inbound: diff --git a/tests/test_sendinblue_webhooks.py b/tests/test_sendinblue_webhooks.py new file mode 100644 index 0000000..bb4ce06 --- /dev/null +++ b/tests/test_sendinblue_webhooks.py @@ -0,0 +1,270 @@ +import json +from datetime import datetime + +from django.utils.timezone import utc +from mock import ANY + +from anymail.signals import AnymailTrackingEvent +from anymail.webhooks.sendinblue import SendinBlueTrackingWebhookView +from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase + + +class SendinBlueWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin): + def call_webhook(self): + return self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps({})) + + # Actual tests are in WebhookBasicAuthTestsMixin + + +class SendinBlueDeliveryTestCase(WebhookTestCase): + # SendinBlue's webhook payload data doesn't seem to be documented anywhere. + # There's a list of webhook events at https://apidocs.sendinblue.com/webhooks/#3. + # The payloads below were obtained through live testing. + + def test_sent_event(self): + raw_event = { + "event": "request", + "email": "recipient@example.com", + "id": 9999999, # this appears to be a SendinBlue account id (not an event id) + "message-id": "<201803062010.27287306012@smtp-relay.mailin.fr>", + "subject": "Test subject", + + # From a message sent at 2018-03-06 11:10:23-08:00 (2018-03-06 19:10:23+00:00)... + "date": "2018-03-06 11:10:23", # uses time zone from SendinBlue account's preferences + "ts": 1520331023, # 2018-03-06 10:10:23 -- what time zone is this? + "ts_event": 1520331023, # unclear if this ever differs from "ts" + "ts_epoch": 1520363423000, # 2018-03-06 19:10:23.000+00:00 -- UTC (milliseconds) + + "X-Mailin-custom": '{"meta": "data"}', + "tag": "test-tag", # note: for template send, is template name if no other tag provided + "template_id": 12, + "sending_ip": "333.33.33.33", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "queued") + self.assertEqual(event.esp_event, raw_event) + self.assertEqual(event.timestamp, datetime(2018, 3, 6, 19, 10, 23, microsecond=0, tzinfo=utc)) + self.assertEqual(event.message_id, "<201803062010.27287306012@smtp-relay.mailin.fr>") + self.assertIsNone(event.event_id) # SendinBlue does not provide a unique event id + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.metadata, {"meta": "data"}) + self.assertEqual(event.tags, ["test-tag"]) + + def test_delivered_event(self): + raw_event = { + # For brevity, this and following tests omit some webhook data + # that was tested earlier, or that is not used by Anymail + "event": "delivered", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "delivered") + self.assertEqual(event.esp_event, raw_event) + self.assertEqual(event.message_id, "<201803011158.9876543210@smtp-relay.mailin.fr>") + self.assertEqual(event.recipient, "recipient@example.com") + self.assertEqual(event.metadata, {}) # empty dict when no X-Mailin-custom header given + self.assertEqual(event.tags, []) # empty list when no tags given + + def test_hard_bounce(self): + raw_event = { + "event": "hard_bounce", + "email": "not-a-user@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + # the leading space in the reason is as received in actual testing: + "reason": " RecipientError: 550 5.5.0 Requested action not taken: mailbox unavailable.", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.mta_response, + " RecipientError: 550 5.5.0 Requested action not taken: mailbox unavailable.") + + def test_soft_bounce_event(self): + raw_event = { + "event": "soft_bounce", + "email": "recipient@no-mx.example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + "reason": "undefined Unable to find MX of domain no-mx.example.com", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.reject_reason, "bounced") + self.assertIsNone(event.description) # no human-readable description consistently available + self.assertEqual(event.mta_response, "undefined Unable to find MX of domain no-mx.example.com") + + def test_blocked(self): + raw_event = { + "event": "blocked", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + "reason": "blocked : due to blacklist user", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "rejected") + self.assertEqual(event.reject_reason, "blocked") + self.assertEqual(event.mta_response, "blocked : due to blacklist user") + + def test_spam(self): + # "When a person who received your email reported that it is a spam." + # (haven't observed "spam" event in actual testing; payload below is a guess) + raw_event = { + "event": "spam", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "complained") + + def test_invalid_email(self): + # "If a ISP again indicated us that the email is not valid or if we discovered that the email is not valid." + # (unclear whether this error originates with the receiving MTA or with SendinBlue pre-send) + # (haven't observed "invalid_email" event in actual testing; payload below is a guess) + raw_event = { + "event": "invalid_email", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + "reason": "(guessing invalid_email includes a reason)", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.reject_reason, "invalid") + self.assertEqual(event.mta_response, "(guessing invalid_email includes a reason)") + + def test_deferred_event(self): + # Note: the example below is an actual event capture (with 'example.com' substituted + # for the real receiving domain). It's pretty clearly a bounce, not a deferral. + # It looks like SendinBlue mis-categorizes this SMTP response code. + raw_event = { + "event": "deferred", + "email": "notauser@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + "reason": "550 RecipientError: 550 5.1.1 : Recipient address rejected: " + "User unknown in virtual alias table", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "deferred") + self.assertIsNone(event.description) # no human-readable description consistently available + self.assertEqual(event.mta_response, + "550 RecipientError: 550 5.1.1 : Recipient address rejected: " + "User unknown in virtual alias table") + + def test_opened_event(self): + # SendinBlue delivers unique_opened *and* opened on the first open. + # To avoid double-counting, you should only enable one of the two events in SendinBlue. + raw_event = { + "event": "opened", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "opened") + self.assertIsNone(event.user_agent) # SendinBlue doesn't report user agent + + def test_unique_opened_event(self): + # SendinBlue delivers unique_opened *and* opened on the first open. + # To avoid double-counting, you should only enable one of the two events in SendinBlue. + raw_event = { + "event": "unique_opened", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "opened") + + def test_clicked_event(self): + raw_event = { + "event": "click", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + "link": "https://example.com/click/me", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "clicked") + self.assertEqual(event.click_url, "https://example.com/click/me") + self.assertIsNone(event.user_agent) # SendinBlue doesn't report user agent + + def test_unsubscribe(self): + # "When a person unsubscribes from the email received." + # (haven't observed "unsubscribe" event in actual testing; payload below is a guess) + raw_event = { + "event": "unsubscribe", + "email": "recipient@example.com", + "ts_epoch": 1520363423000, + "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>", + } + response = self.client.post('/anymail/sendinblue/tracking/', + content_type='application/json', data=json.dumps(raw_event)) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView, + event=ANY, esp_name='SendinBlue') + event = kwargs['event'] + self.assertEqual(event.event_type, "unsubscribed")