SendinBlue: implement tracking webhooks

Completes #84

anymail/urls.py

@@ -5,6 +5,7 @@ from .webhooks.mailjet import MailjetInboundWebhookView, MailjetTrackingWebhookV
 from .webhooks.mandrill import MandrillCombinedWebhookView
 from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView
 from .webhooks.sendgrid import SendGridInboundWebhookView, SendGridTrackingWebhookView
+from .webhooks.sendinblue import SendinBlueTrackingWebhookView
 from .webhooks.sparkpost import SparkPostInboundWebhookView, SparkPostTrackingWebhookView
 
 
@@ -20,6 +21,7 @@ urlpatterns = [
     url(r'^mailjet/tracking/$', MailjetTrackingWebhookView.as_view(), name='mailjet_tracking_webhook'),
     url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'),
     url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'),
+    url(r'^sendinblue/tracking/$', SendinBlueTrackingWebhookView.as_view(), name='sendinblue_tracking_webhook'),
     url(r'^sparkpost/tracking/$', SparkPostTrackingWebhookView.as_view(), name='sparkpost_tracking_webhook'),
 
     # Anymail uses a combined Mandrill webhook endpoint, to simplify Mandrill's key-validation scheme:

anymail/webhooks/sendinblue.py

@@ -0,0 +1,76 @@
+import json
+from datetime import datetime
+
+from django.utils.timezone import utc
+
+from .base import AnymailBaseWebhookView
+from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking
+
+
+class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
+    """Handler for SendinBlue delivery and engagement tracking webhooks"""
+
+    esp_name = "SendinBlue"
+    signal = tracking
+
+    def parse_events(self, request):
+        esp_event = json.loads(request.body.decode('utf-8'))
+        return [self.esp_to_anymail_event(esp_event)]
+
+    # SendinBlue's webhook payload data doesn't seem to be documented anywhere.
+    # There's a list of webhook events at https://apidocs.sendinblue.com/webhooks/#3.
+    event_types = {
+        # Map SendinBlue event type: Anymail normalized (event type, reject reason)
+        "request": (EventType.QUEUED, None),  # received even if message won't be sent (e.g., before "blocked")
+        "delivered": (EventType.DELIVERED, None),
+        "hard_bounce": (EventType.BOUNCED, RejectReason.BOUNCED),
+        "soft_bounce": (EventType.BOUNCED, RejectReason.BOUNCED),
+        "blocked": (EventType.REJECTED, RejectReason.BLOCKED),
+        "spam": (EventType.COMPLAINED, RejectReason.SPAM),
+        "invalid_email": (EventType.BOUNCED, RejectReason.INVALID),
+        "deferred": (EventType.DEFERRED, None),
+        "opened": (EventType.OPENED, None),  # see also unique_opened below
+        "click": (EventType.CLICKED, None),
+        "unsubscribe": (EventType.UNSUBSCRIBED, None),
+        "list_addition": (EventType.SUBSCRIBED, None),  # shouldn't occur for transactional messages
+        "unique_opened": (EventType.OPENED, None),  # you'll *also* receive an "opened"
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        esp_type = esp_event.get("event")
+        event_type, reject_reason = self.event_types.get(esp_type, (EventType.UNKNOWN, None))
+        recipient = esp_event.get("email")
+
+        try:
+            # SendinBlue supplies "ts", "ts_event" and "date" fields, which seem to be based on the
+            # timezone set in the account preferences (and possibly with inconsistent DST adjustment).
+            # "ts_epoch" is the only field that seems to be consistently UTC; it's in milliseconds
+            timestamp = datetime.fromtimestamp(esp_event["ts_epoch"] / 1000.0, tz=utc)
+        except (KeyError, ValueError):
+            timestamp = None
+
+        try:
+            tags = [esp_event["tag"]]
+        except KeyError:
+            tags = []
+
+        try:
+            metadata = json.loads(esp_event["X-Mailin-custom"])
+        except (KeyError, TypeError):
+            metadata = {}
+
+        return AnymailTrackingEvent(
+            description=None,
+            esp_event=esp_event,
+            event_id=None,  # SendinBlue doesn't provide a unique event id
+            event_type=event_type,
+            message_id=esp_event.get("message-id"),
+            metadata=metadata,
+            mta_response=esp_event.get("reason"),
+            recipient=recipient,
+            reject_reason=reject_reason,
+            tags=tags,
+            timestamp=timestamp,
+            user_agent=None,
+            click_url=esp_event.get("link"),
+        )

docs/esps/index.rst

@@ -49,7 +49,7 @@ Email Service Provider                        |Mailgun|    |Mailjet|   |Mandrill
 .. rubric:: :ref:`Status <esp-send-status>` and :ref:`event tracking <event-tracking>`
 -------------------------------------------------------------------------------------------------------------------------------------
 :attr:`~AnymailMessage.anymail_status`        Yes          Yes         Yes          Yes         Yes         Yes           Yes
-|AnymailTrackingEvent| from webhooks          Yes          Yes         Yes          Yes         Yes         *Coming*      Yes
+|AnymailTrackingEvent| from webhooks          Yes          Yes         Yes          Yes         Yes         Yes           Yes
 
 .. rubric:: :ref:`Inbound handling <inbound>`
 -------------------------------------------------------------------------------------------------------------------------------------

docs/esps/sendinblue.rst

@@ -222,8 +222,37 @@ only in *non*-template sends.)
 Status tracking webhooks
 ------------------------
 
-SendinBlue supports status tracking webhooks. Integration with Anymail's normalized
+If you are using Anymail's normalized :ref:`status tracking <event-tracking>`, add
-:ref:`status tracking <event-tracking>` is planned for a future release.
+the url at SendinBlue's site under  `Transactional > Settings > Webhook`_.
+
+The "URL to call" is:
+
+   :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/sendinblue/tracking/`
+
+     * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret
+     * *yoursite.example.com* is your Django site
+
+Be sure to select the checkboxes for all the event types you want to receive. (Also make
+sure you are in the "Transactional" section of their site; SendinBlue has a separate set
+of "Campaign" webhooks, which don't apply to messages sent through Anymail.)
+
+If you are interested in tracking opens, note that SendinBlue has both a "First opening"
+and an "Opened" event type, and will generate both the first time a message is opened.
+Anymail normalizes both of these events to "opened." To avoid double counting, you should
+only enable one of the two.
+
+SendinBlue will report these Anymail :attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s:
+queued, rejected, bounced, deferred, delivered, opened (see note above), clicked, complained,
+unsubscribed, subscribed (though this should never occur for transactional email).
+
+For events that occur in rapid succession, SendinBlue frequently delivers them out of order.
+For example, it's not uncommon to receive a "delivered" event before the corresponding "queued."
+
+The event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` field will be
+a `dict` of raw webhook data received from SendinBlue.
+
+
+.. _Transactional > Settings > Webhook: https://app-smtp.sendinblue.com/webhook
 
 
 .. _sendinblue-inbound:

tests/test_sendinblue_webhooks.py

@@ -0,0 +1,270 @@
+import json
+from datetime import datetime
+
+from django.utils.timezone import utc
+from mock import ANY
+
+from anymail.signals import AnymailTrackingEvent
+from anymail.webhooks.sendinblue import SendinBlueTrackingWebhookView
+from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
+
+
+class SendinBlueWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
+    def call_webhook(self):
+        return self.client.post('/anymail/sendinblue/tracking/',
+                                content_type='application/json', data=json.dumps({}))
+
+    # Actual tests are in WebhookBasicAuthTestsMixin
+
+
+class SendinBlueDeliveryTestCase(WebhookTestCase):
+    # SendinBlue's webhook payload data doesn't seem to be documented anywhere.
+    # There's a list of webhook events at https://apidocs.sendinblue.com/webhooks/#3.
+    # The payloads below were obtained through live testing.
+
+    def test_sent_event(self):
+        raw_event = {
+            "event": "request",
+            "email": "recipient@example.com",
+            "id": 9999999,  # this appears to be a SendinBlue account id (not an event id)
+            "message-id": "<201803062010.27287306012@smtp-relay.mailin.fr>",
+            "subject": "Test subject",
+
+            # From a message sent at 2018-03-06 11:10:23-08:00 (2018-03-06 19:10:23+00:00)...
+            "date": "2018-03-06 11:10:23",  # uses time zone from SendinBlue account's preferences
+            "ts": 1520331023,  # 2018-03-06 10:10:23 -- what time zone is this?
+            "ts_event": 1520331023,  # unclear if this ever differs from "ts"
+            "ts_epoch": 1520363423000,  # 2018-03-06 19:10:23.000+00:00 -- UTC (milliseconds)
+
+            "X-Mailin-custom": '{"meta": "data"}',
+            "tag": "test-tag",  # note: for template send, is template name if no other tag provided
+            "template_id": 12,
+            "sending_ip": "333.33.33.33",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.event_type, "queued")
+        self.assertEqual(event.esp_event, raw_event)
+        self.assertEqual(event.timestamp, datetime(2018, 3, 6, 19, 10, 23, microsecond=0, tzinfo=utc))
+        self.assertEqual(event.message_id, "<201803062010.27287306012@smtp-relay.mailin.fr>")
+        self.assertIsNone(event.event_id)  # SendinBlue does not provide a unique event id
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.metadata, {"meta": "data"})
+        self.assertEqual(event.tags, ["test-tag"])
+
+    def test_delivered_event(self):
+        raw_event = {
+            # For brevity, this and following tests omit some webhook data
+            # that was tested earlier, or that is not used by Anymail
+            "event": "delivered",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.event_type, "delivered")
+        self.assertEqual(event.esp_event, raw_event)
+        self.assertEqual(event.message_id, "<201803011158.9876543210@smtp-relay.mailin.fr>")
+        self.assertEqual(event.recipient, "recipient@example.com")
+        self.assertEqual(event.metadata, {})  # empty dict when no X-Mailin-custom header given
+        self.assertEqual(event.tags, [])  # empty list when no tags given
+
+    def test_hard_bounce(self):
+        raw_event = {
+            "event": "hard_bounce",
+            "email": "not-a-user@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+            # the leading space in the reason is as received in actual testing:
+            "reason": " RecipientError: 550 5.5.0 Requested action not taken: mailbox unavailable.",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "bounced")
+        self.assertEqual(event.reject_reason, "bounced")
+        self.assertEqual(event.mta_response,
+                         " RecipientError: 550 5.5.0 Requested action not taken: mailbox unavailable.")
+
+    def test_soft_bounce_event(self):
+        raw_event = {
+            "event": "soft_bounce",
+            "email": "recipient@no-mx.example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+            "reason": "undefined Unable to find MX of domain no-mx.example.com",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "bounced")
+        self.assertEqual(event.reject_reason, "bounced")
+        self.assertIsNone(event.description)  # no human-readable description consistently available
+        self.assertEqual(event.mta_response, "undefined Unable to find MX of domain no-mx.example.com")
+
+    def test_blocked(self):
+        raw_event = {
+            "event": "blocked",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+            "reason": "blocked : due to blacklist user",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "rejected")
+        self.assertEqual(event.reject_reason, "blocked")
+        self.assertEqual(event.mta_response, "blocked : due to blacklist user")
+
+    def test_spam(self):
+        # "When a person who received your email reported that it is a spam."
+        # (haven't observed "spam" event in actual testing; payload below is a guess)
+        raw_event = {
+            "event": "spam",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "complained")
+
+    def test_invalid_email(self):
+        # "If a ISP again indicated us that the email is not valid or if we discovered that the email is not valid."
+        # (unclear whether this error originates with the receiving MTA or with SendinBlue pre-send)
+        # (haven't observed "invalid_email" event in actual testing; payload below is a guess)
+        raw_event = {
+            "event": "invalid_email",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+            "reason": "(guessing invalid_email includes a reason)",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "bounced")
+        self.assertEqual(event.reject_reason, "invalid")
+        self.assertEqual(event.mta_response, "(guessing invalid_email includes a reason)")
+
+    def test_deferred_event(self):
+        # Note: the example below is an actual event capture (with 'example.com' substituted
+        # for the real receiving domain). It's pretty clearly a bounce, not a deferral.
+        # It looks like SendinBlue mis-categorizes this SMTP response code.
+        raw_event = {
+            "event": "deferred",
+            "email": "notauser@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+            "reason": "550 RecipientError: 550 5.1.1 <notauser@example.com>: Recipient address rejected: "
+                      "User unknown in virtual alias table",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "deferred")
+        self.assertIsNone(event.description)  # no human-readable description consistently available
+        self.assertEqual(event.mta_response,
+                         "550 RecipientError: 550 5.1.1 <notauser@example.com>: Recipient address rejected: "
+                         "User unknown in virtual alias table")
+
+    def test_opened_event(self):
+        # SendinBlue delivers unique_opened *and* opened on the first open.
+        # To avoid double-counting, you should only enable one of the two events in SendinBlue.
+        raw_event = {
+            "event": "opened",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "opened")
+        self.assertIsNone(event.user_agent)  # SendinBlue doesn't report user agent
+
+    def test_unique_opened_event(self):
+        # SendinBlue delivers unique_opened *and* opened on the first open.
+        # To avoid double-counting, you should only enable one of the two events in SendinBlue.
+        raw_event = {
+            "event": "unique_opened",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "opened")
+
+    def test_clicked_event(self):
+        raw_event = {
+            "event": "click",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+            "link": "https://example.com/click/me",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "clicked")
+        self.assertEqual(event.click_url, "https://example.com/click/me")
+        self.assertIsNone(event.user_agent)  # SendinBlue doesn't report user agent
+
+    def test_unsubscribe(self):
+        # "When a person unsubscribes from the email received."
+        # (haven't observed "unsubscribe" event in actual testing; payload below is a guess)
+        raw_event = {
+            "event": "unsubscribe",
+            "email": "recipient@example.com",
+            "ts_epoch": 1520363423000,
+            "message-id": "<201803011158.9876543210@smtp-relay.mailin.fr>",
+        }
+        response = self.client.post('/anymail/sendinblue/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_event))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SendinBlueTrackingWebhookView,
+                                                      event=ANY, esp_name='SendinBlue')
+        event = kwargs['event']
+        self.assertEqual(event.event_type, "unsubscribed")