Brevo: add inbound support

(Also adds "responses" to test requirements,
for mocking fetches of Brevo inbound
attachments.)

Closes #322

CHANGELOG.rst

@@ -50,6 +50,9 @@ Features
   `email.message.EmailMessage`, which provides improved compatibility with
   email standards. (Thanks to `@martinezleoml`_.)
 
+* **Brevo (Sendinblue):** Add support for inbound email. (See
+  `docs <https://anymail.dev/en/latest/esps/sendinblue/#sendinblue-inbound>`_.)
+
 
 Deprecations
 ~~~~~~~~~~~~

anymail/urls.py

@@ -14,7 +14,10 @@ from .webhooks.mandrill import MandrillCombinedWebhookView
 from .webhooks.postal import PostalInboundWebhookView, PostalTrackingWebhookView
 from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView
 from .webhooks.sendgrid import SendGridInboundWebhookView, SendGridTrackingWebhookView
-from .webhooks.sendinblue import SendinBlueTrackingWebhookView
+from .webhooks.sendinblue import (
+    SendinBlueInboundWebhookView,
+    SendinBlueTrackingWebhookView,
+)
 from .webhooks.sparkpost import (
     SparkPostInboundWebhookView,
     SparkPostTrackingWebhookView,
@@ -61,6 +64,11 @@ urlpatterns = [
         SendGridInboundWebhookView.as_view(),
         name="sendgrid_inbound_webhook",
     ),
+    path(
+        "sendinblue/inbound/",
+        SendinBlueInboundWebhookView.as_view(),
+        name="sendinblue_inbound_webhook",
+    ),
     path(
         "sparkpost/inbound/",
         SparkPostInboundWebhookView.as_view(),

anymail/webhooks/sendinblue.py

@@ -1,18 +1,41 @@
 import json
 from datetime import datetime, timezone
+from email.utils import unquote
+from urllib.parse import quote, urljoin
 
-from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking
+import requests
+
+from ..exceptions import AnymailConfigurationError
+from ..inbound import AnymailInboundMessage
+from ..signals import (
+    AnymailInboundEvent,
+    AnymailTrackingEvent,
+    EventType,
+    RejectReason,
+    inbound,
+    tracking,
+)
+from ..utils import get_anymail_setting
 from .base import AnymailBaseWebhookView
 
 
-class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
+class SendinBlueBaseWebhookView(AnymailBaseWebhookView):
+    esp_name = "SendinBlue"
+
+
+class SendinBlueTrackingWebhookView(SendinBlueBaseWebhookView):
     """Handler for SendinBlue delivery and engagement tracking webhooks"""
 
-    esp_name = "SendinBlue"
     signal = tracking
 
     def parse_events(self, request):
         esp_event = json.loads(request.body.decode("utf-8"))
+        if "items" in esp_event:
+            # This is an inbound webhook post
+            raise AnymailConfigurationError(
+                "You seem to have set SendinBlue's *inbound* webhook URL "
+                "to Anymail's SendinBlue *tracking* webhook URL."
+            )
         return [self.esp_to_anymail_event(esp_event)]
 
     # SendinBlue's webhook payload data doesn't seem to be documented anywhere.
@@ -88,3 +111,108 @@ class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
             user_agent=None,
             click_url=esp_event.get("link"),
         )
+
+
+class SendinBlueInboundWebhookView(SendinBlueBaseWebhookView):
+    """Handler for SendinBlue inbound email webhooks"""
+
+    signal = inbound
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # API is required to fetch inbound attachment content:
+        self.api_key = get_anymail_setting(
+            "api_key",
+            esp_name=self.esp_name,
+            kwargs=kwargs,
+            allow_bare=True,
+        )
+        self.api_url = get_anymail_setting(
+            "api_url",
+            esp_name=self.esp_name,
+            kwargs=kwargs,
+            default="https://api.brevo.com/v3/",
+        )
+        if not self.api_url.endswith("/"):
+            self.api_url += "/"
+
+    def parse_events(self, request):
+        payload = json.loads(request.body.decode("utf-8"))
+        try:
+            esp_events = payload["items"]
+        except KeyError:
+            # This is not n inbound webhook post
+            raise AnymailConfigurationError(
+                "You seem to have set SendinBlue's *tracking* webhook URL "
+                "to Anymail's SendinBlue *inbound* webhook URL."
+            )
+        else:
+            return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
+
+    def esp_to_anymail_event(self, esp_event):
+        # Inbound event's "Uuid" is documented as
+        # "A list of recipients UUID (can be used with the Public API)".
+        # In practice, it seems to be a single-item list (even when sending
+        # to multiple inbound recipients at once) that uniquely identifies this
+        # inbound event. (And works as a param for the /inbound/events/{uuid} API
+        # that will "Fetch all events history for one particular received email.")
+        try:
+            event_id = esp_event["Uuid"][0]
+        except (KeyError, IndexError):
+            event_id = None
+
+        attachments = [
+            self._fetch_attachment(attachment)
+            for attachment in esp_event.get("Attachments", [])
+        ]
+        headers = [
+            (name, value)
+            for name, values in esp_event.get("Headers", {}).items()
+            # values is string if single header instance, list of string if multiple
+            for value in ([values] if isinstance(values, str) else values)
+        ]
+
+        # (esp_event From, To, Cc, ReplyTo, Subject, Date, etc. are also in Headers)
+        message = AnymailInboundMessage.construct(
+            headers=headers,
+            text=esp_event.get("RawTextBody", ""),
+            html=esp_event.get("RawHtmlBody", ""),
+            attachments=attachments,
+        )
+
+        if message["Return-Path"]:
+            message.envelope_sender = unquote(message["Return-Path"])
+        if message["Delivered-To"]:
+            message.envelope_recipient = unquote(message["Delivered-To"])
+        message.stripped_text = esp_event.get("ExtractedMarkdownMessage")
+
+        # Documented as "Spam.Score" object, but both example payload
+        # and actual received payload use single "SpamScore" field:
+        message.spam_score = esp_event.get("SpamScore")
+
+        return AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=None,  # Brevo doesn't provide inbound event timestamp
+            event_id=event_id,
+            esp_event=esp_event,
+            message=message,
+        )
+
+    def _fetch_attachment(self, attachment):
+        # Download attachment content from SendinBlue API.
+        # FUTURE: somehow defer download until attachment is accessed?
+        token = attachment["DownloadToken"]
+        url = urljoin(self.api_url, f"inbound/attachments/{quote(token, safe='')}")
+        response = requests.get(url, headers={"api-key": self.api_key})
+        response.raise_for_status()  # or maybe just log and continue?
+
+        content = response.content
+        # Prefer response Content-Type header to attachment ContentType field,
+        # as the header will include charset but the ContentType field won't.
+        content_type = response.headers.get("Content-Type") or attachment["ContentType"]
+        return AnymailInboundMessage.construct_attachment(
+            content_type=content_type,
+            content=content,
+            filename=attachment.get("Name"),
+            content_id=attachment.get("ContentID"),
+        )

docs/esps/brevo.rst

@@ -288,7 +288,29 @@ a `dict` of raw webhook data received from Brevo.
 Inbound webhook
 ---------------
 
-Anymail does not currently support `Brevo's inbound parsing`_.
+.. versionadded:: 10.1
 
-.. _Brevo's inbound parsing:
+If you want to receive email from Brevo through Anymail's normalized
+:ref:`inbound <inbound>` handling, follow Brevo's `Inbound parsing webhooks`_
+guide to enable inbound service and add Anymail's inbound webhook.
+
+At the "Creating the webhook" step, set the ``"url"`` param to:
+
+   :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/sendinblue/inbound/`
+
+     * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret
+     * *yoursite.example.com* is your Django site
+
+Brevo does not currently seem to have a dashboard for managing or monitoring
+inbound service. However, you can run API calls directly from their documentation
+by entering your API key in "Header" field above the example, and then clicking
+"Try It!". The `webhooks management APIs`_ and `inbound events list API`_ can
+be helpful for diagnosing inbound issues.
+
+
+.. _Inbound parsing webhooks:
     https://developers.brevo.com/docs/inbound-parse-webhooks
+.. _webhooks management APIs:
+    https://developers.brevo.com/reference/getwebhooks-1
+.. _inbound events list API:
+    https://developers.brevo.com/reference/getinboundemailevents

docs/esps/index.rst

@@ -58,7 +58,7 @@ Email Service Provider                        |Amazon SES|  |Brevo|  |MailerSend
 |AnymailTrackingEvent| from webhooks          Yes           Yes      Yes           Yes          Yes         Yes          Yes         Yes         Yes         Yes
 .. rubric:: :ref:`Inbound handling <inbound>`
 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-|AnymailInboundEvent| from webhooks           Yes           No       Yes           Yes          Yes         Yes          Yes         Yes         Yes         Yes
+|AnymailInboundEvent| from webhooks           Yes           Yes      Yes           Yes          Yes         Yes          Yes         Yes         Yes         Yes
 ============================================  ============  =======  ============  ===========  ==========  ===========  ==========  ==========  ==========  ===========
 
 

tests/requirements.txt

@@ -1 +1,2 @@
 # Additional packages needed only for running tests
+responses

tests/test_sendinblue_inbound.py

@@ -0,0 +1,246 @@
+from unittest.mock import ANY
+
+import responses
+from django.test import override_settings, tag
+from responses.matchers import header_matcher
+
+from anymail.exceptions import AnymailConfigurationError
+from anymail.inbound import AnymailInboundMessage
+from anymail.signals import AnymailInboundEvent
+from anymail.webhooks.sendinblue import SendinBlueInboundWebhookView
+
+from .utils import sample_email_content, sample_image_content
+from .webhook_cases import WebhookTestCase
+
+
+@tag("sendinblue")
+@override_settings(ANYMAIL_SENDINBLUE_API_KEY="test-api-key")
+class SendinBlueInboundTestCase(WebhookTestCase):
+    def test_inbound_basics(self):
+        # Actual (sanitized) Brevo inbound message payload 7/2023
+        raw_event = {
+            "Uuid": ["aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"],
+            "MessageId": "<ABCDE12345@mail.example.com>",
+            "InReplyTo": None,
+            "From": {"Name": "Sender Name", "Address": "from@example.com"},
+            "To": [{"Name": None, "Address": "test@anymail.dev"}],
+            "Cc": [{"Name": None, "Address": "test+cc@anymail.dev"}],
+            "ReplyTo": None,
+            "SentAtDate": "Mon, 17 Jul 2023 11:11:22 -0700",
+            "Subject": "Testing Brevo inbound",
+            "Attachments": [],
+            "Headers": {
+                # Headers that appear more than once arrive as lists:
+                "Received": [
+                    "by outbound.example.com for <test@anymail.dev>; ...",
+                    "from [10.10.1.22] by smtp.example.com for <test@anymail.dev>; ...",
+                ],
+                # Single appearance headers arrive as strings:
+                "DKIM-Signature": "v=1; a=rsa-sha256; d=example.com; ...",
+                "MIME-Version": "1.0",
+                "From": "Sender Name <from@example.com>",
+                "Date": "Mon, 17 Jul 2023 11:11:22 -0700",
+                "Message-ID": "<ABCDE12345@mail.example.com>",
+                "Subject": "Testing Brevo inbound",
+                "To": "test@anymail.dev",
+                "Cc": "test+cc@anymail.dev",
+                "Content-Type": "multipart/alternative",
+            },
+            "SpamScore": 2.9,
+            "ExtractedMarkdownMessage": "This is a test message.  \n",
+            "ExtractedMarkdownSignature": "- Sender  \n",
+            "RawHtmlBody": '<div dir="ltr">This is a <u>test message</u>.<div><br></div><div>- Mike</div><div><br></div></div>\r\n',  # NOQA: E501
+            "RawTextBody": "This is a *test message*.\r\n\n- Sender\r\n",
+        }
+
+        response = self.client.post(
+            "/anymail/sendinblue/inbound/",
+            content_type="application/json",
+            data={"items": [raw_event]},
+        )
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(
+            self.inbound_handler,
+            sender=SendinBlueInboundWebhookView,
+            event=ANY,
+            esp_name="SendinBlue",
+        )
+        # AnymailInboundEvent
+        event = kwargs["event"]
+        self.assertIsInstance(event, AnymailInboundEvent)
+        self.assertEqual(event.event_type, "inbound")
+        # Brevo doesn't provide inbound event timestamp
+        self.assertIsNone(event.timestamp)
+        self.assertEqual(event.event_id, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")
+        self.assertIsInstance(event.message, AnymailInboundMessage)
+        self.assertEqual(event.esp_event, raw_event)
+
+        # AnymailInboundMessage - convenience properties
+        message = event.message
+
+        self.assertEqual(message.from_email.display_name, "Sender Name")
+        self.assertEqual(message.from_email.addr_spec, "from@example.com")
+        self.assertEqual(
+            [str(e) for e in message.to],
+            ["test@anymail.dev"],
+        )
+        self.assertEqual([str(e) for e in message.cc], ["test+cc@anymail.dev"])
+        self.assertEqual(message.subject, "Testing Brevo inbound")
+        self.assertEqual(message.date.isoformat(" "), "2023-07-17 11:11:22-07:00")
+        self.assertEqual(message.text, "This is a *test message*.\r\n\n- Sender\r\n")
+        self.assertEqual(
+            message.html,
+            '<div dir="ltr">This is a <u>test message</u>.<div><br></div><div>- Mike</div><div><br></div></div>\r\n',  # NOQA: E501
+        )
+
+        self.assertIsNone(message.envelope_sender)
+        self.assertIsNone(message.envelope_recipient)
+
+        # Treat Brevo's ExtractedMarkdownMessage as stripped_text:
+        self.assertEqual(message.stripped_text, "This is a test message.  \n")
+        # Brevo doesn't provide stripped html:
+        self.assertIsNone(message.stripped_html)
+        self.assertIsNone(message.spam_detected)
+        self.assertEqual(message.spam_score, 2.9)
+
+        # AnymailInboundMessage - other headers
+        self.assertEqual(message["Message-ID"], "<ABCDE12345@mail.example.com>")
+        self.assertEqual(
+            message.get_all("Received"),
+            [
+                "by outbound.example.com for <test@anymail.dev>; ...",
+                "from [10.10.1.22] by smtp.example.com for <test@anymail.dev>; ...",
+            ],
+        )
+
+    def test_envelope_attrs(self):
+        # Brevo's example payload shows Return-Path and Delivered-To headers.
+        # They don't seem to be present in our tests, but handle them if they're there:
+        raw_event = {
+            "Headers": {
+                "Return-Path": "<sender@example.com>",
+                "Delivered-To": "recipient@example.org",
+            }
+        }
+        self.client.post(
+            "/anymail/sendinblue/inbound/",
+            content_type="application/json",
+            data={"items": [raw_event]},
+        )
+        kwargs = self.assert_handler_called_once_with(
+            self.inbound_handler,
+            sender=SendinBlueInboundWebhookView,
+            event=ANY,
+            esp_name="SendinBlue",
+        )
+        event = kwargs["event"]
+        message = event.message
+        self.assertEqual(message.envelope_sender, "sender@example.com")
+        self.assertEqual(message.envelope_recipient, "recipient@example.org")
+
+    @responses.activate
+    def test_attachments(self):
+        text_content = "Une pièce jointe"
+        image_content = sample_image_content()
+        email_content = sample_email_content()
+
+        raw_event = {
+            # ... much of payload omitted ...
+            "Headers": {
+                "Content-Type": "multipart/mixed",
+            },
+            "Attachments": [
+                {
+                    "Name": "test.txt",
+                    "ContentType": "text/plain",
+                    "ContentLength": len(text_content),
+                    "ContentID": None,
+                    "DownloadToken": "download-token-text",
+                },
+                {
+                    "Name": "image.png",
+                    "ContentType": "image/png",
+                    "ContentLength": len(image_content),
+                    "ContentID": "abc123",
+                    "DownloadToken": "download-token-image",
+                },
+                {
+                    "Name": "",
+                    "ContentType": "message/rfc822",
+                    "ContentLength": len(email_content),
+                    "ContentID": None,
+                    "DownloadToken": "download-token-email",
+                },
+            ],
+        }
+
+        # Brevo supplies a "DownloadToken" that must be used to fetch
+        # attachment content. Mock those fetches:
+        match_api_key = header_matcher({"api-key": "test-api-key"})
+        responses.add(
+            responses.GET,
+            "https://api.brevo.com/v3/inbound/attachments/download-token-text",
+            match=[match_api_key],
+            content_type="text/plain; charset=iso-8859-1",
+            headers={"content-disposition": 'attachment; filename="test.txt"'},
+            body=text_content.encode("iso-8859-1"),
+        )
+        responses.add(
+            responses.GET,
+            "https://api.brevo.com/v3/inbound/attachments/download-token-image",
+            match=[match_api_key],
+            content_type="image/png",
+            headers={"content-disposition": 'attachment; filename="image.png"'},
+            body=image_content,
+        )
+        responses.add(
+            responses.GET,
+            "https://api.brevo.com/v3/inbound/attachments/download-token-email",
+            match=[match_api_key],
+            content_type="message/rfc822; charset=us-ascii",
+            headers={"content-disposition": "attachment"},
+            body=email_content,
+        )
+
+        response = self.client.post(
+            "/anymail/sendinblue/inbound/",
+            content_type="application/json",
+            data={"items": [raw_event]},
+        )
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(
+            self.inbound_handler,
+            sender=SendinBlueInboundWebhookView,
+            event=ANY,
+            esp_name="SendinBlue",
+        )
+        event = kwargs["event"]
+        message = event.message
+        attachments = message.attachments  # AnymailInboundMessage convenience accessor
+        self.assertEqual(len(attachments), 2)
+        self.assertEqual(attachments[0].get_filename(), "test.txt")
+        self.assertEqual(attachments[0].get_content_type(), "text/plain")
+        self.assertEqual(attachments[0].get_content_text(), "Une pièce jointe")
+        self.assertEqual(attachments[1].get_content_type(), "message/rfc822")
+        self.assertEqualIgnoringHeaderFolding(
+            attachments[1].get_content_bytes(), email_content
+        )
+
+        inlines = message.content_id_map
+        self.assertEqual(len(inlines), 1)
+        inline = inlines["abc123"]
+        self.assertEqual(inline.get_filename(), "image.png")
+        self.assertEqual(inline.get_content_type(), "image/png")
+        self.assertEqual(inline.get_content_bytes(), image_content)
+
+    def test_misconfigured_tracking(self):
+        errmsg = (
+            "You seem to have set SendinBlue's *tracking* webhook URL"
+            " to Anymail's SendinBlue *inbound* webhook URL."
+        )
+        with self.assertRaisesMessage(AnymailConfigurationError, errmsg):
+            self.client.post(
+                "/anymail/sendinblue/inbound/",
+                content_type="application/json",
+                data={"event": "delivered"},
+            )

tests/test_sendinblue_webhooks.py

@@ -4,6 +4,7 @@ from unittest.mock import ANY
 
 from django.test import tag
 
+from anymail.exceptions import AnymailConfigurationError
 from anymail.signals import AnymailTrackingEvent
 from anymail.webhooks.sendinblue import SendinBlueTrackingWebhookView
 
@@ -390,3 +391,15 @@ class SendinBlueDeliveryTestCase(WebhookTestCase):
         )
         event = kwargs["event"]
         self.assertEqual(event.event_type, "unsubscribed")
+
+    def test_misconfigured_inbound(self):
+        errmsg = (
+            "You seem to have set SendinBlue's *inbound* webhook URL"
+            " to Anymail's SendinBlue *tracking* webhook URL."
+        )
+        with self.assertRaisesMessage(AnymailConfigurationError, errmsg):
+            self.client.post(
+                "/anymail/sendinblue/tracking/",
+                content_type="application/json",
+                data={"items": []},
+            )