From c8a5e13c89b73da8747c7e411fdc096d9f477956 Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Thu, 27 Jul 2023 18:13:10 -0700 Subject: [PATCH] Brevo: add inbound support (Also adds "responses" to test requirements, for mocking fetches of Brevo inbound attachments.) Closes #322 --- CHANGELOG.rst | 3 + anymail/urls.py | 10 +- anymail/webhooks/sendinblue.py | 134 +++++++++++++++- docs/esps/brevo.rst | 26 +++- docs/esps/index.rst | 2 +- tests/requirements.txt | 1 + tests/test_sendinblue_inbound.py | 246 ++++++++++++++++++++++++++++++ tests/test_sendinblue_webhooks.py | 13 ++ 8 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 tests/test_sendinblue_inbound.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7ec1607..cd17413 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -50,6 +50,9 @@ Features `email.message.EmailMessage`, which provides improved compatibility with email standards. (Thanks to `@martinezleoml`_.) +* **Brevo (Sendinblue):** Add support for inbound email. (See + `docs `_.) + Deprecations ~~~~~~~~~~~~ diff --git a/anymail/urls.py b/anymail/urls.py index 083708d..2952d27 100644 --- a/anymail/urls.py +++ b/anymail/urls.py @@ -14,7 +14,10 @@ from .webhooks.mandrill import MandrillCombinedWebhookView from .webhooks.postal import PostalInboundWebhookView, PostalTrackingWebhookView from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView from .webhooks.sendgrid import SendGridInboundWebhookView, SendGridTrackingWebhookView -from .webhooks.sendinblue import SendinBlueTrackingWebhookView +from .webhooks.sendinblue import ( + SendinBlueInboundWebhookView, + SendinBlueTrackingWebhookView, +) from .webhooks.sparkpost import ( SparkPostInboundWebhookView, SparkPostTrackingWebhookView, @@ -61,6 +64,11 @@ urlpatterns = [ SendGridInboundWebhookView.as_view(), name="sendgrid_inbound_webhook", ), + path( + "sendinblue/inbound/", + SendinBlueInboundWebhookView.as_view(), + name="sendinblue_inbound_webhook", + ), path( "sparkpost/inbound/", SparkPostInboundWebhookView.as_view(), diff --git a/anymail/webhooks/sendinblue.py b/anymail/webhooks/sendinblue.py index 47c36a2..0fd63ba 100644 --- a/anymail/webhooks/sendinblue.py +++ b/anymail/webhooks/sendinblue.py @@ -1,18 +1,41 @@ import json from datetime import datetime, timezone +from email.utils import unquote +from urllib.parse import quote, urljoin -from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking +import requests + +from ..exceptions import AnymailConfigurationError +from ..inbound import AnymailInboundMessage +from ..signals import ( + AnymailInboundEvent, + AnymailTrackingEvent, + EventType, + RejectReason, + inbound, + tracking, +) +from ..utils import get_anymail_setting from .base import AnymailBaseWebhookView -class SendinBlueTrackingWebhookView(AnymailBaseWebhookView): +class SendinBlueBaseWebhookView(AnymailBaseWebhookView): + esp_name = "SendinBlue" + + +class SendinBlueTrackingWebhookView(SendinBlueBaseWebhookView): """Handler for SendinBlue delivery and engagement tracking webhooks""" - esp_name = "SendinBlue" signal = tracking def parse_events(self, request): esp_event = json.loads(request.body.decode("utf-8")) + if "items" in esp_event: + # This is an inbound webhook post + raise AnymailConfigurationError( + "You seem to have set SendinBlue's *inbound* webhook URL " + "to Anymail's SendinBlue *tracking* webhook URL." + ) return [self.esp_to_anymail_event(esp_event)] # SendinBlue's webhook payload data doesn't seem to be documented anywhere. @@ -88,3 +111,108 @@ class SendinBlueTrackingWebhookView(AnymailBaseWebhookView): user_agent=None, click_url=esp_event.get("link"), ) + + +class SendinBlueInboundWebhookView(SendinBlueBaseWebhookView): + """Handler for SendinBlue inbound email webhooks""" + + signal = inbound + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # API is required to fetch inbound attachment content: + self.api_key = get_anymail_setting( + "api_key", + esp_name=self.esp_name, + kwargs=kwargs, + allow_bare=True, + ) + self.api_url = get_anymail_setting( + "api_url", + esp_name=self.esp_name, + kwargs=kwargs, + default="https://api.brevo.com/v3/", + ) + if not self.api_url.endswith("/"): + self.api_url += "/" + + def parse_events(self, request): + payload = json.loads(request.body.decode("utf-8")) + try: + esp_events = payload["items"] + except KeyError: + # This is not n inbound webhook post + raise AnymailConfigurationError( + "You seem to have set SendinBlue's *tracking* webhook URL " + "to Anymail's SendinBlue *inbound* webhook URL." + ) + else: + return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events] + + def esp_to_anymail_event(self, esp_event): + # Inbound event's "Uuid" is documented as + # "A list of recipients UUID (can be used with the Public API)". + # In practice, it seems to be a single-item list (even when sending + # to multiple inbound recipients at once) that uniquely identifies this + # inbound event. (And works as a param for the /inbound/events/{uuid} API + # that will "Fetch all events history for one particular received email.") + try: + event_id = esp_event["Uuid"][0] + except (KeyError, IndexError): + event_id = None + + attachments = [ + self._fetch_attachment(attachment) + for attachment in esp_event.get("Attachments", []) + ] + headers = [ + (name, value) + for name, values in esp_event.get("Headers", {}).items() + # values is string if single header instance, list of string if multiple + for value in ([values] if isinstance(values, str) else values) + ] + + # (esp_event From, To, Cc, ReplyTo, Subject, Date, etc. are also in Headers) + message = AnymailInboundMessage.construct( + headers=headers, + text=esp_event.get("RawTextBody", ""), + html=esp_event.get("RawHtmlBody", ""), + attachments=attachments, + ) + + if message["Return-Path"]: + message.envelope_sender = unquote(message["Return-Path"]) + if message["Delivered-To"]: + message.envelope_recipient = unquote(message["Delivered-To"]) + message.stripped_text = esp_event.get("ExtractedMarkdownMessage") + + # Documented as "Spam.Score" object, but both example payload + # and actual received payload use single "SpamScore" field: + message.spam_score = esp_event.get("SpamScore") + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + timestamp=None, # Brevo doesn't provide inbound event timestamp + event_id=event_id, + esp_event=esp_event, + message=message, + ) + + def _fetch_attachment(self, attachment): + # Download attachment content from SendinBlue API. + # FUTURE: somehow defer download until attachment is accessed? + token = attachment["DownloadToken"] + url = urljoin(self.api_url, f"inbound/attachments/{quote(token, safe='')}") + response = requests.get(url, headers={"api-key": self.api_key}) + response.raise_for_status() # or maybe just log and continue? + + content = response.content + # Prefer response Content-Type header to attachment ContentType field, + # as the header will include charset but the ContentType field won't. + content_type = response.headers.get("Content-Type") or attachment["ContentType"] + return AnymailInboundMessage.construct_attachment( + content_type=content_type, + content=content, + filename=attachment.get("Name"), + content_id=attachment.get("ContentID"), + ) diff --git a/docs/esps/brevo.rst b/docs/esps/brevo.rst index 09f8eef..4f4ebb2 100644 --- a/docs/esps/brevo.rst +++ b/docs/esps/brevo.rst @@ -288,7 +288,29 @@ a `dict` of raw webhook data received from Brevo. Inbound webhook --------------- -Anymail does not currently support `Brevo's inbound parsing`_. +.. versionadded:: 10.1 -.. _Brevo's inbound parsing: +If you want to receive email from Brevo through Anymail's normalized +:ref:`inbound ` handling, follow Brevo's `Inbound parsing webhooks`_ +guide to enable inbound service and add Anymail's inbound webhook. + +At the "Creating the webhook" step, set the ``"url"`` param to: + + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/sendinblue/inbound/` + + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret + * *yoursite.example.com* is your Django site + +Brevo does not currently seem to have a dashboard for managing or monitoring +inbound service. However, you can run API calls directly from their documentation +by entering your API key in "Header" field above the example, and then clicking +"Try It!". The `webhooks management APIs`_ and `inbound events list API`_ can +be helpful for diagnosing inbound issues. + + +.. _Inbound parsing webhooks: https://developers.brevo.com/docs/inbound-parse-webhooks +.. _webhooks management APIs: + https://developers.brevo.com/reference/getwebhooks-1 +.. _inbound events list API: + https://developers.brevo.com/reference/getinboundemailevents diff --git a/docs/esps/index.rst b/docs/esps/index.rst index 6b92e1a..420e7d8 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -58,7 +58,7 @@ Email Service Provider |Amazon SES| |Brevo| |MailerSend |AnymailTrackingEvent| from webhooks Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes .. rubric:: :ref:`Inbound handling ` ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ -|AnymailInboundEvent| from webhooks Yes No Yes Yes Yes Yes Yes Yes Yes Yes +|AnymailInboundEvent| from webhooks Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes ============================================ ============ ======= ============ =========== ========== =========== ========== ========== ========== =========== diff --git a/tests/requirements.txt b/tests/requirements.txt index 1d35b7c..8a04fe6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1 +1,2 @@ # Additional packages needed only for running tests +responses diff --git a/tests/test_sendinblue_inbound.py b/tests/test_sendinblue_inbound.py new file mode 100644 index 0000000..0e7e65e --- /dev/null +++ b/tests/test_sendinblue_inbound.py @@ -0,0 +1,246 @@ +from unittest.mock import ANY + +import responses +from django.test import override_settings, tag +from responses.matchers import header_matcher + +from anymail.exceptions import AnymailConfigurationError +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.sendinblue import SendinBlueInboundWebhookView + +from .utils import sample_email_content, sample_image_content +from .webhook_cases import WebhookTestCase + + +@tag("sendinblue") +@override_settings(ANYMAIL_SENDINBLUE_API_KEY="test-api-key") +class SendinBlueInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + # Actual (sanitized) Brevo inbound message payload 7/2023 + raw_event = { + "Uuid": ["aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"], + "MessageId": "", + "InReplyTo": None, + "From": {"Name": "Sender Name", "Address": "from@example.com"}, + "To": [{"Name": None, "Address": "test@anymail.dev"}], + "Cc": [{"Name": None, "Address": "test+cc@anymail.dev"}], + "ReplyTo": None, + "SentAtDate": "Mon, 17 Jul 2023 11:11:22 -0700", + "Subject": "Testing Brevo inbound", + "Attachments": [], + "Headers": { + # Headers that appear more than once arrive as lists: + "Received": [ + "by outbound.example.com for ; ...", + "from [10.10.1.22] by smtp.example.com for ; ...", + ], + # Single appearance headers arrive as strings: + "DKIM-Signature": "v=1; a=rsa-sha256; d=example.com; ...", + "MIME-Version": "1.0", + "From": "Sender Name ", + "Date": "Mon, 17 Jul 2023 11:11:22 -0700", + "Message-ID": "", + "Subject": "Testing Brevo inbound", + "To": "test@anymail.dev", + "Cc": "test+cc@anymail.dev", + "Content-Type": "multipart/alternative", + }, + "SpamScore": 2.9, + "ExtractedMarkdownMessage": "This is a test message. \n", + "ExtractedMarkdownSignature": "- Sender \n", + "RawHtmlBody": '
This is a test message.

- Mike

\r\n', # NOQA: E501 + "RawTextBody": "This is a *test message*.\r\n\n- Sender\r\n", + } + + response = self.client.post( + "/anymail/sendinblue/inbound/", + content_type="application/json", + data={"items": [raw_event]}, + ) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.inbound_handler, + sender=SendinBlueInboundWebhookView, + event=ANY, + esp_name="SendinBlue", + ) + # AnymailInboundEvent + event = kwargs["event"] + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, "inbound") + # Brevo doesn't provide inbound event timestamp + self.assertIsNone(event.timestamp) + self.assertEqual(event.event_id, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") + self.assertIsInstance(event.message, AnymailInboundMessage) + self.assertEqual(event.esp_event, raw_event) + + # AnymailInboundMessage - convenience properties + message = event.message + + self.assertEqual(message.from_email.display_name, "Sender Name") + self.assertEqual(message.from_email.addr_spec, "from@example.com") + self.assertEqual( + [str(e) for e in message.to], + ["test@anymail.dev"], + ) + self.assertEqual([str(e) for e in message.cc], ["test+cc@anymail.dev"]) + self.assertEqual(message.subject, "Testing Brevo inbound") + self.assertEqual(message.date.isoformat(" "), "2023-07-17 11:11:22-07:00") + self.assertEqual(message.text, "This is a *test message*.\r\n\n- Sender\r\n") + self.assertEqual( + message.html, + '
This is a test message.

- Mike

\r\n', # NOQA: E501 + ) + + self.assertIsNone(message.envelope_sender) + self.assertIsNone(message.envelope_recipient) + + # Treat Brevo's ExtractedMarkdownMessage as stripped_text: + self.assertEqual(message.stripped_text, "This is a test message. \n") + # Brevo doesn't provide stripped html: + self.assertIsNone(message.stripped_html) + self.assertIsNone(message.spam_detected) + self.assertEqual(message.spam_score, 2.9) + + # AnymailInboundMessage - other headers + self.assertEqual(message["Message-ID"], "") + self.assertEqual( + message.get_all("Received"), + [ + "by outbound.example.com for ; ...", + "from [10.10.1.22] by smtp.example.com for ; ...", + ], + ) + + def test_envelope_attrs(self): + # Brevo's example payload shows Return-Path and Delivered-To headers. + # They don't seem to be present in our tests, but handle them if they're there: + raw_event = { + "Headers": { + "Return-Path": "", + "Delivered-To": "recipient@example.org", + } + } + self.client.post( + "/anymail/sendinblue/inbound/", + content_type="application/json", + data={"items": [raw_event]}, + ) + kwargs = self.assert_handler_called_once_with( + self.inbound_handler, + sender=SendinBlueInboundWebhookView, + event=ANY, + esp_name="SendinBlue", + ) + event = kwargs["event"] + message = event.message + self.assertEqual(message.envelope_sender, "sender@example.com") + self.assertEqual(message.envelope_recipient, "recipient@example.org") + + @responses.activate + def test_attachments(self): + text_content = "Une pièce jointe" + image_content = sample_image_content() + email_content = sample_email_content() + + raw_event = { + # ... much of payload omitted ... + "Headers": { + "Content-Type": "multipart/mixed", + }, + "Attachments": [ + { + "Name": "test.txt", + "ContentType": "text/plain", + "ContentLength": len(text_content), + "ContentID": None, + "DownloadToken": "download-token-text", + }, + { + "Name": "image.png", + "ContentType": "image/png", + "ContentLength": len(image_content), + "ContentID": "abc123", + "DownloadToken": "download-token-image", + }, + { + "Name": "", + "ContentType": "message/rfc822", + "ContentLength": len(email_content), + "ContentID": None, + "DownloadToken": "download-token-email", + }, + ], + } + + # Brevo supplies a "DownloadToken" that must be used to fetch + # attachment content. Mock those fetches: + match_api_key = header_matcher({"api-key": "test-api-key"}) + responses.add( + responses.GET, + "https://api.brevo.com/v3/inbound/attachments/download-token-text", + match=[match_api_key], + content_type="text/plain; charset=iso-8859-1", + headers={"content-disposition": 'attachment; filename="test.txt"'}, + body=text_content.encode("iso-8859-1"), + ) + responses.add( + responses.GET, + "https://api.brevo.com/v3/inbound/attachments/download-token-image", + match=[match_api_key], + content_type="image/png", + headers={"content-disposition": 'attachment; filename="image.png"'}, + body=image_content, + ) + responses.add( + responses.GET, + "https://api.brevo.com/v3/inbound/attachments/download-token-email", + match=[match_api_key], + content_type="message/rfc822; charset=us-ascii", + headers={"content-disposition": "attachment"}, + body=email_content, + ) + + response = self.client.post( + "/anymail/sendinblue/inbound/", + content_type="application/json", + data={"items": [raw_event]}, + ) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.inbound_handler, + sender=SendinBlueInboundWebhookView, + event=ANY, + esp_name="SendinBlue", + ) + event = kwargs["event"] + message = event.message + attachments = message.attachments # AnymailInboundMessage convenience accessor + self.assertEqual(len(attachments), 2) + self.assertEqual(attachments[0].get_filename(), "test.txt") + self.assertEqual(attachments[0].get_content_type(), "text/plain") + self.assertEqual(attachments[0].get_content_text(), "Une pièce jointe") + self.assertEqual(attachments[1].get_content_type(), "message/rfc822") + self.assertEqualIgnoringHeaderFolding( + attachments[1].get_content_bytes(), email_content + ) + + inlines = message.content_id_map + self.assertEqual(len(inlines), 1) + inline = inlines["abc123"] + self.assertEqual(inline.get_filename(), "image.png") + self.assertEqual(inline.get_content_type(), "image/png") + self.assertEqual(inline.get_content_bytes(), image_content) + + def test_misconfigured_tracking(self): + errmsg = ( + "You seem to have set SendinBlue's *tracking* webhook URL" + " to Anymail's SendinBlue *inbound* webhook URL." + ) + with self.assertRaisesMessage(AnymailConfigurationError, errmsg): + self.client.post( + "/anymail/sendinblue/inbound/", + content_type="application/json", + data={"event": "delivered"}, + ) diff --git a/tests/test_sendinblue_webhooks.py b/tests/test_sendinblue_webhooks.py index 6c64272..ce02651 100644 --- a/tests/test_sendinblue_webhooks.py +++ b/tests/test_sendinblue_webhooks.py @@ -4,6 +4,7 @@ from unittest.mock import ANY from django.test import tag +from anymail.exceptions import AnymailConfigurationError from anymail.signals import AnymailTrackingEvent from anymail.webhooks.sendinblue import SendinBlueTrackingWebhookView @@ -390,3 +391,15 @@ class SendinBlueDeliveryTestCase(WebhookTestCase): ) event = kwargs["event"] self.assertEqual(event.event_type, "unsubscribed") + + def test_misconfigured_inbound(self): + errmsg = ( + "You seem to have set SendinBlue's *inbound* webhook URL" + " to Anymail's SendinBlue *tracking* webhook URL." + ) + with self.assertRaisesMessage(AnymailConfigurationError, errmsg): + self.client.post( + "/anymail/sendinblue/tracking/", + content_type="application/json", + data={"items": []}, + )