pacnpal/django-anymail
1
0
Fork 0
mirror of https://github.com/pacnpal/django-anymail.git synced 2025-12-20 03:41:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Reformat code with automated tools

Browse Source 
Apply standardized code style
This commit is contained in:
medmunds
2023-02-06 12:27:43 -08:00
committed by Mike Edmunds
parent 40891fcb4a
commit b4e22c63b3
94 changed files with 12936 additions and 7443 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
anymail/webhooks/amazon_ses.py
View File

@@ -5,24 +5,40 @@ from base64 import b64decode
from django.http import HttpResponse
from django.utils.dateparse import parse_datetime
from .base import AnymailBaseWebhookView
from ..exceptions import (
AnymailAPIError, AnymailConfigurationError, AnymailImproperlyInstalled, AnymailWebhookValidationFailure,
_LazyError)
AnymailAPIError,
AnymailConfigurationError,
AnymailImproperlyInstalled,
AnymailWebhookValidationFailure,
_LazyError,
)
from ..inbound import AnymailInboundMessage
from ..signals import AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason, inbound, tracking
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from ..utils import get_anymail_setting, getfirst
from .base import AnymailBaseWebhookView
try:
import boto3
from botocore.exceptions import ClientError
from ..backends.amazon_ses import _get_anymail_boto3_params
except ImportError:
# This module gets imported by anymail.urls, so don't complain about boto3 missing
# unless one of the Amazon SES webhook views is actually used and needs it
boto3 = _LazyError(AnymailImproperlyInstalled(missing_package='boto3', backend='amazon_ses'))
boto3 = _LazyError(
AnymailImproperlyInstalled(missing_package="boto3", backend="amazon_ses")
)
ClientError = object
_get_anymail_boto3_params = _LazyError(AnymailImproperlyInstalled(missing_package='boto3', backend='amazon_ses'))
_get_anymail_boto3_params = _LazyError(
AnymailImproperlyInstalled(missing_package="boto3", backend="amazon_ses")
)
class AmazonSESBaseWebhookView(AnymailBaseWebhookView):
@@ -31,23 +47,32 @@ class AmazonSESBaseWebhookView(AnymailBaseWebhookView):
esp_name = "Amazon SES"
def __init__(self, **kwargs):
# whether to automatically respond to SNS SubscriptionConfirmation requests; default True
# (Future: could also take a TopicArn or list to auto-confirm)
# whether to automatically respond to SNS SubscriptionConfirmation requests;
# default True. (Future: could also take a TopicArn or list to auto-confirm)
self.auto_confirm_enabled = get_anymail_setting(
"auto_confirm_sns_subscriptions", esp_name=self.esp_name, kwargs=kwargs, default=True)
# boto3 params for connecting to S3 (inbound downloads) and SNS (auto-confirm subscriptions):
self.session_params, self.client_params = _get_anymail_boto3_params(kwargs=kwargs)
"auto_confirm_sns_subscriptions",
esp_name=self.esp_name,
kwargs=kwargs,
default=True,
)
# boto3 params for connecting to S3 (inbound downloads)
# and SNS (auto-confirm subscriptions):
self.session_params, self.client_params = _get_anymail_boto3_params(
kwargs=kwargs
)
super().__init__(**kwargs)
@staticmethod
def _parse_sns_message(request):
# cache so we don't have to parse the json multiple times
if not hasattr(request, '_sns_message'):
if not hasattr(request, "_sns_message"):
try:
body = request.body.decode(request.encoding or 'utf-8')
body = request.body.decode(request.encoding or "utf-8")
request._sns_message = json.loads(body)
except (TypeError, ValueError, UnicodeDecodeError) as err:
raise AnymailAPIError("Malformed SNS message body %r" % request.body) from err
raise AnymailAPIError(
"Malformed SNS message body %r" % request.body
) from err
return request._sns_message
def validate_request(self, request):
@@ -57,18 +82,24 @@ class AmazonSESBaseWebhookView(AnymailBaseWebhookView):
body_type = sns_message.get("Type", "<<missing>>")
if header_type != body_type:
raise AnymailWebhookValidationFailure(
'SNS header "x-amz-sns-message-type: %s" doesn\'t match body "Type": "%s"'
% (header_type, body_type))
'SNS header "x-amz-sns-message-type: %s"'
' doesn\'t match body "Type": "%s"' % (header_type, body_type)
)
if header_type not in ["Notification", "SubscriptionConfirmation", "UnsubscribeConfirmation"]:
if header_type not in [
"Notification",
"SubscriptionConfirmation",
"UnsubscribeConfirmation",
]:
raise AnymailAPIError("Unknown SNS message type '%s'" % header_type)
header_id = request.META.get("HTTP_X_AMZ_SNS_MESSAGE_ID", "<<missing>>")
body_id = sns_message.get("MessageId", "<<missing>>")
if header_id != body_id:
raise AnymailWebhookValidationFailure(
'SNS header "x-amz-sns-message-id: %s" doesn\'t match body "MessageId": "%s"'
% (header_id, body_id))
'SNS header "x-amz-sns-message-id: %s"'
' doesn\'t match body "MessageId": "%s"' % (header_id, body_id)
)
# Future: Verify SNS message signature
# https://docs.aws.amazon.com/sns/latest/dg/SendMessageToHttp.verify.signature.html
@@ -76,7 +107,8 @@ class AmazonSESBaseWebhookView(AnymailBaseWebhookView):
def post(self, request, *args, **kwargs):
# request has *not* yet been validated at this point
if self.basic_auth and not request.META.get("HTTP_AUTHORIZATION"):
# Amazon SNS requires a proper 401 response before it will attempt to send basic auth
# Amazon SNS requires a proper 401 response
# before it will attempt to send basic auth
response = HttpResponse(status=401)
response["WWW-Authenticate"] = 'Basic realm="Anymail WEBHOOK_SECRET"'
return response
@@ -92,10 +124,16 @@ class AmazonSESBaseWebhookView(AnymailBaseWebhookView):
try:
ses_event = json.loads(message_string)
except (TypeError, ValueError) as err:
if message_string == "Successfully validated SNS topic for Amazon SES event publishing.":
pass # this Notification is generated after SubscriptionConfirmation
if (
"Successfully validated SNS topic for Amazon SES event publishing."
== message_string
):
# this Notification is generated after SubscriptionConfirmation
pass
else:
raise AnymailAPIError("Unparsable SNS Message %r" % message_string) from err
raise AnymailAPIError(
"Unparsable SNS Message %r" % message_string
) from err
else:
events = self.esp_to_anymail_events(ses_event, sns_message)
elif sns_type == "SubscriptionConfirmation":
@@ -107,43 +145,63 @@ class AmazonSESBaseWebhookView(AnymailBaseWebhookView):
raise NotImplementedError()
def auto_confirm_sns_subscription(self, sns_message):
"""Automatically accept a subscription to Amazon SNS topics, if the request is expected.
"""
Automatically accept a subscription to Amazon SNS topics,
if the request is expected.
If an SNS SubscriptionConfirmation arrives with HTTP basic auth proving it is meant for us,
automatically load the SubscribeURL to confirm the subscription.
If an SNS SubscriptionConfirmation arrives with HTTP basic auth proving it is
meant for us, automatically load the SubscribeURL to confirm the subscription.
"""
if not self.auto_confirm_enabled:
return
if not self.basic_auth:
# Note: basic_auth (shared secret) confirms the notification was meant for us.
# If WEBHOOK_SECRET isn't set, Anymail logs a warning but allows the request.
# (Also, verifying the SNS message signature would be insufficient here:
# if someone else tried to point their own SNS topic at our webhook url,
# SNS would send a SubscriptionConfirmation with a valid Amazon signature.)
# basic_auth (shared secret) confirms the notification was meant for us.
# If WEBHOOK_SECRET isn't set, Anymail logs a warning but allows the
# request. (Also, verifying the SNS message signature would be insufficient
# here: if someone else tried to point their own SNS topic at our webhook
# url, SNS would send a SubscriptionConfirmation with a valid Amazon
# signature.)
raise AnymailWebhookValidationFailure(
"Anymail received an unexpected SubscriptionConfirmation request for Amazon SNS topic "
"'{topic_arn!s}'. (Anymail can automatically confirm SNS subscriptions if you set a "
"WEBHOOK_SECRET and use that in your SNS notification url. Or you can manually confirm "
"this subscription in the SNS dashboard with token '{token!s}'.)"
"".format(topic_arn=sns_message.get('TopicArn'), token=sns_message.get('Token')))
"Anymail received an unexpected SubscriptionConfirmation request for "
"Amazon SNS topic '{topic_arn!s}'. (Anymail can automatically confirm "
"SNS subscriptions if you set a WEBHOOK_SECRET and use that in your "
"SNS notification url. Or you can manually confirm this subscription "
"in the SNS dashboard with token '{token!s}'.)".format(
topic_arn=sns_message.get("TopicArn"),
token=sns_message.get("Token"),
)
)
# WEBHOOK_SECRET *is* set, so the request's basic auth has been verified by now (in run_validators).
# We're good to confirm...
# WEBHOOK_SECRET *is* set, so the request's basic auth has been verified by now
# (in run_validators). We're good to confirm...
topic_arn = sns_message["TopicArn"]
token = sns_message["Token"]
# Must confirm in TopicArn's own region (which may be different from the default)
# Must confirm in TopicArn's own region
# (which may be different from the default)
try:
(_arn_tag, _partition, _service, region, _account, _resource) = topic_arn.split(":", maxsplit=6)
(
_arn_tag,
_partition,
_service,
region,
_account,
_resource,
) = topic_arn.split(":", maxsplit=6)
except (TypeError, ValueError):
raise ValueError("Invalid ARN format '{topic_arn!s}'".format(topic_arn=topic_arn))
raise ValueError(
"Invalid ARN format '{topic_arn!s}'".format(topic_arn=topic_arn)
)
client_params = self.client_params.copy()
client_params["region_name"] = region
sns_client = boto3.session.Session(**self.session_params).client('sns', **client_params)
sns_client = boto3.session.Session(**self.session_params).client(
"sns", **client_params
)
sns_client.confirm_subscription(
TopicArn=topic_arn, Token=token, AuthenticateOnUnsubscribe='true')
TopicArn=topic_arn, Token=token, AuthenticateOnUnsubscribe="true"
)
class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
@@ -153,16 +211,19 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
def esp_to_anymail_events(self, ses_event, sns_message):
# Amazon SES has two notification formats, which are almost exactly the same:
# - https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
# - https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html
# https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
# https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html
# This code should handle either.
ses_event_type = getfirst(ses_event, ["eventType", "notificationType"], "<<type missing>>")
ses_event_type = getfirst(
ses_event, ["eventType", "notificationType"], "<<type missing>>"
)
if ses_event_type == "Received":
# This is an inbound event
raise AnymailConfigurationError(
"You seem to have set an Amazon SES *inbound* receipt rule to publish "
"to an SNS Topic that posts to Anymail's *tracking* webhook URL. "
"(SNS TopicArn %s)" % sns_message.get("TopicArn"))
"(SNS TopicArn %s)" % sns_message.get("TopicArn")
)
event_id = sns_message.get("MessageId") # unique to the SNS notification
try:
@@ -171,7 +232,8 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
timestamp = None
mail_object = ses_event.get("mail", {})
message_id = mail_object.get("messageId") # same as MessageId in SendRawEmail response
# same as MessageId in SendRawEmail response:
message_id = mail_object.get("messageId")
all_recipients = mail_object.get("destination", [])
# Recover tags and metadata from custom headers
@@ -187,7 +249,8 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
except (ValueError, TypeError, KeyError):
pass
common_props = dict( # AnymailTrackingEvent props for all recipients
# AnymailTrackingEvent props for all recipients:
common_props = dict(
esp_event=ses_event,
event_id=event_id,
message_id=message_id,
@@ -195,12 +258,13 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
tags=tags,
timestamp=timestamp,
)
per_recipient_props = [ # generate individual events for each of these
dict(recipient=email_address)
for email_address in all_recipients
# generate individual events for each of these:
per_recipient_props = [
dict(recipient=email_address) for email_address in all_recipients
]
event_object = ses_event.get(ses_event_type.lower(), {}) # e.g., ses_event["bounce"]
# event-type-specific data (e.g., ses_event["bounce"]):
event_object = ses_event.get(ses_event_type.lower(), {})
if ses_event_type == "Bounce":
common_props.update(
@@ -208,10 +272,13 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
description="{bounceType}: {bounceSubType}".format(**event_object),
reject_reason=RejectReason.BOUNCED,
)
per_recipient_props = [dict(
recipient=recipient["emailAddress"],
mta_response=recipient.get("diagnosticCode"),
) for recipient in event_object["bouncedRecipients"]]
per_recipient_props = [
dict(
recipient=recipient["emailAddress"],
mta_response=recipient.get("diagnosticCode"),
)
for recipient in event_object["bouncedRecipients"]
]
elif ses_event_type == "Complaint":
common_props.update(
event_type=EventType.COMPLAINED,
@@ -219,17 +286,18 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
reject_reason=RejectReason.SPAM,
user_agent=event_object.get("userAgent"),
)
per_recipient_props = [dict(
recipient=recipient["emailAddress"],
) for recipient in event_object["complainedRecipients"]]
per_recipient_props = [
dict(recipient=recipient["emailAddress"])
for recipient in event_object["complainedRecipients"]
]
elif ses_event_type == "Delivery":
common_props.update(
event_type=EventType.DELIVERED,
mta_response=event_object.get("smtpResponse"),
)
per_recipient_props = [dict(
recipient=recipient,
) for recipient in event_object["recipients"]]
per_recipient_props = [
dict(recipient=recipient) for recipient in event_object["recipients"]
]
elif ses_event_type == "Send":
common_props.update(
event_type=EventType.SENT,
@@ -256,7 +324,8 @@ class AmazonSESTrackingWebhookView(AmazonSESBaseWebhookView):
click_url=event_object.get("link"),
)
elif ses_event_type == "Rendering Failure":
event_object = ses_event["failure"] # rather than ses_event["rendering failure"]
# (this type doesn't follow usual event_object naming)
event_object = ses_event["failure"]
common_props.update(
event_type=EventType.FAILED,
description=event_object["errorMessage"],
@@ -285,8 +354,9 @@ class AmazonSESInboundWebhookView(AmazonSESBaseWebhookView):
# This is not an inbound event
raise AnymailConfigurationError(
"You seem to have set an Amazon SES *sending* event or notification "
"to publish to an SNS Topic that posts to Anymail's *inbound* webhook URL. "
"(SNS TopicArn %s)" % sns_message.get("TopicArn"))
"to publish to an SNS Topic that posts to Anymail's *inbound* webhook "
"URL. (SNS TopicArn %s)" % sns_message.get("TopicArn")
)
receipt_object = ses_event.get("receipt", {})
action_object = receipt_object.get("action", {})
@@ -301,11 +371,13 @@ class AmazonSESInboundWebhookView(AmazonSESBaseWebhookView):
else:
message = AnymailInboundMessage.parse_raw_mime(content)
elif action_type == "S3":
# download message from s3 into memory, then parse
# (SNS has 15s limit for an http response; hope download doesn't take that long)
# download message from s3 into memory, then parse. (SNS has 15s limit
# for an http response; hope download doesn't take that long)
bucket_name = action_object["bucketName"]
object_key = action_object["objectKey"]
s3 = boto3.session.Session(**self.session_params).client("s3", **self.client_params)
s3 = boto3.session.Session(**self.session_params).client(
"s3", **self.client_params
)
content = io.BytesIO()
try:
s3.download_fileobj(bucket_name, object_key, content)
@@ -314,46 +386,62 @@ class AmazonSESInboundWebhookView(AmazonSESBaseWebhookView):
except ClientError as err:
# improve the botocore error message
raise AnymailBotoClientAPIError(
"Anymail AmazonSESInboundWebhookView couldn't download S3 object '{bucket_name}:{object_key}'"
"Anymail AmazonSESInboundWebhookView couldn't download"
" S3 object '{bucket_name}:{object_key}'"
"".format(bucket_name=bucket_name, object_key=object_key),
client_error=err) from err
client_error=err,
) from err
finally:
content.close()
else:
raise AnymailConfigurationError(
"Anymail's Amazon SES inbound webhook works only with 'SNS' or 'S3' receipt rule actions, "
"not SNS notifications for {action_type!s} actions. (SNS TopicArn {topic_arn!s})"
"".format(action_type=action_type, topic_arn=sns_message.get("TopicArn")))
"Anymail's Amazon SES inbound webhook works only with 'SNS' or 'S3'"
" receipt rule actions, not SNS notifications for {action_type!s}"
" actions. (SNS TopicArn {topic_arn!s})"
"".format(
action_type=action_type, topic_arn=sns_message.get("TopicArn")
)
)
message.envelope_sender = mail_object.get("source") # "the envelope MAIL FROM address"
# "the envelope MAIL FROM address":
message.envelope_sender = mail_object.get("source")
try:
# "recipients that were matched by the active receipt rule"
message.envelope_recipient = receipt_object["recipients"][0]
except (KeyError, TypeError, IndexError):
pass
spam_status = receipt_object.get("spamVerdict", {}).get("status", "").upper()
message.spam_detected = {"PASS": False, "FAIL": True}.get(spam_status) # else None if unsure
# spam_detected = False if no spam, True if spam, or None if unsure:
message.spam_detected = {"PASS": False, "FAIL": True}.get(spam_status)
event_id = mail_object.get("messageId") # "unique ID assigned to the email by Amazon SES"
# "unique ID assigned to the email by Amazon SES":
event_id = mail_object.get("messageId")
try:
timestamp = parse_datetime(mail_object["timestamp"]) # "time at which the email was received"
# "time at which the email was received":
timestamp = parse_datetime(mail_object["timestamp"])
except (KeyError, ValueError):
timestamp = None
return [AnymailInboundEvent(
event_type=EventType.INBOUND,
event_id=event_id,
message=message,
timestamp=timestamp,
esp_event=ses_event,
)]
return [
AnymailInboundEvent(
event_type=EventType.INBOUND,
event_id=event_id,
message=message,
timestamp=timestamp,
esp_event=ses_event,
)
]
class AnymailBotoClientAPIError(AnymailAPIError, ClientError):
"""An AnymailAPIError that is also a Boto ClientError"""
def __init__(self, *args, client_error):
assert isinstance(client_error, ClientError)
# init self as boto ClientError (which doesn't cooperatively subclass):
super().__init__(error_response=client_error.response, operation_name=client_error.operation_name)
super().__init__(
error_response=client_error.response,
operation_name=client_error.operation_name,
)
# emulate AnymailError init:
self.args = args

41
anymail/webhooks/base.py
View File

@@ -7,7 +7,7 @@ from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from ..exceptions import AnymailInsecureWebhookWarning, AnymailWebhookValidationFailure
from ..utils import get_anymail_setting, collect_all_methods, get_request_basic_auth
from ..utils import collect_all_methods, get_anymail_setting, get_request_basic_auth
# Mixin note: Django's View.__init__ doesn't cooperate with chaining,
@@ -25,7 +25,7 @@ class AnymailCoreWebhookView(View):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.validators = collect_all_methods(self.__class__, 'validate_request')
self.validators = collect_all_methods(self.__class__, "validate_request")
# Subclass implementation:
@@ -99,8 +99,10 @@ class AnymailCoreWebhookView(View):
esp_name = "Postmark"
esp_name = "SendGrid" # (use ESP's preferred capitalization)
"""
raise NotImplementedError("%s.%s must declare esp_name class attr" %
(self.__class__.__module__, self.__class__.__name__))
raise NotImplementedError(
"%s.%s must declare esp_name class attr"
% (self.__class__.__module__, self.__class__.__name__)
)
class AnymailBasicAuthMixin(AnymailCoreWebhookView):
@@ -113,11 +115,16 @@ class AnymailBasicAuthMixin(AnymailCoreWebhookView):
warn_if_no_basic_auth = True
# List of allowable HTTP basic-auth 'user:pass' strings.
basic_auth = None # (Declaring class attr allows override by kwargs in View.as_view.)
# (Declaring class attr allows override by kwargs in View.as_view.):
basic_auth = None
def __init__(self, **kwargs):
self.basic_auth = get_anymail_setting('webhook_secret', default=[],
kwargs=kwargs) # no esp_name -- auth is shared between ESPs
self.basic_auth = get_anymail_setting(
"webhook_secret",
default=[],
# no esp_name -- auth is shared between ESPs
kwargs=kwargs,
)
# Allow a single string:
if isinstance(self.basic_auth, str):
@@ -127,25 +134,31 @@ class AnymailBasicAuthMixin(AnymailCoreWebhookView):
"Your Anymail webhooks are insecure and open to anyone on the web. "
"You should set WEBHOOK_SECRET in your ANYMAIL settings. "
"See 'Securing webhooks' in the Anymail docs.",
AnymailInsecureWebhookWarning)
AnymailInsecureWebhookWarning,
)
super().__init__(**kwargs)
def validate_request(self, request):
"""If configured for webhook basic auth, validate request has correct auth."""
if self.basic_auth:
request_auth = get_request_basic_auth(request)
# Use constant_time_compare to avoid timing attack on basic auth. (It's OK that any()
# can terminate early: we're not trying to protect how many auth strings are allowed,
# just the contents of each individual auth string.)
auth_ok = any(constant_time_compare(request_auth, allowed_auth)
for allowed_auth in self.basic_auth)
# Use constant_time_compare to avoid timing attack on basic auth. (It's OK
# that any() can terminate early: we're not trying to protect how many auth
# strings are allowed, just the contents of each individual auth string.)
auth_ok = any(
constant_time_compare(request_auth, allowed_auth)
for allowed_auth in self.basic_auth
)
if not auth_ok:
raise AnymailWebhookValidationFailure(
"Missing or invalid basic auth in Anymail %s webhook" % self.esp_name)
"Missing or invalid basic auth in Anymail %s webhook"
% self.esp_name
)
class AnymailBaseWebhookView(AnymailBasicAuthMixin, AnymailCoreWebhookView):
"""
Abstract base class for most webhook views, enforcing HTTP basic auth security
"""
pass

411
anymail/webhooks/mailgun.py
View File

@@ -1,15 +1,32 @@
import hashlib
import hmac
import json
from datetime import datetime, timezone
import hashlib
import hmac
from django.utils.crypto import constant_time_compare
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailConfigurationError, AnymailWebhookValidationFailure, AnymailInvalidAddress
from ..exceptions import (
AnymailConfigurationError,
AnymailInvalidAddress,
AnymailWebhookValidationFailure,
)
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
from ..utils import get_anymail_setting, combine, querydict_getfirst, parse_single_address, UNSET
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from ..utils import (
UNSET,
combine,
get_anymail_setting,
parse_single_address,
querydict_getfirst,
)
from .base import AnymailBaseWebhookView
class MailgunBaseWebhookView(AnymailBaseWebhookView):
@@ -18,18 +35,30 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView):
esp_name = "Mailgun"
warn_if_no_basic_auth = False # because we validate against signature
webhook_signing_key = None # (Declaring class attr allows override by kwargs in View.as_view.)
# (Declaring class attr allows override by kwargs in View.as_view.)
webhook_signing_key = None
# The `api_key` attribute name is still allowed for compatibility with earlier Anymail releases.
# The `api_key` attribute name is still allowed for compatibility
# with earlier Anymail releases.
api_key = None # (Declaring class attr allows override by kwargs in View.as_view.)
def __init__(self, **kwargs):
# webhook_signing_key: falls back to api_key if webhook_signing_key not provided
api_key = get_anymail_setting('api_key', esp_name=self.esp_name,
kwargs=kwargs, allow_bare=True, default=None)
webhook_signing_key = get_anymail_setting('webhook_signing_key', esp_name=self.esp_name,
kwargs=kwargs, default=UNSET if api_key is None else api_key)
self.webhook_signing_key = webhook_signing_key.encode('ascii') # hmac.new requires bytes key
api_key = get_anymail_setting(
"api_key",
esp_name=self.esp_name,
kwargs=kwargs,
allow_bare=True,
default=None,
)
webhook_signing_key = get_anymail_setting(
"webhook_signing_key",
esp_name=self.esp_name,
kwargs=kwargs,
default=UNSET if api_key is None else api_key,
)
# hmac.new requires bytes key:
self.webhook_signing_key = webhook_signing_key.encode("ascii")
super().__init__(**kwargs)
def validate_request(self, request):
@@ -37,30 +66,38 @@ class MailgunBaseWebhookView(AnymailBaseWebhookView):
if request.content_type == "application/json":
# New-style webhook: json payload with separate signature block
try:
event = json.loads(request.body.decode('utf-8'))
signature_block = event['signature']
token = signature_block['token']
timestamp = signature_block['timestamp']
signature = signature_block['signature']
event = json.loads(request.body.decode("utf-8"))
signature_block = event["signature"]
token = signature_block["token"]
timestamp = signature_block["timestamp"]
signature = signature_block["signature"]
except (KeyError, ValueError, UnicodeDecodeError) as err:
raise AnymailWebhookValidationFailure(
"Mailgun webhook called with invalid payload format") from err
"Mailgun webhook called with invalid payload format"
) from err
else:
# Legacy webhook: signature fields are interspersed with other POST data
try:
# Must use the *last* value of these fields if there are conflicting merged user-variables.
# (Fortunately, Django QueryDict is specced to return the last value.)
token = request.POST['token']
timestamp = request.POST['timestamp']
signature = request.POST['signature']
# Must use the *last* value of these fields if there are conflicting
# merged user-variables. (Fortunately, Django QueryDict is specced to
# return the last value.)
token = request.POST["token"]
timestamp = request.POST["timestamp"]
signature = request.POST["signature"]
except KeyError as err:
raise AnymailWebhookValidationFailure(
"Mailgun webhook called without required security fields") from err
"Mailgun webhook called without required security fields"
) from err
expected_signature = hmac.new(key=self.webhook_signing_key, msg='{}{}'.format(timestamp, token).encode('ascii'),
digestmod=hashlib.sha256).hexdigest()
expected_signature = hmac.new(
key=self.webhook_signing_key,
msg="{}{}".format(timestamp, token).encode("ascii"),
digestmod=hashlib.sha256,
).hexdigest()
if not constant_time_compare(signature, expected_signature):
raise AnymailWebhookValidationFailure("Mailgun webhook called with incorrect signature")
raise AnymailWebhookValidationFailure(
"Mailgun webhook called with incorrect signature"
)
class MailgunTrackingWebhookView(MailgunBaseWebhookView):
@@ -70,75 +107,82 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
def parse_events(self, request):
if request.content_type == "application/json":
esp_event = json.loads(request.body.decode('utf-8'))
esp_event = json.loads(request.body.decode("utf-8"))
return [self.esp_to_anymail_event(esp_event)]
else:
return [self.mailgun_legacy_to_anymail_event(request.POST)]
event_types = {
# Map Mailgun event: Anymail normalized type
'accepted': EventType.QUEUED, # not delivered to webhooks (8/2018)
'rejected': EventType.REJECTED,
'delivered': EventType.DELIVERED,
'failed': EventType.BOUNCED,
'opened': EventType.OPENED,
'clicked': EventType.CLICKED,
'unsubscribed': EventType.UNSUBSCRIBED,
'complained': EventType.COMPLAINED,
"accepted": EventType.QUEUED, # not delivered to webhooks (8/2018)
"rejected": EventType.REJECTED,
"delivered": EventType.DELIVERED,
"failed": EventType.BOUNCED,
"opened": EventType.OPENED,
"clicked": EventType.CLICKED,
"unsubscribed": EventType.UNSUBSCRIBED,
"complained": EventType.COMPLAINED,
}
reject_reasons = {
# Map Mailgun event_data.reason: Anymail normalized RejectReason
# (these appear in webhook doc examples, but aren't actually documented anywhere)
# Map Mailgun event_data.reason: Anymail normalized RejectReason (these appear
# in webhook doc examples, but aren't actually documented anywhere)
"bounce": RejectReason.BOUNCED,
"suppress-bounce": RejectReason.BOUNCED,
"generic": RejectReason.OTHER, # ??? appears to be used for any temporary failure?
# ??? "generic" appears to be used for any temporary failure?
"generic": RejectReason.OTHER,
}
severities = {
# Remap some event types based on "severity" payload field
(EventType.BOUNCED, 'temporary'): EventType.DEFERRED
(EventType.BOUNCED, "temporary"): EventType.DEFERRED
}
def esp_to_anymail_event(self, esp_event):
event_data = esp_event.get('event-data', {})
event_data = esp_event.get("event-data", {})
event_type = self.event_types.get(event_data['event'], EventType.UNKNOWN)
event_type = self.event_types.get(event_data["event"], EventType.UNKNOWN)
event_type = self.severities.get((EventType.BOUNCED, event_data.get('severity')), event_type)
event_type = self.severities.get(
(EventType.BOUNCED, event_data.get("severity")), event_type
)
# Use signature.token for event_id, rather than event_data.id,
# because the latter is only "guaranteed to be unique within a day".
event_id = esp_event.get('signature', {}).get('token')
event_id = esp_event.get("signature", {}).get("token")
recipient = event_data.get('recipient')
recipient = event_data.get("recipient")
try:
timestamp = datetime.fromtimestamp(float(event_data['timestamp']), tz=timezone.utc)
timestamp = datetime.fromtimestamp(
float(event_data["timestamp"]), tz=timezone.utc
)
except KeyError:
timestamp = None
try:
message_id = event_data['message']['headers']['message-id']
message_id = event_data["message"]["headers"]["message-id"]
except KeyError:
message_id = None
if message_id and not message_id.startswith('<'):
if message_id and not message_id.startswith("<"):
message_id = "<{}>".format(message_id)
metadata = event_data.get('user-variables', {})
tags = event_data.get('tags', [])
metadata = event_data.get("user-variables", {})
tags = event_data.get("tags", [])
try:
delivery_status = event_data['delivery-status']
delivery_status = event_data["delivery-status"]
except KeyError:
description = None
mta_response = None
else:
description = delivery_status.get('description')
mta_response = delivery_status.get('message')
description = delivery_status.get("description")
mta_response = delivery_status.get("message")
if 'reason' in event_data:
reject_reason = self.reject_reasons.get(event_data['reason'], RejectReason.OTHER)
if "reason" in event_data:
reject_reason = self.reject_reasons.get(
event_data["reason"], RejectReason.OTHER
)
else:
reject_reason = None
@@ -149,7 +193,8 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
if not recipient:
try:
to_email = parse_single_address(
event_data["message"]["headers"]["to"])
event_data["message"]["headers"]["to"]
)
except (AnymailInvalidAddress, KeyError):
pass
else:
@@ -166,8 +211,8 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
mta_response=mta_response,
tags=tags,
metadata=metadata,
click_url=event_data.get('url'),
user_agent=event_data.get('client-info', {}).get('user-agent'),
click_url=event_data.get("url"),
user_agent=event_data.get("client-info", {}).get("user-agent"),
esp_event=esp_event,
)
@@ -176,13 +221,13 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
legacy_event_types = {
# Map Mailgun event: Anymail normalized type
'delivered': EventType.DELIVERED,
'dropped': EventType.REJECTED,
'bounced': EventType.BOUNCED,
'complained': EventType.COMPLAINED,
'unsubscribed': EventType.UNSUBSCRIBED,
'opened': EventType.OPENED,
'clicked': EventType.CLICKED,
"delivered": EventType.DELIVERED,
"dropped": EventType.REJECTED,
"bounced": EventType.BOUNCED,
"complained": EventType.COMPLAINED,
"unsubscribed": EventType.UNSUBSCRIBED,
"opened": EventType.OPENED,
"clicked": EventType.CLICKED,
# Mailgun does not send events corresponding to QUEUED or DEFERRED
}
@@ -190,7 +235,8 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
# Map Mailgun (SMTP) error codes to Anymail normalized reject_reason.
# By default, we will treat anything 400-599 as REJECT_BOUNCED
# so only exceptions are listed here.
499: RejectReason.TIMED_OUT, # unable to connect to MX (also covers invalid recipients)
499: RejectReason.TIMED_OUT, # unable to connect to MX
# (499 also covers invalid recipients)
# These 6xx codes appear to be Mailgun extensions to SMTP
# (and don't seem to be documented anywhere):
605: RejectReason.BOUNCED, # previous bounce
@@ -205,123 +251,163 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
# to avoid potential conflicting user-data.
esp_event.getfirst = querydict_getfirst.__get__(esp_event)
if 'event' not in esp_event and 'sender' in esp_event:
if "event" not in esp_event and "sender" in esp_event:
# Inbound events don't (currently) have an event field
raise AnymailConfigurationError(
"You seem to have set Mailgun's *inbound* route "
"to Anymail's Mailgun *tracking* webhook URL.")
"to Anymail's Mailgun *tracking* webhook URL."
)
event_type = self.legacy_event_types.get(esp_event.getfirst('event'), EventType.UNKNOWN)
timestamp = datetime.fromtimestamp(
int(esp_event['timestamp']), tz=timezone.utc) # use *last* value of timestamp
event_type = self.legacy_event_types.get(
esp_event.getfirst("event"), EventType.UNKNOWN
)
# use *last* value of timestamp:
timestamp = datetime.fromtimestamp(int(esp_event["timestamp"]), tz=timezone.utc)
# Message-Id is not documented for every event, but seems to always be included.
# (It's sometimes spelled as 'message-id', lowercase, and missing the <angle-brackets>.)
message_id = esp_event.getfirst('Message-Id', None) or esp_event.getfirst('message-id', None)
if message_id and not message_id.startswith('<'):
# (It's sometimes spelled as 'message-id', lowercase, and missing the
# <angle-brackets>.)
message_id = esp_event.getfirst("Message-Id", None) or esp_event.getfirst(
"message-id", None
)
if message_id and not message_id.startswith("<"):
message_id = "<{}>".format(message_id)
description = esp_event.getfirst('description', None)
mta_response = esp_event.getfirst('error', None) or esp_event.getfirst('notification', None)
description = esp_event.getfirst("description", None)
mta_response = esp_event.getfirst("error", None) or esp_event.getfirst(
"notification", None
)
reject_reason = None
try:
mta_status = int(esp_event.getfirst('code'))
mta_status = int(esp_event.getfirst("code"))
except (KeyError, TypeError):
pass
except ValueError:
# RFC-3463 extended SMTP status code (class.subject.detail, where class is "2", "4" or "5")
# RFC-3463 extended SMTP status code
# (class.subject.detail, where class is "2", "4" or "5")
try:
status_class = esp_event.getfirst('code').split('.')[0]
status_class = esp_event.getfirst("code").split(".")[0]
except (TypeError, IndexError):
# illegal SMTP status code format
pass
else:
reject_reason = RejectReason.BOUNCED if status_class in ("4", "5") else RejectReason.OTHER
reject_reason = (
RejectReason.BOUNCED
if status_class in ("4", "5")
else RejectReason.OTHER
)
else:
reject_reason = self.legacy_reject_reasons.get(
mta_status,
RejectReason.BOUNCED if 400 <= mta_status < 600
else RejectReason.OTHER)
RejectReason.BOUNCED if 400 <= mta_status < 600 else RejectReason.OTHER,
)
metadata = self._extract_legacy_metadata(esp_event)
# tags are supposed to be in 'tag' fields, but are sometimes in undocumented X-Mailgun-Tag
tags = esp_event.getlist('tag', None) or esp_event.getlist('X-Mailgun-Tag', [])
# tags are supposed to be in 'tag' fields,
# but are sometimes in undocumented X-Mailgun-Tag
tags = esp_event.getlist("tag", None) or esp_event.getlist("X-Mailgun-Tag", [])
return AnymailTrackingEvent(
event_type=event_type,
timestamp=timestamp,
message_id=message_id,
event_id=esp_event.get('token', None), # use *last* value of token
recipient=esp_event.getfirst('recipient', None),
event_id=esp_event.get("token", None), # use *last* value of token
recipient=esp_event.getfirst("recipient", None),
reject_reason=reject_reason,
description=description,
mta_response=mta_response,
tags=tags,
metadata=metadata,
click_url=esp_event.getfirst('url', None),
user_agent=esp_event.getfirst('user-agent', None),
click_url=esp_event.getfirst("url", None),
user_agent=esp_event.getfirst("user-agent", None),
esp_event=esp_event,
)
def _extract_legacy_metadata(self, esp_event):
# Mailgun merges user-variables into the POST fields. If you know which user variable
# you want to retrieve--and it doesn't conflict with a Mailgun event field--that's fine.
# But if you want to extract all user-variables (like we do), it's more complicated...
event_type = esp_event.getfirst('event')
# Mailgun merges user-variables into the POST fields. If you know which user
# variable you want to retrieve--and it doesn't conflict with a Mailgun event
# field--that's fine. But if you want to extract all user-variables (like we
# do), it's more complicated...
event_type = esp_event.getfirst("event")
metadata = {}
if 'message-headers' in esp_event:
# For events where original message headers are available, it's most reliable
# to recover user-variables from the X-Mailgun-Variables header(s).
headers = json.loads(esp_event['message-headers'])
variables = [value for [field, value] in headers if field == 'X-Mailgun-Variables']
if "message-headers" in esp_event:
# For events where original message headers are available, it's most
# reliable to recover user-variables from the X-Mailgun-Variables header(s).
headers = json.loads(esp_event["message-headers"])
variables = [
value for [field, value] in headers if field == "X-Mailgun-Variables"
]
if len(variables) >= 1:
# Each X-Mailgun-Variables value is JSON. Parse and merge them all into single dict:
# Each X-Mailgun-Variables value is JSON. Parse and merge them all into
# single dict:
metadata = combine(*[json.loads(value) for value in variables])
elif event_type in self._known_legacy_event_fields:
# For other events, we must extract from the POST fields, ignoring known Mailgun
# event parameters, and treating all other values as user-variables.
# For other events, we must extract from the POST fields, ignoring known
# Mailgun event parameters, and treating all other values as user-variables.
known_fields = self._known_legacy_event_fields[event_type]
for field, values in esp_event.lists():
if field not in known_fields:
# Unknown fields are assumed to be user-variables. (There should really only be
# a single value, but just in case take the last one to match QueryDict semantics.)
# Unknown fields are assumed to be user-variables. (There should
# really only be a single value, but just in case take the last one
# to match QueryDict semantics.)
metadata[field] = values[-1]
elif field == 'tag':
# There's no way to distinguish a user-variable named 'tag' from an actual tag,
# so don't treat this/these value(s) as metadata.
elif field == "tag":
# There's no way to distinguish a user-variable named 'tag' from
# an actual tag, so don't treat this/these value(s) as metadata.
pass
elif len(values) == 1:
# This is an expected event parameter, and since there's only a single value
# it must be the event param, not metadata.
# This is an expected event parameter, and since there's only a
# single value it must be the event param, not metadata.
pass
else:
# This is an expected event parameter, but there are (at least) two values.
# One is the event param, and the other is a user-variable metadata value.
# Which is which depends on the field:
if field in {'signature', 'timestamp', 'token'}:
metadata[field] = values[0] # values = [user-variable, event-param]
# This is an expected event parameter, but there are (at least) two
# values. One is the event param, and the other is a user-variable
# metadata value. Which is which depends on the field:
if field in {"signature", "timestamp", "token"}:
# values = [user-variable, event-param]
metadata[field] = values[0]
else:
metadata[field] = values[-1] # values = [event-param, user-variable]
# values = [event-param, user-variable]
metadata[field] = values[-1]
return metadata
_common_legacy_event_fields = {
# These fields are documented to appear in all Mailgun opened, clicked and unsubscribed events:
'event', 'recipient', 'domain', 'ip', 'country', 'region', 'city', 'user-agent', 'device-type',
'client-type', 'client-name', 'client-os', 'campaign-id', 'campaign-name', 'tag', 'mailing-list',
'timestamp', 'token', 'signature',
# These fields are documented to appear in all Mailgun
# opened, clicked and unsubscribed events:
"event",
"recipient",
"domain",
"ip",
"country",
"region",
"city",
"user-agent",
"device-type",
"client-type",
"client-name",
"client-os",
"campaign-id",
"campaign-name",
"tag",
"mailing-list",
"timestamp",
"token",
"signature",
# Undocumented, but observed in actual events:
'body-plain', 'h', 'message-id',
"body-plain",
"h",
"message-id",
}
_known_legacy_event_fields = {
# For all Mailgun event types that *don't* include message-headers,
# map Mailgun (not normalized) event type to set of expected event fields.
# Used for metadata extraction.
'clicked': _common_legacy_event_fields | {'url'},
'opened': _common_legacy_event_fields,
'unsubscribed': _common_legacy_event_fields,
"clicked": _common_legacy_event_fields | {"url"},
"opened": _common_legacy_event_fields,
"unsubscribed": _common_legacy_event_fields,
}
@@ -332,57 +418,63 @@ class MailgunInboundWebhookView(MailgunBaseWebhookView):
def parse_events(self, request):
if request.content_type == "application/json":
esp_event = json.loads(request.body.decode('utf-8'))
event_type = esp_event.get('event-data', {}).get('event', '')
esp_event = json.loads(request.body.decode("utf-8"))
event_type = esp_event.get("event-data", {}).get("event", "")
raise AnymailConfigurationError(
"You seem to have set Mailgun's *%s tracking* webhook "
"to Anymail's Mailgun *inbound* webhook URL. "
"(Or Mailgun has changed inbound events to use json.)"
% event_type)
"(Or Mailgun has changed inbound events to use json.)" % event_type
)
return [self.esp_to_anymail_event(request)]
def esp_to_anymail_event(self, request):
# Inbound uses the entire Django request as esp_event, because we need POST and FILES.
# Note that request.POST is case-sensitive (unlike email.message.Message headers).
# Inbound uses the entire Django request as esp_event, because
# we need POST and FILES. Note that request.POST is case-sensitive
# (unlike email.message.Message headers).
esp_event = request
if request.POST.get('event', 'inbound') != 'inbound':
if request.POST.get("event", "inbound") != "inbound":
# (Legacy) tracking event
raise AnymailConfigurationError(
"You seem to have set Mailgun's *%s tracking* webhook "
"to Anymail's Mailgun *inbound* webhook URL." % request.POST['event'])
"to Anymail's Mailgun *inbound* webhook URL." % request.POST["event"]
)
if 'attachments' in request.POST:
if "attachments" in request.POST:
# Inbound route used store() rather than forward().
# ("attachments" seems to be the only POST param that differs between
# store and forward; Anymail could support store by handling the JSON
# attachments param in message_from_mailgun_parsed.)
raise AnymailConfigurationError(
"You seem to have configured Mailgun's receiving route using the store()"
" action. Anymail's inbound webhook requires the forward() action.")
"You seem to have configured Mailgun's receiving route using"
" the store() action. Anymail's inbound webhook requires"
" the forward() action."
)
if 'body-mime' in request.POST:
if "body-mime" in request.POST:
# Raw-MIME
message = AnymailInboundMessage.parse_raw_mime(request.POST['body-mime'])
message = AnymailInboundMessage.parse_raw_mime(request.POST["body-mime"])
else:
# Fully-parsed
message = self.message_from_mailgun_parsed(request)
message.envelope_sender = request.POST.get('sender', None)
message.envelope_recipient = request.POST.get('recipient', None)
message.stripped_text = request.POST.get('stripped-text', None)
message.stripped_html = request.POST.get('stripped-html', None)
message.envelope_sender = request.POST.get("sender", None)
message.envelope_recipient = request.POST.get("recipient", None)
message.stripped_text = request.POST.get("stripped-text", None)
message.stripped_html = request.POST.get("stripped-html", None)
message.spam_detected = message.get('X-Mailgun-Sflag', 'No').lower() == 'yes'
message.spam_detected = message.get("X-Mailgun-Sflag", "No").lower() == "yes"
try:
message.spam_score = float(message['X-Mailgun-Sscore'])
message.spam_score = float(message["X-Mailgun-Sscore"])
except (TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=datetime.fromtimestamp(int(request.POST['timestamp']), tz=timezone.utc),
event_id=request.POST.get('token', None),
timestamp=datetime.fromtimestamp(
int(request.POST["timestamp"]), tz=timezone.utc
),
event_id=request.POST.get("token", None),
esp_event=esp_event,
message=message,
)
@@ -391,35 +483,44 @@ class MailgunInboundWebhookView(MailgunBaseWebhookView):
"""Construct a Message from Mailgun's "fully-parsed" fields"""
# Mailgun transcodes all fields to UTF-8 for "fully parsed" messages
try:
attachment_count = int(request.POST['attachment-count'])
attachment_count = int(request.POST["attachment-count"])
except (KeyError, TypeError):
attachments = None
else:
# Load attachments from posted files: attachment-1, attachment-2, etc.
# content-id-map is {content-id: attachment-id}, identifying which files are inline attachments.
# Invert it to {attachment-id: content-id}, while handling potentially duplicate content-ids.
# content-id-map is {content-id: attachment-id}, identifying which files
# are inline attachments. Invert it to {attachment-id: content-id}, while
# handling potentially duplicate content-ids.
field_to_content_id = json.loads(
request.POST.get('content-id-map', '{}'),
object_pairs_hook=lambda pairs: {att_id: cid for (cid, att_id) in pairs})
request.POST.get("content-id-map", "{}"),
object_pairs_hook=lambda pairs: {
att_id: cid for (cid, att_id) in pairs
},
)
attachments = []
for n in range(1, attachment_count+1):
for n in range(1, attachment_count + 1):
attachment_id = "attachment-%d" % n
try:
file = request.FILES[attachment_id]
except KeyError:
# Django's multipart/form-data handling drops FILES with certain
# filenames (for security) or with empty filenames (Django ticket 15879).
# filenames (for security) or with empty filenames (Django ticket
# 15879).
# (To avoid this problem, use Mailgun's "raw MIME" inbound option.)
pass
else:
content_id = field_to_content_id.get(attachment_id)
attachment = AnymailInboundMessage.construct_attachment_from_uploaded_file(
file, content_id=content_id)
attachment = (
AnymailInboundMessage.construct_attachment_from_uploaded_file(
file, content_id=content_id
)
)
attachments.append(attachment)
return AnymailInboundMessage.construct(
headers=json.loads(request.POST['message-headers']), # includes From, To, Cc, Subject, etc.
text=request.POST.get('body-plain', None),
html=request.POST.get('body-html', None),
# message-headers includes From, To, Cc, Subject, etc.
headers=json.loads(request.POST["message-headers"]),
text=request.POST.get("body-plain", None),
html=request.POST.get("body-html", None),
attachments=attachments,
)

147
anymail/webhooks/mailjet.py
View File

@@ -1,10 +1,16 @@
import json
from datetime import datetime, timezone
from .base import AnymailBaseWebhookView
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from .base import AnymailBaseWebhookView
class MailjetTrackingWebhookView(AnymailBaseWebhookView):
@@ -14,7 +20,7 @@ class MailjetTrackingWebhookView(AnymailBaseWebhookView):
signal = tracking
def parse_events(self, request):
esp_events = json.loads(request.body.decode('utf-8'))
esp_events = json.loads(request.body.decode("utf-8"))
# Mailjet webhook docs say the payload is "a JSON array of event objects,"
# but that's not true if "group events" isn't enabled in webhook config...
try:
@@ -28,65 +34,71 @@ class MailjetTrackingWebhookView(AnymailBaseWebhookView):
# https://dev.mailjet.com/guides/#events
event_types = {
# Map Mailjet event: Anymail normalized type
'sent': EventType.DELIVERED, # accepted by receiving MTA
'open': EventType.OPENED,
'click': EventType.CLICKED,
'bounce': EventType.BOUNCED,
'blocked': EventType.REJECTED,
'spam': EventType.COMPLAINED,
'unsub': EventType.UNSUBSCRIBED,
"sent": EventType.DELIVERED, # accepted by receiving MTA
"open": EventType.OPENED,
"click": EventType.CLICKED,
"bounce": EventType.BOUNCED,
"blocked": EventType.REJECTED,
"spam": EventType.COMPLAINED,
"unsub": EventType.UNSUBSCRIBED,
}
reject_reasons = {
# Map Mailjet error strings to Anymail normalized reject_reason
# error_related_to: recipient
'user unknown': RejectReason.BOUNCED,
'mailbox inactive': RejectReason.BOUNCED,
'quota exceeded': RejectReason.BOUNCED,
'blacklisted': RejectReason.BLOCKED, # might also be previous unsubscribe
'spam reporter': RejectReason.SPAM,
"user unknown": RejectReason.BOUNCED,
"mailbox inactive": RejectReason.BOUNCED,
"quota exceeded": RejectReason.BOUNCED,
"blacklisted": RejectReason.BLOCKED, # might also be previous unsubscribe
"spam reporter": RejectReason.SPAM,
# error_related_to: domain
'invalid domain': RejectReason.BOUNCED,
'no mail host': RejectReason.BOUNCED,
'relay/access denied': RejectReason.BOUNCED,
'greylisted': RejectReason.OTHER, # see special handling below
'typofix': RejectReason.INVALID,
# error_related_to: spam (all Mailjet policy/filtering; see above for spam complaints)
'sender blocked': RejectReason.BLOCKED,
'content blocked': RejectReason.BLOCKED,
'policy issue': RejectReason.BLOCKED,
"invalid domain": RejectReason.BOUNCED,
"no mail host": RejectReason.BOUNCED,
"relay/access denied": RejectReason.BOUNCED,
"greylisted": RejectReason.OTHER, # see special handling below
"typofix": RejectReason.INVALID,
# error_related_to: spam
# (all Mailjet policy/filtering; see above for spam complaints)
"sender blocked": RejectReason.BLOCKED,
"content blocked": RejectReason.BLOCKED,
"policy issue": RejectReason.BLOCKED,
# error_related_to: mailjet
'preblocked': RejectReason.BLOCKED,
'duplicate in campaign': RejectReason.OTHER,
"preblocked": RejectReason.BLOCKED,
"duplicate in campaign": RejectReason.OTHER,
}
def esp_to_anymail_event(self, esp_event):
event_type = self.event_types.get(esp_event['event'], EventType.UNKNOWN)
if esp_event.get('error', None) == 'greylisted' and not esp_event.get('hard_bounce', False):
# "This is a temporary error due to possible unrecognised senders. Delivery will be re-attempted."
event_type = self.event_types.get(esp_event["event"], EventType.UNKNOWN)
if esp_event.get("error", None) == "greylisted" and not esp_event.get(
"hard_bounce", False
):
# "This is a temporary error due to possible unrecognised senders.
# Delivery will be re-attempted."
event_type = EventType.DEFERRED
try:
timestamp = datetime.fromtimestamp(esp_event['time'], tz=timezone.utc)
timestamp = datetime.fromtimestamp(esp_event["time"], tz=timezone.utc)
except (KeyError, ValueError):
timestamp = None
try:
# convert bigint MessageID to str to match backend AnymailRecipientStatus
message_id = str(esp_event['MessageID'])
message_id = str(esp_event["MessageID"])
except (KeyError, TypeError):
message_id = None
if 'error' in esp_event:
reject_reason = self.reject_reasons.get(esp_event['error'], RejectReason.OTHER)
if "error" in esp_event:
reject_reason = self.reject_reasons.get(
esp_event["error"], RejectReason.OTHER
)
else:
reject_reason = None
tag = esp_event.get('customcampaign', None)
tag = esp_event.get("customcampaign", None)
tags = [tag] if tag else []
try:
metadata = json.loads(esp_event['Payload'])
metadata = json.loads(esp_event["Payload"])
except (KeyError, ValueError):
metadata = {}
@@ -95,13 +107,13 @@ class MailjetTrackingWebhookView(AnymailBaseWebhookView):
timestamp=timestamp,
message_id=message_id,
event_id=None,
recipient=esp_event.get('email', None),
recipient=esp_event.get("email", None),
reject_reason=reject_reason,
mta_response=esp_event.get('smtp_reply', None),
mta_response=esp_event.get("smtp_reply", None),
tags=tags,
metadata=metadata,
click_url=esp_event.get('url', None),
user_agent=esp_event.get('agent', None),
click_url=esp_event.get("url", None),
user_agent=esp_event.get("agent", None),
esp_event=esp_event,
)
@@ -113,21 +125,23 @@ class MailjetInboundWebhookView(AnymailBaseWebhookView):
signal = inbound
def parse_events(self, request):
esp_event = json.loads(request.body.decode('utf-8'))
esp_event = json.loads(request.body.decode("utf-8"))
return [self.esp_to_anymail_event(esp_event)]
def esp_to_anymail_event(self, esp_event):
# You could _almost_ reconstruct the raw mime message from Mailjet's Headers and Parts fields,
# but it's not clear which multipart boundary to use on each individual Part. Although each Part's
# Content-Type header still has the multipart boundary, not knowing the parent part means typical
# nested multipart structures can't be reliably recovered from the data Mailjet provides.
# We'll just use our standarized multipart inbound constructor.
# You could _almost_ reconstruct the raw mime message from Mailjet's Headers
# and Parts fields, but it's not clear which multipart boundary to use on each
# individual Part. Although each Part's Content-Type header still has the
# multipart boundary, not knowing the parent part means typical nested multipart
# structures can't be reliably recovered from the data Mailjet provides.
# Just use our standardized multipart inbound constructor.
headers = self._flatten_mailjet_headers(esp_event.get("Headers", {}))
attachments = [
self._construct_mailjet_attachment(part, esp_event)
for part in esp_event.get("Parts", [])
if "Attachment" in part.get("ContentRef", "") # Attachment<N> or InlineAttachment<N>
# if ContentRef is Attachment<N> or InlineAttachment<N>:
if "Attachment" in part.get("ContentRef", "")
]
message = AnymailInboundMessage.construct(
headers=headers,
@@ -139,49 +153,62 @@ class MailjetInboundWebhookView(AnymailBaseWebhookView):
message.envelope_sender = esp_event.get("Sender", None)
message.envelope_recipient = esp_event.get("Recipient", None)
message.spam_detected = None # Mailjet doesn't provide a boolean; you'll have to interpret spam_score
# Mailjet doesn't provide a spam boolean; you'll have to interpret spam_score
message.spam_detected = None
try:
message.spam_score = float(esp_event['SpamAssassinScore'])
message.spam_score = float(esp_event["SpamAssassinScore"])
except (KeyError, TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # Mailjet doesn't provide inbound event timestamp (esp_event['Date'] is time sent)
event_id=None, # Mailjet doesn't provide an idempotent inbound event id
# Mailjet doesn't provide inbound event timestamp
# (esp_event["Date"] is time sent):
timestamp=None,
# Mailjet doesn't provide an idempotent inbound event id:
event_id=None,
esp_event=esp_event,
message=message,
)
@staticmethod
def _flatten_mailjet_headers(headers):
"""Convert Mailjet's dict-of-strings-and/or-lists header format to our list-of-name-value-pairs
"""
Convert Mailjet's dict-of-strings-and/or-lists header format
to our list-of-name-value-pairs
{'name1': 'value', 'name2': ['value1', 'value2']}
--> [('name1', 'value'), ('name2', 'value1'), ('name2', 'value2')]
"""
result = []
for name, values in headers.items():
if isinstance(values, list): # Mailjet groups repeated headers together as a list of values
if isinstance(values, list):
# Mailjet groups repeated headers together as a list of values
for value in values:
result.append((name, value))
else:
result.append((name, values)) # single-valued (non-list) header
# single-valued (non-list) header
result.append((name, values))
return result
def _construct_mailjet_attachment(self, part, esp_event):
# Mailjet includes unparsed attachment headers in each part; it's easiest to temporarily
# attach them to a MIMEPart for parsing. (We could just turn this into the attachment,
# but we want to use the payload handling from AnymailInboundMessage.construct_attachment later.)
part_headers = AnymailInboundMessage() # temporary container for parsed attachment headers
# Mailjet includes unparsed attachment headers in each part; it's easiest to
# temporarily attach them to a MIMEPart for parsing. (We could just turn this
# into the attachment, but we want to use the payload handling from
# AnymailInboundMessage.construct_attachment later.)
# temporary container for parsed attachment headers:
part_headers = AnymailInboundMessage()
for name, value in self._flatten_mailjet_headers(part.get("Headers", {})):
part_headers.add_header(name, value)
content_base64 = esp_event[part["ContentRef"]] # Mailjet *always* base64-encodes attachments
# Mailjet *always* base64-encodes attachments
content_base64 = esp_event[part["ContentRef"]]
return AnymailInboundMessage.construct_attachment(
content_type=part_headers.get_content_type(),
content=content_base64, base64=True,
content=content_base64,
base64=True,
filename=part_headers.get_filename(None),
content_id=part_headers.get("Content-ID", "") or None,
)

152
anymail/webhooks/mandrill.py
View File

@@ -1,16 +1,22 @@
import json
from datetime import datetime, timezone
import hashlib
import hmac
import json
from base64 import b64encode
from datetime import datetime, timezone
from django.utils.crypto import constant_time_compare
from .base import AnymailBaseWebhookView, AnymailCoreWebhookView
from ..exceptions import AnymailWebhookValidationFailure
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType
from ..utils import get_anymail_setting, getfirst, get_request_uri
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
inbound,
tracking,
)
from ..utils import get_anymail_setting, get_request_uri, getfirst
from .base import AnymailBaseWebhookView, AnymailCoreWebhookView
class MandrillSignatureMixin(AnymailCoreWebhookView):
@@ -22,38 +28,60 @@ class MandrillSignatureMixin(AnymailCoreWebhookView):
def __init__(self, **kwargs):
esp_name = self.esp_name
# webhook_key is required for POST, but not for HEAD when Mandrill validates webhook url.
# Defer "missing setting" error until we actually try to use it in the POST...
webhook_key = get_anymail_setting('webhook_key', esp_name=esp_name, default=None,
kwargs=kwargs, allow_bare=True)
# webhook_key is required for POST, but not for HEAD when Mandrill validates
# webhook url. Defer "missing setting" error until we actually try to use it in
# the POST...
webhook_key = get_anymail_setting(
"webhook_key",
esp_name=esp_name,
default=None,
kwargs=kwargs,
allow_bare=True,
)
if webhook_key is not None:
self.webhook_key = webhook_key.encode('ascii') # hmac.new requires bytes key
self.webhook_url = get_anymail_setting('webhook_url', esp_name=esp_name, default=None,
kwargs=kwargs, allow_bare=True)
# hmac.new requires bytes key
self.webhook_key = webhook_key.encode("ascii")
self.webhook_url = get_anymail_setting(
"webhook_url",
esp_name=esp_name,
default=None,
kwargs=kwargs,
allow_bare=True,
)
super().__init__(**kwargs)
def validate_request(self, request):
if self.webhook_key is None:
# issue deferred "missing setting" error (re-call get-setting without a default)
get_anymail_setting('webhook_key', esp_name=self.esp_name, allow_bare=True)
# issue deferred "missing setting" error
# (re-call get-setting without a default)
get_anymail_setting("webhook_key", esp_name=self.esp_name, allow_bare=True)
try:
signature = request.META["HTTP_X_MANDRILL_SIGNATURE"]
except KeyError:
raise AnymailWebhookValidationFailure("X-Mandrill-Signature header missing from webhook POST") from None
raise AnymailWebhookValidationFailure(
"X-Mandrill-Signature header missing from webhook POST"
) from None
# Mandrill signs the exact URL (including basic auth, if used) plus the sorted POST params:
# Mandrill signs the exact URL (including basic auth, if used)
# plus the sorted POST params:
url = self.webhook_url or get_request_uri(request)
params = request.POST.dict()
signed_data = url
for key in sorted(params.keys()):
signed_data += key + params[key]
expected_signature = b64encode(hmac.new(key=self.webhook_key, msg=signed_data.encode('utf-8'),
digestmod=hashlib.sha1).digest())
expected_signature = b64encode(
hmac.new(
key=self.webhook_key,
msg=signed_data.encode("utf-8"),
digestmod=hashlib.sha1,
).digest()
)
if not constant_time_compare(signature, expected_signature):
raise AnymailWebhookValidationFailure(
"Mandrill webhook called with incorrect signature (for url %r)" % url)
"Mandrill webhook called with incorrect signature (for url %r)" % url
)
class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView):
@@ -65,19 +93,19 @@ class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView
signal = None # set in esp_to_anymail_event
def parse_events(self, request):
esp_events = json.loads(request.POST['mandrill_events'])
esp_events = json.loads(request.POST["mandrill_events"])
return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
def esp_to_anymail_event(self, esp_event):
"""Route events to the inbound or tracking handler"""
esp_type = getfirst(esp_event, ['event', 'type'], 'unknown')
esp_type = getfirst(esp_event, ["event", "type"], "unknown")
if esp_type == 'inbound':
assert self.signal is not tracking # Mandrill should never mix event types in the same batch
if esp_type == "inbound":
assert self.signal is not tracking # batch must not mix event types
self.signal = inbound
return self.mandrill_inbound_to_anymail_event(esp_event)
else:
assert self.signal is not inbound # Mandrill should never mix event types in the same batch
assert self.signal is not inbound # batch must not mix event types
self.signal = tracking
return self.mandrill_tracking_to_anymail_event(esp_event)
@@ -87,72 +115,74 @@ class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView
event_types = {
# Message events:
'send': EventType.SENT,
'deferral': EventType.DEFERRED,
'hard_bounce': EventType.BOUNCED,
'soft_bounce': EventType.BOUNCED,
'open': EventType.OPENED,
'click': EventType.CLICKED,
'spam': EventType.COMPLAINED,
'unsub': EventType.UNSUBSCRIBED,
'reject': EventType.REJECTED,
"send": EventType.SENT,
"deferral": EventType.DEFERRED,
"hard_bounce": EventType.BOUNCED,
"soft_bounce": EventType.BOUNCED,
"open": EventType.OPENED,
"click": EventType.CLICKED,
"spam": EventType.COMPLAINED,
"unsub": EventType.UNSUBSCRIBED,
"reject": EventType.REJECTED,
# Sync events (we don't really normalize these well):
'whitelist': EventType.UNKNOWN,
'blacklist': EventType.UNKNOWN,
"whitelist": EventType.UNKNOWN,
"blacklist": EventType.UNKNOWN,
# Inbound events:
'inbound': EventType.INBOUND,
"inbound": EventType.INBOUND,
}
def mandrill_tracking_to_anymail_event(self, esp_event):
esp_type = getfirst(esp_event, ['event', 'type'], None)
esp_type = getfirst(esp_event, ["event", "type"], None)
event_type = self.event_types.get(esp_type, EventType.UNKNOWN)
try:
timestamp = datetime.fromtimestamp(esp_event['ts'], tz=timezone.utc)
timestamp = datetime.fromtimestamp(esp_event["ts"], tz=timezone.utc)
except (KeyError, ValueError):
timestamp = None
try:
recipient = esp_event['msg']['email']
recipient = esp_event["msg"]["email"]
except KeyError:
try:
recipient = esp_event['reject']['email'] # sync events
recipient = esp_event["reject"]["email"] # sync events
except KeyError:
recipient = None
try:
mta_response = esp_event['msg']['diag']
mta_response = esp_event["msg"]["diag"]
except KeyError:
mta_response = None
try:
description = getfirst(esp_event['reject'], ['detail', 'reason'])
description = getfirst(esp_event["reject"], ["detail", "reason"])
except KeyError:
description = None
try:
metadata = esp_event['msg']['metadata']
metadata = esp_event["msg"]["metadata"]
except KeyError:
metadata = {}
try:
tags = esp_event['msg']['tags']
tags = esp_event["msg"]["tags"]
except KeyError:
tags = []
return AnymailTrackingEvent(
click_url=esp_event.get('url', None),
click_url=esp_event.get("url", None),
description=description,
esp_event=esp_event,
event_type=event_type,
message_id=esp_event.get('_id', None),
message_id=esp_event.get("_id", None),
metadata=metadata,
mta_response=mta_response,
recipient=recipient,
reject_reason=None, # probably map esp_event['msg']['bounce_description'], but insufficient docs
# reject_reason should probably map esp_event['msg']['bounce_description'],
# but Mandrill docs are insufficient to determine how
reject_reason=None,
tags=tags,
timestamp=timestamp,
user_agent=esp_event.get('user_agent', None),
user_agent=esp_event.get("user_agent", None),
)
#
@@ -160,27 +190,33 @@ class MandrillCombinedWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView
#
def mandrill_inbound_to_anymail_event(self, esp_event):
# It's easier (and more accurate) to just work from the original raw mime message
message = AnymailInboundMessage.parse_raw_mime(esp_event['msg']['raw_msg'])
message.envelope_sender = None # (Mandrill's 'sender' field only applies to outbound messages)
message.envelope_recipient = esp_event['msg'].get('email', None)
# It's easier (and more accurate) to just work
# from the original raw mime message
message = AnymailInboundMessage.parse_raw_mime(esp_event["msg"]["raw_msg"])
message.spam_detected = None # no simple boolean field; would need to parse the spam_report
message.spam_score = esp_event['msg'].get('spam_report', {}).get('score', None)
# (Mandrill's "sender" field only applies to outbound messages)
message.envelope_sender = None
message.envelope_recipient = esp_event["msg"].get("email", None)
# no simple boolean spam; would need to parse the spam_report
message.spam_detected = None
message.spam_score = esp_event["msg"].get("spam_report", {}).get("score", None)
try:
timestamp = datetime.fromtimestamp(esp_event['ts'], tz=timezone.utc)
timestamp = datetime.fromtimestamp(esp_event["ts"], tz=timezone.utc)
except (KeyError, ValueError):
timestamp = None
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=timestamp,
event_id=None, # Mandrill doesn't provide an idempotent inbound message event id
# Mandrill doesn't provide an idempotent inbound message event id
event_id=None,
esp_event=esp_event,
message=message,
)
# Backwards-compatibility: earlier Anymail versions had only MandrillTrackingWebhookView:
# Backwards-compatibility:
# earlier Anymail versions had only MandrillTrackingWebhookView:
MandrillTrackingWebhookView = MandrillCombinedWebhookView

75
anymail/webhooks/postal.py
View File

@@ -3,35 +3,36 @@ import json
from base64 import b64decode
from datetime import datetime, timezone
from .base import AnymailBaseWebhookView
from ..exceptions import (
AnymailConfigurationError,
AnymailImproperlyInstalled,
AnymailInvalidAddress,
AnymailWebhookValidationFailure,
AnymailImproperlyInstalled,
_LazyError,
AnymailConfigurationError,
)
from ..inbound import AnymailInboundMessage
from ..signals import (
inbound,
tracking,
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from ..utils import parse_single_address, get_anymail_setting
from ..utils import get_anymail_setting, parse_single_address
from .base import AnymailBaseWebhookView
try:
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import padding
except ImportError:
# This module gets imported by anymail.urls, so don't complain about cryptography missing
# unless one of the Postal webhook views is actually used and needs it
error = _LazyError(AnymailImproperlyInstalled(missing_package='cryptography', backend='postal'))
# This module gets imported by anymail.urls, so don't complain about cryptography
# missing unless one of the Postal webhook views is actually used and needs it
error = _LazyError(
AnymailImproperlyInstalled(missing_package="cryptography", backend="postal")
)
serialization = error
hashes = error
default_backend = error
@@ -50,7 +51,9 @@ class PostalBaseWebhookView(AnymailBaseWebhookView):
webhook_key = None
def __init__(self, **kwargs):
self.webhook_key = get_anymail_setting('webhook_key', esp_name=self.esp_name, kwargs=kwargs, allow_bare=True)
self.webhook_key = get_anymail_setting(
"webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True
)
super().__init__(**kwargs)
@@ -58,23 +61,27 @@ class PostalBaseWebhookView(AnymailBaseWebhookView):
try:
signature = request.META["HTTP_X_POSTAL_SIGNATURE"]
except KeyError:
raise AnymailWebhookValidationFailure("X-Postal-Signature header missing from webhook")
raise AnymailWebhookValidationFailure(
"X-Postal-Signature header missing from webhook"
)
public_key = serialization.load_pem_public_key(
('-----BEGIN PUBLIC KEY-----\n' + self.webhook_key + '\n-----END PUBLIC KEY-----').encode(),
backend=default_backend()
(
"-----BEGIN PUBLIC KEY-----\n"
+ self.webhook_key
+ "\n-----END PUBLIC KEY-----"
).encode(),
backend=default_backend(),
)
try:
public_key.verify(
b64decode(signature),
request.body,
padding.PKCS1v15(),
hashes.SHA1()
b64decode(signature), request.body, padding.PKCS1v15(), hashes.SHA1()
)
except (InvalidSignature, binascii.Error):
raise AnymailWebhookValidationFailure(
"Postal webhook called with incorrect signature")
"Postal webhook called with incorrect signature"
)
class PostalTrackingWebhookView(PostalBaseWebhookView):
@@ -85,10 +92,11 @@ class PostalTrackingWebhookView(PostalBaseWebhookView):
def parse_events(self, request):
esp_event = json.loads(request.body.decode("utf-8"))
if 'rcpt_to' in esp_event:
if "rcpt_to" in esp_event:
raise AnymailConfigurationError(
"You seem to have set Postal's *inbound* webhook "
"to Anymail's Postal *tracking* webhook URL.")
"to Anymail's Postal *tracking* webhook URL."
)
raw_timestamp = esp_event.get("timestamp")
timestamp = (
@@ -133,8 +141,8 @@ class PostalTrackingWebhookView(PostalBaseWebhookView):
if message.get("direction") == "incoming":
# Let's ignore tracking events about an inbound emails.
# This happens when an inbound email could not be forwarded.
# The email didn't originate from Anymail, so the user can't do much about it.
# It is part of normal Postal operation, not a configuration error.
# The email didn't originate from Anymail, so the user can't do much about
# it. It is part of normal Postal operation, not a configuration error.
return []
# only for MessageLinkClicked
@@ -144,7 +152,7 @@ class PostalTrackingWebhookView(PostalBaseWebhookView):
event = AnymailTrackingEvent(
event_type=event_type,
timestamp=timestamp,
event_id=esp_event.get('uuid'),
event_id=esp_event.get("uuid"),
esp_event=esp_event,
click_url=click_url,
description=description,
@@ -152,7 +160,9 @@ class PostalTrackingWebhookView(PostalBaseWebhookView):
metadata=None,
mta_response=mta_response,
recipient=recipient,
reject_reason=RejectReason.BOUNCED if event_type == EventType.BOUNCED else None,
reject_reason=(
RejectReason.BOUNCED if event_type == EventType.BOUNCED else None
),
tags=[tag],
user_agent=user_agent,
)
@@ -168,18 +178,19 @@ class PostalInboundWebhookView(PostalBaseWebhookView):
def parse_events(self, request):
esp_event = json.loads(request.body.decode("utf-8"))
if 'status' in esp_event:
if "status" in esp_event:
raise AnymailConfigurationError(
"You seem to have set Postal's *tracking* webhook "
"to Anymail's Postal *inbound* webhook URL.")
"to Anymail's Postal *inbound* webhook URL."
)
raw_mime = esp_event["message"]
if esp_event.get("base64") is True:
raw_mime = b64decode(esp_event["message"]).decode("utf-8")
message = AnymailInboundMessage.parse_raw_mime(raw_mime)
message.envelope_sender = esp_event.get('mail_from', None)
message.envelope_recipient = esp_event.get('rcpt_to', None)
message.envelope_sender = esp_event.get("mail_from", None)
message.envelope_recipient = esp_event.get("rcpt_to", None)
event = AnymailInboundEvent(
event_type=EventType.INBOUND,

177
anymail/webhooks/postmark.py
View File

@@ -2,11 +2,18 @@ import json
from django.utils.dateparse import parse_datetime
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailConfigurationError
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
from ..utils import getfirst, EmailAddress
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from ..utils import EmailAddress, getfirst
from .base import AnymailBaseWebhookView
class PostmarkBaseWebhookView(AnymailBaseWebhookView):
@@ -15,7 +22,7 @@ class PostmarkBaseWebhookView(AnymailBaseWebhookView):
esp_name = "Postmark"
def parse_events(self, request):
esp_event = json.loads(request.body.decode('utf-8'))
esp_event = json.loads(request.body.decode("utf-8"))
return [self.esp_to_anymail_event(esp_event)]
def esp_to_anymail_event(self, esp_event):
@@ -29,40 +36,44 @@ class PostmarkTrackingWebhookView(PostmarkBaseWebhookView):
event_record_types = {
# Map Postmark event RecordType --> Anymail normalized event type
'Bounce': EventType.BOUNCED, # but check Type field for further info (below)
'Click': EventType.CLICKED,
'Delivery': EventType.DELIVERED,
'Open': EventType.OPENED,
'SpamComplaint': EventType.COMPLAINED,
'SubscriptionChange': EventType.UNSUBSCRIBED,
'Inbound': EventType.INBOUND, # future, probably
"Bounce": EventType.BOUNCED, # but check Type field for further info (below)
"Click": EventType.CLICKED,
"Delivery": EventType.DELIVERED,
"Open": EventType.OPENED,
"SpamComplaint": EventType.COMPLAINED,
"SubscriptionChange": EventType.UNSUBSCRIBED,
"Inbound": EventType.INBOUND, # future, probably
}
event_types = {
# Map Postmark bounce/spam event Type --> Anymail normalized (event type, reject reason)
'HardBounce': (EventType.BOUNCED, RejectReason.BOUNCED),
'Transient': (EventType.DEFERRED, None),
'Unsubscribe': (EventType.UNSUBSCRIBED, RejectReason.UNSUBSCRIBED),
'Subscribe': (EventType.SUBSCRIBED, None),
'AutoResponder': (EventType.AUTORESPONDED, None),
'AddressChange': (EventType.AUTORESPONDED, None),
'DnsError': (EventType.DEFERRED, None), # "temporary DNS error"
'SpamNotification': (EventType.COMPLAINED, RejectReason.SPAM),
'OpenRelayTest': (EventType.DEFERRED, None), # Receiving MTA is testing Postmark
'Unknown': (EventType.UNKNOWN, None),
'SoftBounce': (EventType.BOUNCED, RejectReason.BOUNCED), # might also receive HardBounce later
'VirusNotification': (EventType.BOUNCED, RejectReason.OTHER),
'ChallengeVerification': (EventType.AUTORESPONDED, None),
'BadEmailAddress': (EventType.REJECTED, RejectReason.INVALID),
'SpamComplaint': (EventType.COMPLAINED, RejectReason.SPAM),
'ManuallyDeactivated': (EventType.REJECTED, RejectReason.BLOCKED),
'Unconfirmed': (EventType.REJECTED, None),
'Blocked': (EventType.REJECTED, RejectReason.BLOCKED),
'SMTPApiError': (EventType.FAILED, None), # could occur if user also using Postmark SMTP directly
'InboundError': (EventType.INBOUND_FAILED, None),
'DMARCPolicy': (EventType.REJECTED, RejectReason.BLOCKED),
'TemplateRenderingFailed': (EventType.FAILED, None),
'ManualSuppression': (EventType.UNSUBSCRIBED, RejectReason.UNSUBSCRIBED),
# Map Postmark bounce/spam event Type
# --> Anymail normalized (event type, reject reason)
"HardBounce": (EventType.BOUNCED, RejectReason.BOUNCED),
"Transient": (EventType.DEFERRED, None),
"Unsubscribe": (EventType.UNSUBSCRIBED, RejectReason.UNSUBSCRIBED),
"Subscribe": (EventType.SUBSCRIBED, None),
"AutoResponder": (EventType.AUTORESPONDED, None),
"AddressChange": (EventType.AUTORESPONDED, None),
"DnsError": (EventType.DEFERRED, None), # "temporary DNS error"
"SpamNotification": (EventType.COMPLAINED, RejectReason.SPAM),
# Receiving MTA is testing Postmark:
"OpenRelayTest": (EventType.DEFERRED, None),
"Unknown": (EventType.UNKNOWN, None),
# might also receive HardBounce later:
"SoftBounce": (EventType.BOUNCED, RejectReason.BOUNCED),
"VirusNotification": (EventType.BOUNCED, RejectReason.OTHER),
"ChallengeVerification": (EventType.AUTORESPONDED, None),
"BadEmailAddress": (EventType.REJECTED, RejectReason.INVALID),
"SpamComplaint": (EventType.COMPLAINED, RejectReason.SPAM),
"ManuallyDeactivated": (EventType.REJECTED, RejectReason.BLOCKED),
"Unconfirmed": (EventType.REJECTED, None),
"Blocked": (EventType.REJECTED, RejectReason.BLOCKED),
# could occur if user also using Postmark SMTP directly:
"SMTPApiError": (EventType.FAILED, None),
"InboundError": (EventType.INBOUND_FAILED, None),
"DMARCPolicy": (EventType.REJECTED, RejectReason.BLOCKED),
"TemplateRenderingFailed": (EventType.FAILED, None),
"ManualSuppression": (EventType.UNSUBSCRIBED, RejectReason.UNSUBSCRIBED),
}
def esp_to_anymail_event(self, esp_event):
@@ -70,7 +81,7 @@ class PostmarkTrackingWebhookView(PostmarkBaseWebhookView):
try:
esp_record_type = esp_event["RecordType"]
except KeyError:
if 'FromFull' in esp_event:
if "FromFull" in esp_event:
# This is an inbound event
event_type = EventType.INBOUND
else:
@@ -81,59 +92,65 @@ class PostmarkTrackingWebhookView(PostmarkBaseWebhookView):
if event_type == EventType.INBOUND:
raise AnymailConfigurationError(
"You seem to have set Postmark's *inbound* webhook "
"to Anymail's Postmark *tracking* webhook URL.")
"to Anymail's Postmark *tracking* webhook URL."
)
if event_type in (EventType.BOUNCED, EventType.COMPLAINED):
# additional info is in the Type field
try:
event_type, reject_reason = self.event_types[esp_event['Type']]
event_type, reject_reason = self.event_types[esp_event["Type"]]
except KeyError:
pass
if event_type == EventType.UNSUBSCRIBED:
if esp_event['SuppressSending']:
if esp_event["SuppressSending"]:
# Postmark doesn't provide a way to distinguish between
# explicit unsubscribes and bounces
try:
event_type, reject_reason = self.event_types[esp_event['SuppressionReason']]
event_type, reject_reason = self.event_types[
esp_event["SuppressionReason"]
]
except KeyError:
pass
else:
event_type, reject_reason = self.event_types['Subscribe']
event_type, reject_reason = self.event_types["Subscribe"]
recipient = getfirst(esp_event, ['Email', 'Recipient'], None) # Email for bounce; Recipient for open
# Email for bounce; Recipient for open:
recipient = getfirst(esp_event, ["Email", "Recipient"], None)
try:
timestr = getfirst(esp_event, ['DeliveredAt', 'BouncedAt', 'ReceivedAt', 'ChangedAt'])
timestr = getfirst(
esp_event, ["DeliveredAt", "BouncedAt", "ReceivedAt", "ChangedAt"]
)
except KeyError:
timestamp = None
else:
timestamp = parse_datetime(timestr)
try:
event_id = str(esp_event['ID']) # only in bounce events
event_id = str(esp_event["ID"]) # only in bounce events
except KeyError:
event_id = None
metadata = esp_event.get('Metadata', {})
metadata = esp_event.get("Metadata", {})
try:
tags = [esp_event['Tag']]
tags = [esp_event["Tag"]]
except KeyError:
tags = []
return AnymailTrackingEvent(
description=esp_event.get('Description', None),
description=esp_event.get("Description", None),
esp_event=esp_event,
event_id=event_id,
event_type=event_type,
message_id=esp_event.get('MessageID', None),
message_id=esp_event.get("MessageID", None),
metadata=metadata,
mta_response=esp_event.get('Details', None),
mta_response=esp_event.get("Details", None),
recipient=recipient,
reject_reason=reject_reason,
tags=tags,
timestamp=timestamp,
user_agent=esp_event.get('UserAgent', None),
click_url=esp_event.get('OriginalLink', None),
user_agent=esp_event.get("UserAgent", None),
click_url=esp_event.get("OriginalLink", None),
)
@@ -146,12 +163,14 @@ class PostmarkInboundWebhookView(PostmarkBaseWebhookView):
if esp_event.get("RecordType", "Inbound") != "Inbound":
raise AnymailConfigurationError(
"You seem to have set Postmark's *%s* webhook "
"to Anymail's Postmark *inbound* webhook URL." % esp_event["RecordType"])
"to Anymail's Postmark *inbound* webhook URL." % esp_event["RecordType"]
)
attachments = [
AnymailInboundMessage.construct_attachment(
content_type=attachment["ContentType"],
content=attachment["Content"], base64=True,
content=attachment["Content"],
base64=True,
filename=attachment.get("Name", "") or None,
content_id=attachment.get("ContentID", "") or None,
)
@@ -160,11 +179,15 @@ class PostmarkInboundWebhookView(PostmarkBaseWebhookView):
message = AnymailInboundMessage.construct(
from_email=self._address(esp_event.get("FromFull")),
to=', '.join([self._address(to) for to in esp_event.get("ToFull", [])]),
cc=', '.join([self._address(cc) for cc in esp_event.get("CcFull", [])]),
# bcc? Postmark specs this for inbound events, but it's unclear how it could occur
to=", ".join([self._address(to) for to in esp_event.get("ToFull", [])]),
cc=", ".join([self._address(cc) for cc in esp_event.get("CcFull", [])]),
# bcc? Postmark specs this for inbound events,
# but it's unclear how it could occur
subject=esp_event.get("Subject", ""),
headers=[(header["Name"], header["Value"]) for header in esp_event.get("Headers", [])],
headers=[
(header["Name"], header["Value"])
for header in esp_event.get("Headers", [])
],
text=esp_event.get("TextBody", ""),
html=esp_event.get("HtmlBody", ""),
attachments=attachments,
@@ -176,36 +199,48 @@ class PostmarkInboundWebhookView(PostmarkBaseWebhookView):
if "ReplyTo" in esp_event and "Reply-To" not in message:
message["Reply-To"] = esp_event["ReplyTo"]
# Postmark doesn't have a separate envelope-sender field, but it can be extracted
# from the Received-SPF header that Postmark will have added:
if len(message.get_all("Received-SPF", [])) == 1: # (more than one? someone's up to something weird)
# Postmark doesn't have a separate envelope-sender field, but it can
# be extracted from the Received-SPF header that Postmark will have added.
# (More than one Received-SPF? someone's up to something weird?)
if len(message.get_all("Received-SPF", [])) == 1:
received_spf = message["Received-SPF"].lower()
if received_spf.startswith("pass") or received_spf.startswith("neutral"): # not fail/softfail
message.envelope_sender = message.get_param("envelope-from", None, header="Received-SPF")
if received_spf.startswith( # not fail/softfail
"pass"
) or received_spf.startswith("neutral"):
message.envelope_sender = message.get_param(
"envelope-from", None, header="Received-SPF"
)
message.envelope_recipient = esp_event.get("OriginalRecipient", None)
message.stripped_text = esp_event.get("StrippedTextReply", None)
message.spam_detected = message.get('X-Spam-Status', 'No').lower() == 'yes'
message.spam_detected = message.get("X-Spam-Status", "No").lower() == "yes"
try:
message.spam_score = float(message['X-Spam-Score'])
message.spam_score = float(message["X-Spam-Score"])
except (TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # Postmark doesn't provide inbound event timestamp
event_id=esp_event.get("MessageID", None), # Postmark uuid, different from Message-ID mime header
# Postmark doesn't provide inbound event timestamp:
timestamp=None,
# Postmark uuid, different from Message-ID mime header:
event_id=esp_event.get("MessageID", None),
esp_event=esp_event,
message=message,
)
@staticmethod
def _address(full):
"""Return an formatted email address from a Postmark inbound {From,To,Cc}Full dict"""
"""
Return a formatted email address
from a Postmark inbound {From,To,Cc}Full dict
"""
if full is None:
return ""
return str(EmailAddress(
display_name=full.get('Name', ""),
addr_spec=full.get("Email", ""),
))
return str(
EmailAddress(
display_name=full.get("Name", ""),
addr_spec=full.get("Email", ""),
)
)

231
anymail/webhooks/sendgrid.py
View File

@@ -3,10 +3,16 @@ from datetime import datetime, timezone
from email.parser import BytesParser
from email.policy import default as default_policy
from .base import AnymailBaseWebhookView
from ..inbound import AnymailInboundMessage
from ..signals import AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason, inbound, tracking
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from .base import AnymailBaseWebhookView
class SendGridTrackingWebhookView(AnymailBaseWebhookView):
@@ -16,47 +22,50 @@ class SendGridTrackingWebhookView(AnymailBaseWebhookView):
signal = tracking
def parse_events(self, request):
esp_events = json.loads(request.body.decode('utf-8'))
esp_events = json.loads(request.body.decode("utf-8"))
return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
event_types = {
# Map SendGrid event: Anymail normalized type
'bounce': EventType.BOUNCED,
'deferred': EventType.DEFERRED,
'delivered': EventType.DELIVERED,
'dropped': EventType.REJECTED,
'processed': EventType.QUEUED,
'click': EventType.CLICKED,
'open': EventType.OPENED,
'spamreport': EventType.COMPLAINED,
'unsubscribe': EventType.UNSUBSCRIBED,
'group_unsubscribe': EventType.UNSUBSCRIBED,
'group_resubscribe': EventType.SUBSCRIBED,
"bounce": EventType.BOUNCED,
"deferred": EventType.DEFERRED,
"delivered": EventType.DELIVERED,
"dropped": EventType.REJECTED,
"processed": EventType.QUEUED,
"click": EventType.CLICKED,
"open": EventType.OPENED,
"spamreport": EventType.COMPLAINED,
"unsubscribe": EventType.UNSUBSCRIBED,
"group_unsubscribe": EventType.UNSUBSCRIBED,
"group_resubscribe": EventType.SUBSCRIBED,
}
reject_reasons = {
# Map SendGrid reason/type strings (lowercased) to Anymail normalized reject_reason
'invalid': RejectReason.INVALID,
'unsubscribed address': RejectReason.UNSUBSCRIBED,
'bounce': RejectReason.BOUNCED,
'blocked': RejectReason.BLOCKED,
'expired': RejectReason.TIMED_OUT,
# Map SendGrid reason/type strings (lowercased)
# to Anymail normalized reject_reason
"invalid": RejectReason.INVALID,
"unsubscribed address": RejectReason.UNSUBSCRIBED,
"bounce": RejectReason.BOUNCED,
"blocked": RejectReason.BLOCKED,
"expired": RejectReason.TIMED_OUT,
}
def esp_to_anymail_event(self, esp_event):
event_type = self.event_types.get(esp_event['event'], EventType.UNKNOWN)
event_type = self.event_types.get(esp_event["event"], EventType.UNKNOWN)
try:
timestamp = datetime.fromtimestamp(esp_event['timestamp'], tz=timezone.utc)
timestamp = datetime.fromtimestamp(esp_event["timestamp"], tz=timezone.utc)
except (KeyError, ValueError):
timestamp = None
if esp_event['event'] == 'dropped':
mta_response = None # dropped at ESP before even getting to MTA
reason = esp_event.get('type', esp_event.get('reason', '')) # cause could be in 'type' or 'reason'
if esp_event["event"] == "dropped":
# message dropped at ESP before even getting to MTA:
mta_response = None
# cause could be in "type" or "reason":
reason = esp_event.get("type", esp_event.get("reason", ""))
reject_reason = self.reject_reasons.get(reason.lower(), RejectReason.OTHER)
else:
# MTA response is in 'response' for delivered; 'reason' for bounce
mta_response = esp_event.get('response', esp_event.get('reason', None))
# MTA response is in "response" for delivered; "reason" for bounce
mta_response = esp_event.get("response", esp_event.get("reason", None))
reject_reason = None
# SendGrid merges metadata ('unique_args') with the event.
@@ -73,49 +82,50 @@ class SendGridTrackingWebhookView(AnymailBaseWebhookView):
return AnymailTrackingEvent(
event_type=event_type,
timestamp=timestamp,
message_id=esp_event.get('anymail_id', esp_event.get('smtp-id')), # backwards compatibility
event_id=esp_event.get('sg_event_id', None),
recipient=esp_event.get('email', None),
# (smtp-id for backwards compatibility)
message_id=esp_event.get("anymail_id", esp_event.get("smtp-id")),
event_id=esp_event.get("sg_event_id", None),
recipient=esp_event.get("email", None),
reject_reason=reject_reason,
mta_response=mta_response,
tags=esp_event.get('category', []),
tags=esp_event.get("category", []),
metadata=metadata,
click_url=esp_event.get('url', None),
user_agent=esp_event.get('useragent', None),
click_url=esp_event.get("url", None),
user_agent=esp_event.get("useragent", None),
esp_event=esp_event,
)
# Known keys in SendGrid events (used to recover metadata above)
sendgrid_event_keys = {
'anymail_id',
'asm_group_id',
'attempt', # MTA deferred count
'category',
'cert_err',
'email',
'event',
'ip',
'marketing_campaign_id',
'marketing_campaign_name',
'newsletter', # ???
'nlvx_campaign_id',
'nlvx_campaign_split_id',
'nlvx_user_id',
'pool',
'post_type',
'reason', # MTA bounce/drop reason; SendGrid suppression reason
'response', # MTA deferred/delivered message
'send_at',
'sg_event_id',
'sg_message_id',
'smtp-id',
'status', # SMTP status code
'timestamp',
'tls',
'type', # suppression reject reason ("bounce", "blocked", "expired")
'url', # click tracking
'url_offset', # click tracking
'useragent', # click/open tracking
"anymail_id",
"asm_group_id",
"attempt", # MTA deferred count
"category",
"cert_err",
"email",
"event",
"ip",
"marketing_campaign_id",
"marketing_campaign_name",
"newsletter", # ???
"nlvx_campaign_id",
"nlvx_campaign_split_id",
"nlvx_user_id",
"pool",
"post_type",
"reason", # MTA bounce/drop reason; SendGrid suppression reason
"response", # MTA deferred/delivered message
"send_at",
"sg_event_id",
"sg_message_id",
"smtp-id",
"status", # SMTP status code
"timestamp",
"tls",
"type", # suppression reject reason ("bounce", "blocked", "expired")
"url", # click tracking
"url_offset", # click tracking
"useragent", # click/open tracking
}
@@ -129,39 +139,46 @@ class SendGridInboundWebhookView(AnymailBaseWebhookView):
return [self.esp_to_anymail_event(request)]
def esp_to_anymail_event(self, request):
# Inbound uses the entire Django request as esp_event, because we need POST and FILES.
# Note that request.POST is case-sensitive (unlike email.message.Message headers).
# Inbound uses the entire Django request as esp_event, because we need
# POST and FILES. Note that request.POST is case-sensitive (unlike
# email.message.Message headers).
esp_event = request
# Must access body before any POST fields, or it won't be available if we need
# it later (see text_charset and html_charset handling below).
_ensure_body_is_available_later = request.body # noqa: F841
if 'headers' in request.POST:
if "headers" in request.POST:
# Default (not "Send Raw") inbound fields
message = self.message_from_sendgrid_parsed(esp_event)
elif 'email' in request.POST:
elif "email" in request.POST:
# "Send Raw" full MIME
message = AnymailInboundMessage.parse_raw_mime(request.POST['email'])
message = AnymailInboundMessage.parse_raw_mime(request.POST["email"])
else:
raise KeyError("Invalid SendGrid inbound event data (missing both 'headers' and 'email' fields)")
raise KeyError(
"Invalid SendGrid inbound event data"
" (missing both 'headers' and 'email' fields)"
)
try:
envelope = json.loads(request.POST['envelope'])
envelope = json.loads(request.POST["envelope"])
except (KeyError, TypeError, ValueError):
pass
else:
message.envelope_sender = envelope['from']
message.envelope_recipient = envelope['to'][0]
message.envelope_sender = envelope["from"]
message.envelope_recipient = envelope["to"][0]
message.spam_detected = None # no simple boolean field; would need to parse the spam_report
# no simple boolean spam; would need to parse the spam_report
message.spam_detected = None
try:
message.spam_score = float(request.POST['spam_score'])
message.spam_score = float(request.POST["spam_score"])
except (KeyError, TypeError, ValueError):
pass
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # SendGrid doesn't provide an inbound event timestamp
event_id=None, # SendGrid doesn't provide an idempotent inbound message event id
# SendGrid doesn't provide an inbound event timestamp:
timestamp=None,
# SendGrid doesn't provide an idempotent inbound message event id:
event_id=None,
esp_event=esp_event,
message=message,
)
@@ -170,12 +187,12 @@ class SendGridInboundWebhookView(AnymailBaseWebhookView):
"""Construct a Message from SendGrid's "default" (non-raw) fields"""
try:
charsets = json.loads(request.POST['charsets'])
charsets = json.loads(request.POST["charsets"])
except (KeyError, ValueError):
charsets = {}
try:
attachment_info = json.loads(request.POST['attachment-info'])
attachment_info = json.loads(request.POST["attachment-info"])
except (KeyError, ValueError):
attachments = None
else:
@@ -186,44 +203,60 @@ class SendGridInboundWebhookView(AnymailBaseWebhookView):
file = request.FILES[attachment_id]
except KeyError:
# Django's multipart/form-data handling drops FILES with certain
# filenames (for security) or with empty filenames (Django ticket 15879).
# (To avoid this problem, enable SendGrid's "raw, full MIME" inbound option.)
# filenames (for security) or with empty filenames (Django ticket
# 15879). (To avoid this problem, enable SendGrid's "raw, full MIME"
# inbound option.)
pass
else:
# (This deliberately ignores attachment_info[attachment_id]["filename"],
# (This deliberately ignores
# attachment_info[attachment_id]["filename"],
# which has not passed through Django's filename sanitization.)
content_id = attachment_info[attachment_id].get("content-id")
attachment = AnymailInboundMessage.construct_attachment_from_uploaded_file(
file, content_id=content_id)
attachment = (
AnymailInboundMessage.construct_attachment_from_uploaded_file(
file, content_id=content_id
)
)
attachments.append(attachment)
default_charset = request.POST.encoding.lower() # (probably utf-8)
text = request.POST.get('text')
text_charset = charsets.get('text', default_charset).lower()
html = request.POST.get('html')
html_charset = charsets.get('html', default_charset).lower()
if (text and text_charset != default_charset) or (html and html_charset != default_charset):
text = request.POST.get("text")
text_charset = charsets.get("text", default_charset).lower()
html = request.POST.get("html")
html_charset = charsets.get("html", default_charset).lower()
if (text and text_charset != default_charset) or (
html and html_charset != default_charset
):
# Django has parsed text and/or html fields using the wrong charset.
# We need to re-parse the raw form data and decode each field separately,
# using the indicated charsets. The email package parses multipart/form-data
# retaining bytes content. (In theory, we could instead just change
# request.encoding and access the POST fields again, per Django docs,
# but that seems to be have bugs around the cached request._files.)
raw_data = b"".join([
b"Content-Type: ", request.META['CONTENT_TYPE'].encode('ascii'),
b"\r\n\r\n",
request.body
])
parsed_parts = BytesParser(policy=default_policy).parsebytes(raw_data).get_payload()
raw_data = b"".join(
[
b"Content-Type: ",
request.META["CONTENT_TYPE"].encode("ascii"),
b"\r\n\r\n",
request.body,
]
)
parsed_parts = (
BytesParser(policy=default_policy).parsebytes(raw_data).get_payload()
)
for part in parsed_parts:
name = part.get_param('name', header='content-disposition')
if name == 'text':
name = part.get_param("name", header="content-disposition")
if name == "text":
text = part.get_payload(decode=True).decode(text_charset)
elif name == 'html':
elif name == "html":
html = part.get_payload(decode=True).decode(html_charset)
# (subject, from, to, etc. are parsed from raw headers field,
# so no need to worry about their separate POST field charsets)
return AnymailInboundMessage.construct(
raw_headers=request.POST.get('headers', ""), # includes From, To, Cc, Subject, etc.
text=text, html=html, attachments=attachments)
# POST["headers"] includes From, To, Cc, Subject, etc.
raw_headers=request.POST.get("headers", ""),
text=text,
html=html,
attachments=attachments,
)

37
anymail/webhooks/sendinblue.py
View File

@@ -1,8 +1,8 @@
import json
from datetime import datetime, timezone
from .base import AnymailBaseWebhookView
from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking
from .base import AnymailBaseWebhookView
class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
@@ -12,14 +12,15 @@ class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
signal = tracking
def parse_events(self, request):
esp_event = json.loads(request.body.decode('utf-8'))
esp_event = json.loads(request.body.decode("utf-8"))
return [self.esp_to_anymail_event(esp_event)]
# SendinBlue's webhook payload data doesn't seem to be documented anywhere.
# There's a list of webhook events at https://apidocs.sendinblue.com/webhooks/#3.
event_types = {
# Map SendinBlue event type: Anymail normalized (event type, reject reason)
"request": (EventType.QUEUED, None), # received even if message won't be sent (e.g., before "blocked")
# received even if message won't be sent (e.g., before "blocked"):
"request": (EventType.QUEUED, None),
"delivered": (EventType.DELIVERED, None),
"hard_bounce": (EventType.BOUNCED, RejectReason.BOUNCED),
"soft_bounce": (EventType.BOUNCED, RejectReason.BOUNCED),
@@ -30,32 +31,39 @@ class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
"opened": (EventType.OPENED, None), # see also unique_opened below
"click": (EventType.CLICKED, None),
"unsubscribe": (EventType.UNSUBSCRIBED, None),
"list_addition": (EventType.SUBSCRIBED, None), # shouldn't occur for transactional messages
# shouldn't occur for transactional messages:
"list_addition": (EventType.SUBSCRIBED, None),
"unique_opened": (EventType.OPENED, None), # you'll *also* receive an "opened"
}
def esp_to_anymail_event(self, esp_event):
esp_type = esp_event.get("event")
event_type, reject_reason = self.event_types.get(esp_type, (EventType.UNKNOWN, None))
event_type, reject_reason = self.event_types.get(
esp_type, (EventType.UNKNOWN, None)
)
recipient = esp_event.get("email")
try:
# SendinBlue supplies "ts", "ts_event" and "date" fields, which seem to be based on the
# timezone set in the account preferences (and possibly with inconsistent DST adjustment).
# "ts_epoch" is the only field that seems to be consistently UTC; it's in milliseconds
timestamp = datetime.fromtimestamp(esp_event["ts_epoch"] / 1000.0, tz=timezone.utc)
# SendinBlue supplies "ts", "ts_event" and "date" fields, which seem to be
# based on the timezone set in the account preferences (and possibly with
# inconsistent DST adjustment). "ts_epoch" is the only field that seems to
# be consistently UTC; it's in milliseconds
timestamp = datetime.fromtimestamp(
esp_event["ts_epoch"] / 1000.0, tz=timezone.utc
)
except (KeyError, ValueError):
timestamp = None
tags = []
try:
# If `tags` param set on send, webhook payload includes 'tags' array field.
tags = esp_event['tags']
tags = esp_event["tags"]
except KeyError:
try:
# If `X-Mailin-Tag` header set on send, webhook payload includes single 'tag' string.
# (If header not set, webhook 'tag' will be the template name for template sends.)
tags = [esp_event['tag']]
# If `X-Mailin-Tag` header set on send, webhook payload includes single
# 'tag' string. (If header not set, webhook 'tag' will be the template
# name for template sends.)
tags = [esp_event["tag"]]
except KeyError:
pass
@@ -67,7 +75,8 @@ class SendinBlueTrackingWebhookView(AnymailBaseWebhookView):
return AnymailTrackingEvent(
description=None,
esp_event=esp_event,
event_id=None, # SendinBlue doesn't provide a unique event id
# SendinBlue doesn't provide a unique event id:
event_id=None,
event_type=event_type,
message_id=esp_event.get("message-id"),
metadata=metadata,

173
anymail/webhooks/sparkpost.py
View File

@@ -2,11 +2,18 @@ import json
from base64 import b64decode
from datetime import datetime, timezone
from .base import AnymailBaseWebhookView
from ..exceptions import AnymailConfigurationError
from ..inbound import AnymailInboundMessage
from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
from ..signals import (
AnymailInboundEvent,
AnymailTrackingEvent,
EventType,
RejectReason,
inbound,
tracking,
)
from ..utils import get_anymail_setting
from .base import AnymailBaseWebhookView
class SparkPostBaseWebhookView(AnymailBaseWebhookView):
@@ -15,7 +22,7 @@ class SparkPostBaseWebhookView(AnymailBaseWebhookView):
esp_name = "SparkPost"
def parse_events(self, request):
raw_events = json.loads(request.body.decode('utf-8'))
raw_events = json.loads(request.body.decode("utf-8"))
unwrapped_events = [self.unwrap_event(raw_event) for raw_event in raw_events]
return [
self.esp_to_anymail_event(event_class, event, raw_event)
@@ -30,17 +37,19 @@ class SparkPostBaseWebhookView(AnymailBaseWebhookView):
Can return None, None, raw_event for SparkPost "ping" raw_event={'msys': {}}
"""
event_classes = raw_event['msys'].keys()
event_classes = raw_event["msys"].keys()
try:
(event_class,) = event_classes
event = raw_event['msys'][event_class]
event = raw_event["msys"][event_class]
except ValueError: # too many/not enough event_classes to unpack
if len(event_classes) == 0:
# Empty event (SparkPost sometimes sends as a "ping")
event_class = event = None
else:
raise TypeError(
"Invalid SparkPost webhook event has multiple event classes: %r" % raw_event) from None
"Invalid SparkPost webhook event has multiple event classes: %r"
% raw_event
) from None
return event_class, event, raw_event
def esp_to_anymail_event(self, event_class, event, raw_event):
@@ -54,54 +63,54 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
event_types = {
# Map SparkPost event.type: Anymail normalized type
'bounce': EventType.BOUNCED,
'delivery': EventType.DELIVERED,
'injection': EventType.QUEUED,
'spam_complaint': EventType.COMPLAINED,
'out_of_band': EventType.BOUNCED,
'policy_rejection': EventType.REJECTED,
'delay': EventType.DEFERRED,
'click': EventType.CLICKED,
'open': EventType.OPENED,
'amp_click': EventType.CLICKED,
'amp_open': EventType.OPENED,
'generation_failure': EventType.FAILED,
'generation_rejection': EventType.REJECTED,
'list_unsubscribe': EventType.UNSUBSCRIBED,
'link_unsubscribe': EventType.UNSUBSCRIBED,
"bounce": EventType.BOUNCED,
"delivery": EventType.DELIVERED,
"injection": EventType.QUEUED,
"spam_complaint": EventType.COMPLAINED,
"out_of_band": EventType.BOUNCED,
"policy_rejection": EventType.REJECTED,
"delay": EventType.DEFERRED,
"click": EventType.CLICKED,
"open": EventType.OPENED,
"amp_click": EventType.CLICKED,
"amp_open": EventType.OPENED,
"generation_failure": EventType.FAILED,
"generation_rejection": EventType.REJECTED,
"list_unsubscribe": EventType.UNSUBSCRIBED,
"link_unsubscribe": EventType.UNSUBSCRIBED,
}
# Additional event_types mapping when Anymail setting
# SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED is enabled.
initial_open_event_types = {
'initial_open': EventType.OPENED,
'amp_initial_open': EventType.OPENED,
"initial_open": EventType.OPENED,
"amp_initial_open": EventType.OPENED,
}
reject_reasons = {
# Map SparkPost event.bounce_class: Anymail normalized reject reason.
# Can also supply (RejectReason, EventType) for bounce_class that affects our event_type.
# https://support.sparkpost.com/customer/portal/articles/1929896
'1': RejectReason.OTHER, # Undetermined (response text could not be identified)
'10': RejectReason.INVALID, # Invalid Recipient
'20': RejectReason.BOUNCED, # Soft Bounce
'21': RejectReason.BOUNCED, # DNS Failure
'22': RejectReason.BOUNCED, # Mailbox Full
'23': RejectReason.BOUNCED, # Too Large
'24': RejectReason.TIMED_OUT, # Timeout
'25': RejectReason.BLOCKED, # Admin Failure (configured policies)
'30': RejectReason.BOUNCED, # Generic Bounce: No RCPT
'40': RejectReason.BOUNCED, # Generic Bounce: unspecified reasons
'50': RejectReason.BLOCKED, # Mail Block (by the receiver)
'51': RejectReason.SPAM, # Spam Block (by the receiver)
'52': RejectReason.SPAM, # Spam Content (by the receiver)
'53': RejectReason.OTHER, # Prohibited Attachment (by the receiver)
'54': RejectReason.BLOCKED, # Relaying Denied (by the receiver)
'60': (RejectReason.OTHER, EventType.AUTORESPONDED), # Auto-Reply/vacation
'70': RejectReason.BOUNCED, # Transient Failure
'80': (RejectReason.OTHER, EventType.SUBSCRIBED), # Subscribe
'90': (RejectReason.UNSUBSCRIBED, EventType.UNSUBSCRIBED), # Unsubscribe
'100': (RejectReason.OTHER, EventType.AUTORESPONDED), # Challenge-Response
# Can also supply (RejectReason, EventType) for bounce_class that affects our
# event_type. https://support.sparkpost.com/customer/portal/articles/1929896
"1": RejectReason.OTHER, # Undetermined (response text could not be identified)
"10": RejectReason.INVALID, # Invalid Recipient
"20": RejectReason.BOUNCED, # Soft Bounce
"21": RejectReason.BOUNCED, # DNS Failure
"22": RejectReason.BOUNCED, # Mailbox Full
"23": RejectReason.BOUNCED, # Too Large
"24": RejectReason.TIMED_OUT, # Timeout
"25": RejectReason.BLOCKED, # Admin Failure (configured policies)
"30": RejectReason.BOUNCED, # Generic Bounce: No RCPT
"40": RejectReason.BOUNCED, # Generic Bounce: unspecified reasons
"50": RejectReason.BLOCKED, # Mail Block (by the receiver)
"51": RejectReason.SPAM, # Spam Block (by the receiver)
"52": RejectReason.SPAM, # Spam Content (by the receiver)
"53": RejectReason.OTHER, # Prohibited Attachment (by the receiver)
"54": RejectReason.BLOCKED, # Relaying Denied (by the receiver)
"60": (RejectReason.OTHER, EventType.AUTORESPONDED), # Auto-Reply/vacation
"70": RejectReason.BOUNCED, # Transient Failure
"80": (RejectReason.OTHER, EventType.SUBSCRIBED), # Subscribe
"90": (RejectReason.UNSUBSCRIBED, EventType.UNSUBSCRIBED), # Unsubscribe
"100": (RejectReason.OTHER, EventType.AUTORESPONDED), # Challenge-Response
}
def __init__(self, **kwargs):
@@ -111,34 +120,43 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
# other ESPs.) Handling "initial_open" is opt-in, to help avoid duplicate
# "opened" events on the same first open.
track_initial_open_as_opened = get_anymail_setting(
'track_initial_open_as_opened', default=False,
esp_name=self.esp_name, kwargs=kwargs)
"track_initial_open_as_opened",
default=False,
esp_name=self.esp_name,
kwargs=kwargs,
)
if track_initial_open_as_opened:
self.event_types = {**self.event_types, **self.initial_open_event_types}
super().__init__(**kwargs)
def esp_to_anymail_event(self, event_class, event, raw_event):
if event_class == 'relay_message':
if event_class == "relay_message":
# This is an inbound event
raise AnymailConfigurationError(
"You seem to have set SparkPost's *inbound* relay webhook URL "
"to Anymail's SparkPost *tracking* webhook URL.")
"to Anymail's SparkPost *tracking* webhook URL."
)
event_type = self.event_types.get(event['type'], EventType.UNKNOWN)
event_type = self.event_types.get(event["type"], EventType.UNKNOWN)
try:
timestamp = datetime.fromtimestamp(int(event['timestamp']), tz=timezone.utc)
timestamp = datetime.fromtimestamp(int(event["timestamp"]), tz=timezone.utc)
except (KeyError, TypeError, ValueError):
timestamp = None
try:
tag = event['campaign_id'] # not 'rcpt_tags' -- those don't come from sending a message
tag = event["campaign_id"]
# not "rcpt_tags" -- those don't come from sending a message
tags = [tag] if tag else None
except KeyError:
tags = []
try:
reject_reason = self.reject_reasons.get(event['bounce_class'], RejectReason.OTHER)
try: # unpack (RejectReason, EventType) for reasons that change our event type
reject_reason = self.reject_reasons.get(
event["bounce_class"], RejectReason.OTHER
)
try:
# unpack (RejectReason, EventType)
# for reasons that change our event type
reject_reason, event_type = reject_reason
except ValueError:
pass
@@ -148,16 +166,19 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
return AnymailTrackingEvent(
event_type=event_type,
timestamp=timestamp,
message_id=event.get('transmission_id', None), # not 'message_id' -- see SparkPost backend
event_id=event.get('event_id', None),
recipient=event.get('raw_rcpt_to', None), # preserves email case (vs. 'rcpt_to')
# use transmission_id, not message_id -- see SparkPost backend
message_id=event.get("transmission_id", None),
event_id=event.get("event_id", None),
# raw_rcpt_to preserves email case (vs. rcpt_to)
recipient=event.get("raw_rcpt_to", None),
reject_reason=reject_reason,
mta_response=event.get('raw_reason', None),
mta_response=event.get("raw_reason", None),
# description=???,
tags=tags,
metadata=event.get('rcpt_meta', None) or {}, # message + recipient metadata
click_url=event.get('target_link_url', None),
user_agent=event.get('user_agent', None),
# metadata includes message + recipient metadata
metadata=event.get("rcpt_meta", None) or {},
click_url=event.get("target_link_url", None),
user_agent=event.get("user_agent", None),
esp_event=raw_event,
)
@@ -168,29 +189,35 @@ class SparkPostInboundWebhookView(SparkPostBaseWebhookView):
signal = inbound
def esp_to_anymail_event(self, event_class, event, raw_event):
if event_class != 'relay_message':
if event_class != "relay_message":
# This is not an inbound event
raise AnymailConfigurationError(
"You seem to have set SparkPost's *tracking* webhook URL "
"to Anymail's SparkPost *inbound* relay webhook URL.")
"to Anymail's SparkPost *inbound* relay webhook URL."
)
if event['protocol'] != 'smtp':
if event["protocol"] != "smtp":
raise AnymailConfigurationError(
"You cannot use Anymail's webhooks for SparkPost '{protocol}' relay events. "
"Anymail only handles the 'smtp' protocol".format(protocol=event['protocol']))
"You cannot use Anymail's webhooks for SparkPost '{protocol}' relay"
" events. Anymail only handles the 'smtp' protocol".format(
protocol=event["protocol"]
)
)
raw_mime = event['content']['email_rfc822']
if event['content']['email_rfc822_is_base64']:
raw_mime = b64decode(raw_mime).decode('utf-8')
raw_mime = event["content"]["email_rfc822"]
if event["content"]["email_rfc822_is_base64"]:
raw_mime = b64decode(raw_mime).decode("utf-8")
message = AnymailInboundMessage.parse_raw_mime(raw_mime)
message.envelope_sender = event.get('msg_from', None)
message.envelope_recipient = event.get('rcpt_to', None)
message.envelope_sender = event.get("msg_from", None)
message.envelope_recipient = event.get("rcpt_to", None)
return AnymailInboundEvent(
event_type=EventType.INBOUND,
timestamp=None, # SparkPost does not provide a relay event timestamp
event_id=None, # SparkPost does not provide an idempotent id for relay events
# SparkPost does not provide a relay event timestamp
timestamp=None,
# SparkPost does not provide an idempotent id for relay events
event_id=None,
esp_event=raw_event,
message=message,
)