From bd9d92f5a0f457102b95a76403658d98c816aa4b Mon Sep 17 00:00:00 2001 From: medmunds Date: Mon, 26 Feb 2018 12:02:48 -0800 Subject: [PATCH] Cleanup: centralize Reply-To header handling; case-insensitive headers Django allows setting the reply address with either message.reply_to or message.extra_headers["Reply-To"]. If both are supplied, the extra headers version takes precedence. (See EmailMessage.message().) Several Anymail backends had duplicate logic to handle conflicting properties. Move that logic into the base Payload. (Also prepares for common handling of extra_headers['From'], later.) Related changes: * Use CaseInsensitiveDict for processing extra_headers. This is potentially a breaking change, but any code that was trying to send multiple headers differing only in case was likely already broken. (Email header field names are case-insensitive, per RFC-822.) * Handle CaseInsensitiveDict in RequestsPayload.serialize_json(). (Several backends had duplicate code for handling this, too.) * Fixes SparkPost backend, which had been incorrectly treating message.reply_to and message.extra_headers['Reply-To'] differently. --- anymail/backends/base.py | 19 ++++++++++++++++++ anymail/backends/base_requests.py | 12 +++++++++-- anymail/backends/postmark.py | 7 +------ anymail/backends/sendgrid.py | 11 ++--------- anymail/backends/sendgrid_v2.py | 6 ++++-- anymail/backends/sendinblue.py | 16 ++++----------- anymail/backends/sparkpost.py | 2 +- tests/test_general_backend.py | 33 +++++++++++++++++++++++++++++++ tests/test_sparkpost_backend.py | 2 +- 9 files changed, 75 insertions(+), 33 deletions(-) diff --git a/anymail/backends/base.py b/anymail/backends/base.py index c5f658a..fe52bea 100644 --- a/anymail/backends/base.py +++ b/anymail/backends/base.py @@ -4,6 +4,7 @@ import six from django.conf import settings from django.core.mail.backends.base import BaseEmailBackend from django.utils.timezone import is_naive, get_current_timezone, make_aware, utc +from requests.structures import CaseInsensitiveDict from ..exceptions import AnymailCancelSend, AnymailError, AnymailUnsupportedFeature, AnymailRecipientsRefused from ..message import AnymailStatus @@ -268,6 +269,8 @@ class BasePayload(object): setter = self.set_html_body if message.content_subtype == 'html' else self.set_text_body elif attr == 'from_email': setter = self.set_from_email_list + elif attr == 'extra_headers': + setter = self.process_extra_headers else: # AttributeError here? Your Payload subclass is missing a set_ implementation setter = getattr(self, 'set_%s' % attr) @@ -278,6 +281,21 @@ class BasePayload(object): raise AnymailUnsupportedFeature("%s does not support %s" % (self.esp_name, feature), email_message=self.message, payload=self, backend=self.backend) + def process_extra_headers(self, headers): + # Handle some special-case headers, and pass the remainder to set_extra_headers. + # (Subclasses shouldn't need to override this.) + headers = CaseInsensitiveDict(headers) # email headers are case-insensitive per RFC-822 et seq + + reply_to = headers.pop('Reply-To', None) + if reply_to: + # message.extra_headers['Reply-To'] will override message.reply_to + # (because the extra_headers attr is processed after reply_to). + # This matches the behavior of Django's EmailMessage.message(). + self.set_reply_to(parse_address_list([reply_to])) + + if headers: + self.set_extra_headers(headers) + # # Attribute validators # @@ -380,6 +398,7 @@ class BasePayload(object): self.unsupported_feature('reply_to') def set_extra_headers(self, headers): + # headers is a CaseInsensitiveDict, and is a copy (so is safe to modify) self.unsupported_feature('extra_headers') def set_text_body(self, body): diff --git a/anymail/backends/base_requests.py b/anymail/backends/base_requests.py index 7b4589f..1e4edd1 100644 --- a/anymail/backends/base_requests.py +++ b/anymail/backends/base_requests.py @@ -1,7 +1,7 @@ import json import requests -# noinspection PyUnresolvedReferences +from requests.structures import CaseInsensitiveDict from six.moves.urllib.parse import urljoin from anymail.utils import get_anymail_setting @@ -156,8 +156,16 @@ class RequestsPayload(BasePayload): Useful for implementing serialize_data in a subclass, """ try: - return json.dumps(data) + return json.dumps(data, default=self._json_default) except TypeError as err: # Add some context to the "not JSON serializable" message raise AnymailSerializationError(orig_err=err, email_message=self.message, backend=self.backend, payload=self) + + @staticmethod + def _json_default(o): + """json.dump default function that handles some common Payload data types""" + if isinstance(o, CaseInsensitiveDict): # used for headers + return dict(o) + raise TypeError("Object of type '%s' is not JSON serializable" % + o.__class__.__name__) diff --git a/anymail/backends/postmark.py b/anymail/backends/postmark.py index 3a2e840..b12df82 100644 --- a/anymail/backends/postmark.py +++ b/anymail/backends/postmark.py @@ -1,7 +1,5 @@ import re -from requests.structures import CaseInsensitiveDict - from ..exceptions import AnymailRequestsAPIError from ..message import AnymailRecipientStatus from ..utils import get_anymail_setting @@ -149,12 +147,9 @@ class PostmarkPayload(RequestsPayload): self.data["ReplyTo"] = reply_to def set_extra_headers(self, headers): - header_dict = CaseInsensitiveDict(headers) - if 'Reply-To' in header_dict: - self.data["ReplyTo"] = header_dict.pop('Reply-To') self.data["Headers"] = [ {"Name": key, "Value": value} - for key, value in header_dict.items() + for key, value in headers.items() ] def set_text_body(self, body): diff --git a/anymail/backends/sendgrid.py b/anymail/backends/sendgrid.py index 89ddff2..fe4c3c2 100644 --- a/anymail/backends/sendgrid.py +++ b/anymail/backends/sendgrid.py @@ -7,7 +7,7 @@ from requests.structures import CaseInsensitiveDict from .base_requests import AnymailRequestsBackend, RequestsPayload from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError, AnymailWarning from ..message import AnymailRecipientStatus -from ..utils import BASIC_NUMERIC_TYPES, get_anymail_setting, timestamp, update_deep, parse_address_list +from ..utils import BASIC_NUMERIC_TYPES, get_anymail_setting, timestamp, update_deep class EmailBackend(AnymailRequestsBackend): @@ -102,14 +102,7 @@ class SendGridPayload(RequestsPayload): self.ensure_message_id() self.build_merge_data() - headers = self.data["headers"] - if "Reply-To" in headers: - # Reply-To must be in its own param - reply_to = headers.pop('Reply-To') - self.set_reply_to(parse_address_list([reply_to])) - if len(headers) > 0: - self.data["headers"] = dict(headers) # flatten to normal dict for json serialization - else: + if not self.data["headers"]: del self.data["headers"] # don't send empty headers return self.serialize_json(self.data) diff --git a/anymail/backends/sendgrid_v2.py b/anymail/backends/sendgrid_v2.py index 41138b2..a12ae33 100644 --- a/anymail/backends/sendgrid_v2.py +++ b/anymail/backends/sendgrid_v2.py @@ -129,8 +129,10 @@ class SendGridPayload(RequestsPayload): self.data["x-smtpapi"] = self.serialize_json(self.data["x-smtpapi"]) # Serialize extra headers to json: - headers = self.data["headers"] - self.data["headers"] = self.serialize_json(dict(headers.items())) + if self.data["headers"]: + self.data["headers"] = self.serialize_json(self.data["headers"]) + else: + del self.data["headers"] return self.data diff --git a/anymail/backends/sendinblue.py b/anymail/backends/sendinblue.py index 7f2a30d..6ab6195 100644 --- a/anymail/backends/sendinblue.py +++ b/anymail/backends/sendinblue.py @@ -3,7 +3,7 @@ from requests.structures import CaseInsensitiveDict from .base_requests import AnymailRequestsBackend, RequestsPayload from ..exceptions import AnymailRequestsAPIError from ..message import AnymailRecipientStatus -from ..utils import get_anymail_setting, parse_address_list +from ..utils import get_anymail_setting class EmailBackend(AnymailRequestsBackend): @@ -88,15 +88,8 @@ class SendinBluePayload(RequestsPayload): def serialize_data(self): """Performs any necessary serialization on self.data, and returns the result.""" - headers = self.data["headers"] - if "Reply-To" in headers: - # Reply-To must be in its own param - reply_to = headers.pop('Reply-To') - self.set_reply_to(parse_address_list([reply_to])) - if len(headers) > 0: - self.data["headers"] = dict(headers) # flatten to normal dict for json serialization - else: - del self.data["headers"] # don't send empty headers + if not self.data['headers']: + del self.data['headers'] # don't send empty headers # SendinBlue use different argument's name if we use template functionality if self.template_id: @@ -179,8 +172,7 @@ class SendinBluePayload(RequestsPayload): self.data['replyTo'] = self.email_object(emails[0]) def set_extra_headers(self, headers): - for key in headers.keys(): - self.data['headers'][key] = headers[key] + self.data['headers'].update(headers) def set_tags(self, tags): if len(tags) > 0: diff --git a/anymail/backends/sparkpost.py b/anymail/backends/sparkpost.py index 622925f..95eadd2 100644 --- a/anymail/backends/sparkpost.py +++ b/anymail/backends/sparkpost.py @@ -147,7 +147,7 @@ class SparkPostPayload(BasePayload): def set_extra_headers(self, headers): if headers: - self.params['custom_headers'] = headers + self.params['custom_headers'] = dict(headers) # convert CaseInsensitiveDict to plain dict for SP lib def set_text_body(self, body): self.params['text'] = body diff --git a/tests/test_general_backend.py b/tests/test_general_backend.py index 8468de4..1d61ca2 100644 --- a/tests/test_general_backend.py +++ b/tests/test_general_backend.py @@ -318,3 +318,36 @@ class CatchCommonErrorsTests(TestBackendTestCase): self.message.reply_to = ugettext_lazy("single-reply-to@example.com") with self.assertRaisesMessage(TypeError, '"reply_to" attribute must be a list or other iterable'): self.message.send() + + +def flatten_emails(emails): + return [str(email) for email in emails] + + +class SpecialHeaderTests(TestBackendTestCase): + """Anymail should handle special extra_headers the same way Django does""" + + def test_reply_to(self): + """Django allows message.reply_to and message.extra_headers['Reply-To'], and the latter takes precedence""" + self.message.reply_to = ["attr@example.com"] + self.message.extra_headers = {"X-Extra": "extra"} + self.message.send() + params = self.get_send_params() + self.assertEqual(flatten_emails(params['reply_to']), ["attr@example.com"]) + self.assertEqual(params['extra_headers'], {"X-Extra": "extra"}) + + self.message.reply_to = None + self.message.extra_headers = {"Reply-To": "header@example.com", "X-Extra": "extra"} + self.message.send() + params = self.get_send_params() + self.assertEqual(flatten_emails(params['reply_to']), ["header@example.com"]) + self.assertEqual(params['extra_headers'], {"X-Extra": "extra"}) # Reply-To no longer there + + # If both are supplied, the header wins (to match Django EmailMessage.message() behavior). + # Also, header names are case-insensitive. + self.message.reply_to = ["attr@example.com"] + self.message.extra_headers = {"REPLY-to": "header@example.com", "X-Extra": "extra"} + self.message.send() + params = self.get_send_params() + self.assertEqual(flatten_emails(params['reply_to']), ["header@example.com"]) + self.assertEqual(params['extra_headers'], {"X-Extra": "extra"}) # Reply-To no longer there diff --git a/tests/test_sparkpost_backend.py b/tests/test_sparkpost_backend.py index 11b8738..4c0d3bd 100644 --- a/tests/test_sparkpost_backend.py +++ b/tests/test_sparkpost_backend.py @@ -135,8 +135,8 @@ class SparkPostBackendStandardEmailTests(SparkPostBackendMockAPITestCase): self.assertEqual(params['recipients'], ['to1@example.com', 'Also To ']) self.assertEqual(params['bcc'], ['bcc1@example.com', 'Also BCC ']) self.assertEqual(params['cc'], ['cc1@example.com', 'Also CC ']) + self.assertEqual(params['reply_to'], 'another@example.com') self.assertEqual(params['custom_headers'], { - 'Reply-To': 'another@example.com', 'X-MyHeader': 'my value', 'Message-ID': 'mycustommsgid@example.com'})