Amazon SES: Work around SES bug that corrupts non-ASCII message bodies.

If you are using an SES ConfigurationSet with open or click tracking
enabled, SES replaces non-ASCII characters with question marks as it
rewrites the message to add tracking, if the bodies are sent with
`Content-Transfer-Encoding: 8bit` (which is Django's default for utf8
body parts).

Force potentially problematic parts to use CTE: quoted-printable
as a workaround.

Fixes #115.

CHANGELOG.rst

@@ -60,6 +60,11 @@ Fixes
   ending in ".com" could cause Gmail to block messages sent with inline attachments.
   (Mailgun, Mailjet, Mandrill and SparkPost have APIs affected by this.
   See `#112`_ for more details.)
+* **Amazon SES:** Work around an
+  `Amazon SES bug <https://forums.aws.amazon.com/thread.jspa?threadID=287048>`__
+  that can corrupt non-ASCII message bodies if you are using SES's open or click
+  tracking. (See `#115`_ for more details. Thanks to `@varche1`_ for isolating
+  the specific conditions that trigger the bug.)
 
 Other
 ~~~~~
@@ -774,6 +779,7 @@ Features
 .. _#110: https://github.com/anymail/issues/110
 .. _#111: https://github.com/anymail/issues/111
 .. _#112: https://github.com/anymail/issues/112
+.. _#115: https://github.com/anymail/issues/115
 
 .. _@calvin: https://github.com/calvin
 .. _@joshkersey: https://github.com/joshkersey
@@ -781,4 +787,5 @@ Features
 .. _@lewistaylor: https://github.com/lewistaylor
 .. _@RignonNoel: https://github.com/RignonNoel
 .. _@sebbacon: https://github.com/sebbacon
+.. _@varche1: https://github.com/varche1
 .. _@yourcelf: https://github.com/yourcelf

anymail/backends/amazon_ses.py

@@ -1,5 +1,7 @@
+from email.charset import Charset, QP
 from email.header import Header
 from email.mime.base import MIMEBase
+from email.mime.text import MIMEText
 
 from django.core.mail import BadHeaderError
 
@@ -130,6 +132,23 @@ class AmazonSESSendRawEmailPayload(AmazonSESBasePayload):
             # (message.message() will have already checked subject for BadHeaderError)
             self.mime_message.replace_header("Subject", HeaderBugWorkaround(self.message.subject))
 
+        # Work around an Amazon SES bug where, if all of:
+        #   - the message body (text or html) contains non-ASCII characters
+        #   - the body is sent with `Content-Transfer-Encoding: 8bit`
+        #     (which is Django email's default for most non-ASCII bodies)
+        #   - you are using an SES ConfigurationSet with open or click tracking enabled
+        # then SES replaces the non-ASCII characters with question marks as it rewrites
+        # the message to add tracking. Forcing `CTE: quoted-printable` avoids the problem.
+        # (https://forums.aws.amazon.com/thread.jspa?threadID=287048)
+        for part in self.mime_message.walk():
+            if part.get_content_maintype() == "text" and part["Content-Transfer-Encoding"] == "8bit":
+                content = part.get_payload()
+                del part["Content-Transfer-Encoding"]
+                qp_charset = Charset(part.get_content_charset("us-ascii"))
+                qp_charset.body_encoding = QP
+                # (can't use part.set_payload, because SafeMIMEText can undo this workaround)
+                MIMEText.set_payload(part, content, charset=qp_charset)
+
     def call_send_api(self, ses_client):
         self.params["RawMessage"] = {
             # Note: "Destinations" is determined from message headers if not provided

tests/test_amazon_ses_backend.py

@@ -244,6 +244,26 @@ class AmazonSESBackendStandardEmailTests(AmazonSESBackendMockAPITestCase):
         sent_message = self.get_sent_message()
         self.assertEqual(sent_message["Subject"], self.message.subject)
 
+    def test_body_avoids_cte_8bit(self):
+        """Anymail works around an Amazon SES bug that can corrupt non-ASCII bodies."""
+        # (see detailed comments in the backend code)
+        self.message.body = "Это text body"
+        self.message.attach_alternative("<p>Это html body</p>", "text/html")
+        self.message.send()
+        sent_message = self.get_sent_message()
+
+        # Make sure none of the text parts use `Content-Transfer-Encoding: 8bit`.
+        # (Technically, either quoted-printable or base64 would be OK, but base64 text parts
+        # have a reputation for triggering spam filters, so just require quoted-printable.)
+        text_part_encodings = [
+            (part.get_content_type(), part["Content-Transfer-Encoding"])
+            for part in sent_message.walk()
+            if part.get_content_maintype() == "text"]
+        self.assertEqual(text_part_encodings, [
+            ("text/plain", "quoted-printable"),
+            ("text/html", "quoted-printable"),
+        ])
+
     def test_api_failure(self):
         error_response = {
             'Error': {