django-anymail/anymail/backends/sendgrid.py

from email.utils import quote as rfc822_quote
import warnings

from django.core.mail import make_msgid
from requests.structures import CaseInsensitiveDict

from .base_requests import AnymailRequestsBackend, RequestsPayload
from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError, AnymailWarning, AnymailDeprecationWarning
from ..message import AnymailRecipientStatus
from ..utils import get_anymail_setting, timestamp, update_deep, parse_address_list


class EmailBackend(AnymailRequestsBackend):
    """
    SendGrid v3 API Email Backend
    """

    esp_name = "SendGrid"

    def __init__(self, **kwargs):
        """Init options from Django settings"""
        esp_name = self.esp_name

        # Warn if v2-only username or password settings found
        username = get_anymail_setting('username', esp_name=esp_name, kwargs=kwargs, default=None, allow_bare=True)
        password = get_anymail_setting('password', esp_name=esp_name, kwargs=kwargs, default=None, allow_bare=True)
        if username or password:
            raise AnymailConfigurationError(
                "SendGrid v3 API doesn't support username/password auth; Please change to API key.\n"
                "(For legacy v2 API, use anymail.backends.sendgrid_v2.EmailBackend.)")

        self.api_key = get_anymail_setting('api_key', esp_name=esp_name, kwargs=kwargs, allow_bare=True)

        self.generate_message_id = get_anymail_setting('generate_message_id', esp_name=esp_name,
                                                       kwargs=kwargs, default=True)
        self.merge_field_format = get_anymail_setting('merge_field_format', esp_name=esp_name,
                                                      kwargs=kwargs, default=None)

        # Undocumented setting to disable workaround for SendGrid display-name quoting bug (see below).
        # If/when SendGrid fixes their API, recipient names will end up with extra double quotes
        # until Anymail is updated to remove the workaround. In the meantime, you can disable it
        # by adding `"SENDGRID_WORKAROUND_NAME_QUOTE_BUG": False` to your `ANYMAIL` settings.
        self.workaround_name_quote_bug = get_anymail_setting('workaround_name_quote_bug', esp_name=esp_name,
                                                             kwargs=kwargs, default=True)

        # This is SendGrid's newer Web API v3
        api_url = get_anymail_setting('api_url', esp_name=esp_name, kwargs=kwargs,
                                      default="https://api.sendgrid.com/v3/")
        if not api_url.endswith("/"):
            api_url += "/"
        super(EmailBackend, self).__init__(api_url, **kwargs)

    def build_message_payload(self, message, defaults):
        return SendGridPayload(message, defaults, self)

    def raise_for_status(self, response, payload, message):
        if response.status_code < 200 or response.status_code >= 300:
            raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response,
                                          backend=self)

    def parse_recipient_status(self, response, payload, message):
        # If we get here, the send call was successful.
        # (SendGrid uses a non-2xx response for any failures, caught in raise_for_status.)
        # SendGrid v3 doesn't provide any information in the response for a successful send,
        # so simulate a per-recipient status of "queued":
        status = AnymailRecipientStatus(message_id=payload.message_id, status="queued")
        return {recipient.email: status for recipient in payload.all_recipients}


# Pre-v0.8 naming (deprecated)
class SendGridBackend(EmailBackend):
    def __init__(self, **kwargs):
        warnings.warn(AnymailDeprecationWarning(
            "Please update your EMAIL_BACKEND setting to "
            "'anymail.backends.sendgrid.EmailBackend'"))
        super(SendGridBackend, self).__init__(**kwargs)


class SendGridPayload(RequestsPayload):

    def __init__(self, message, defaults, backend, *args, **kwargs):
        self.all_recipients = []  # used for backend.parse_recipient_status
        self.generate_message_id = backend.generate_message_id
        self.workaround_name_quote_bug = backend.workaround_name_quote_bug
        self.message_id = None  # Message-ID -- assigned in serialize_data unless provided in headers
        self.merge_field_format = backend.merge_field_format
        self.merge_data = None  # late-bound per-recipient data
        self.merge_global_data = None

        http_headers = kwargs.pop('headers', {})
        http_headers['Authorization'] = 'Bearer %s' % backend.api_key
        http_headers['Content-Type'] = 'application/json'
        http_headers['Accept'] = 'application/json'
        super(SendGridPayload, self).__init__(message, defaults, backend,
                                              headers=http_headers,
                                              *args, **kwargs)

    def get_api_endpoint(self):
        return "mail/send"

    def init_payload(self):
        self.data = {  # becomes json
            "personalizations": [{}],
            "headers": CaseInsensitiveDict(),
        }

    def serialize_data(self):
        """Performs any necessary serialization on self.data, and returns the result."""

        if self.generate_message_id:
            self.ensure_message_id()
        self.build_merge_data()

        headers = self.data["headers"]
        if "Reply-To" in headers:
            # Reply-To must be in its own param
            reply_to = headers.pop('Reply-To')
            self.set_reply_to(parse_address_list([reply_to]))
        if len(headers) > 0:
            self.data["headers"] = dict(headers)  # flatten to normal dict for json serialization
        else:
            del self.data["headers"]  # don't send empty headers

        return self.serialize_json(self.data)

    def ensure_message_id(self):
        """Ensure message has a known Message-ID for later event tracking"""
        if "Message-ID" not in self.data["headers"]:
            # Only make our own if caller hasn't already provided one
            self.data["headers"]["Message-ID"] = self.make_message_id()
        self.message_id = self.data["headers"]["Message-ID"]

        # Workaround for missing message ID (smtp-id) in SendGrid engagement events
        # (click and open tracking): because unique_args get merged into the raw event
        # record, we can supply the 'smtp-id' field for any events missing it.
        self.data.setdefault("custom_args", {})["smtp-id"] = self.message_id

    def make_message_id(self):
        """Returns a Message-ID that could be used for this payload

        Tries to use the from_email's domain as the Message-ID's domain
        """
        try:
            _, domain = self.data["from"]["email"].split("@")
        except (AttributeError, KeyError, TypeError, ValueError):
            domain = None
        return make_msgid(domain=domain)

    def build_merge_data(self):
        """Set personalizations[...]['substitutions'] and data['sections']"""
        merge_field_format = self.merge_field_format or '{}'

        if self.merge_data is not None:
            # Burst apart each to-email in personalizations[0] into a separate
            # personalization, and add merge_data for that recipient
            assert len(self.data["personalizations"]) == 1
            base_personalizations = self.data["personalizations"].pop()
            to_list = base_personalizations.pop("to")  # {email, name?} for each message.to
            all_fields = set()
            for recipient in to_list:
                personalization = base_personalizations.copy()  # captures cc, bcc, and any esp_extra
                personalization["to"] = [recipient]
                try:
                    recipient_data = self.merge_data[recipient["email"]]
                    personalization["substitutions"] = {merge_field_format.format(field): data
                                                        for field, data in recipient_data.items()}
                    all_fields = all_fields.union(recipient_data.keys())
                except KeyError:
                    pass  # no merge_data for this recipient
                self.data["personalizations"].append(personalization)

            if self.merge_field_format is None and all(field.isalnum() for field in all_fields):
                warnings.warn(
                    "Your SendGrid merge fields don't seem to have delimiters, "
                    "which can cause unexpected results with Anymail's merge_data. "
                    "Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
                    AnymailWarning)

        if self.merge_global_data is not None:
            # (merge into any existing 'sections' from esp_extra)
            self.data.setdefault("sections", {}).update({
                merge_field_format.format(field): data
                for field, data in self.merge_global_data.items()
            })

            # Confusingly, "Section tags have to be contained within a Substitution tag"
            # (https://sendgrid.com/docs/API_Reference/SMTP_API/section_tags.html),
            # so we need to insert a "-field-": "-field-" identity fallback for each
            # missing global field in the recipient substitutions...
            global_fields = [merge_field_format.format(field)
                             for field in self.merge_global_data.keys()]
            for personalization in self.data["personalizations"]:
                substitutions = personalization.setdefault("substitutions", {})
                substitutions.update({field: field for field in global_fields
                                      if field not in substitutions})

            if (self.merge_field_format is None and
                    all(field.isalnum() for field in self.merge_global_data.keys())):
                warnings.warn(
                    "Your SendGrid global merge fields don't seem to have delimiters, "
                    "which can cause unexpected results with Anymail's merge_data. "
                    "Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
                    AnymailWarning)

    #
    # Payload construction
    #

    @staticmethod
    def email_object(email, workaround_name_quote_bug=False):
        """Converts ParsedEmail to SendGrid API {email, name} dict"""
        obj = {"email": email.email}
        if email.name:
            # Work around SendGrid API bug: v3 fails to properly quote display-names
            # containing commas or semicolons in personalizations (but not in from_email
            # or reply_to). See https://github.com/sendgrid/sendgrid-python/issues/291.
            # We can work around the problem by quoting the name for SendGrid.
            if workaround_name_quote_bug:
                obj["name"] = '"%s"' % rfc822_quote(email.name)
            else:
                obj["name"] = email.name
        return obj

    def set_from_email(self, email):
        self.data["from"] = self.email_object(email)

    def set_recipients(self, recipient_type, emails):
        assert recipient_type in ["to", "cc", "bcc"]
        if emails:
            workaround_name_quote_bug = self.workaround_name_quote_bug
            # Normally, exactly one "personalizations" entry for all recipients
            # (Exception: with merge_data; will be burst apart later.)
            self.data["personalizations"][0][recipient_type] = \
                [self.email_object(email, workaround_name_quote_bug) for email in emails]
            self.all_recipients += emails  # used for backend.parse_recipient_status

    def set_subject(self, subject):
        if subject != "":  # see note in set_text_body about template rendering
            self.data["subject"] = subject

    def set_reply_to(self, emails):
        # SendGrid only supports a single address in the reply_to API param.
        if len(emails) > 1:
            self.unsupported_feature("multiple reply_to addresses")
        if len(emails) > 0:
            self.data["reply_to"] = self.email_object(emails[0])

    def set_extra_headers(self, headers):
        # SendGrid requires header values to be strings -- not integers.
        # We'll stringify ints and floats; anything else is the caller's responsibility.
        self.data["headers"].update({
            k: str(v) if isinstance(v, (int, float)) else v
            for k, v in headers.items()
        })

    def set_text_body(self, body):
        # Empty strings (the EmailMessage default) can cause unexpected SendGrid
        # template rendering behavior, such as ignoring the HTML template and
        # rendering HTML from the plaintext template instead.
        # Treat an empty string as a request to omit the body
        # (which means use the template content if present.)
        if body != "":
            self.data.setdefault("content", []).append({
                "type": "text/plain",
                "value": body,
            })

    def set_html_body(self, body):
        # SendGrid's API permits multiple html bodies
        # "If you choose to include the text/plain or text/html mime types, they must be
        # the first indices of the content array in the order text/plain, text/html."
        if body != "":  # see note in set_text_body about template rendering
            self.data.setdefault("content", []).append({
                "type": "text/html",
                "value": body,
            })

    def add_alternative(self, content, mimetype):
        # SendGrid is one of the few ESPs that supports arbitrary alternative parts in their API
        self.data.setdefault("content", []).append({
            "type": mimetype,
            "value": content,
        })

    def add_attachment(self, attachment):
        att = {
            "content": attachment.b64content,
            "type": attachment.mimetype,
            "filename": attachment.name or '',  # required -- submit empty string if unknown
        }
        if attachment.inline:
            att["disposition"] = "inline"
            att["content_id"] = attachment.cid
        self.data.setdefault("attachments", []).append(att)

    def set_metadata(self, metadata):
        # SendGrid requires custom_args values to be strings -- not integers.
        # (And issues the cryptic error {"field": null, "message": "Bad Request", "help": null}
        # if they're not.)
        # We'll stringify ints and floats; anything else is the caller's responsibility.
        self.data["custom_args"] = {
            k: str(v) if isinstance(v, (int, float)) else v
            for k, v in metadata.items()
        }

    def set_send_at(self, send_at):
        # Backend has converted pretty much everything to
        # a datetime by here; SendGrid expects unix timestamp
        self.data["send_at"] = int(timestamp(send_at))  # strip microseconds

    def set_tags(self, tags):
        self.data["categories"] = tags

    def set_track_clicks(self, track_clicks):
        self.data.setdefault("tracking_settings", {})["click_tracking"] = {
            "enable": track_clicks,
        }

    def set_track_opens(self, track_opens):
        # SendGrid's open_tracking setting also supports a "substitution_tag" parameter,
        # which Anymail doesn't offer directly. (You could add it through esp_extra.)
        self.data.setdefault("tracking_settings", {})["open_tracking"] = {
            "enable": track_opens,
        }

    def set_template_id(self, template_id):
        self.data["template_id"] = template_id

    def set_merge_data(self, merge_data):
        # Becomes personalizations[...]['substitutions'] in build_merge_data,
        # after we know recipients and merge_field_format.
        self.merge_data = merge_data

    def set_merge_global_data(self, merge_global_data):
        # Becomes data['section'] in build_merge_data, after we know merge_field_format.
        self.merge_global_data = merge_global_data

    def set_esp_extra(self, extra):
        self.merge_field_format = extra.pop("merge_field_format", self.merge_field_format)
        if "x-smtpapi" in extra:
            raise AnymailConfigurationError(
                "You are attempting to use SendGrid v2 API-style x-smtpapi params "
                "with the SendGrid v3 API. Please update your `esp_extra` to the new API, "
                "or use 'anymail.backends.sendgrid_v2.EmailBackend' for the old API."
            )
        update_deep(self.data, extra)