pacnpal/django-anymail
1
0
Fork 0
mirror of https://github.com/pacnpal/django-anymail.git synced 2025-12-20 03:41:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

SendGrid: update to new v3 send API (#50)

Browse Source 
SendGrid: update to v3 send API

**SendGrid:** **[possibly-breaking]** Update SendGrid backend to newer Web API v3. This should be a transparent change for most projects. Exceptions: if you use SendGrid username/password auth, esp_extra with "x-smtpapi", or multiple Reply-To addresses, please review the [porting notes](http://anymail.readthedocs.io/en/latest/esps/sendgrid/#sendgrid-v3-upgrade).

Closes #28
This commit is contained in:
Mike Edmunds
2017-01-19 14:29:15 -08:00
committed by GitHub
parent ac8147b0b8
commit e568e50d0c
11 changed files with 1891 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

353
anymail/backends/sendgrid.py
View File

@@ -1,44 +1,49 @@
from email.utils import quote as rfc822_quote
import warnings
from django.core.mail import make_msgid
from requests.structures import CaseInsensitiveDict
from .base_requests import AnymailRequestsBackend, RequestsPayload
from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError, AnymailWarning
from ..message import AnymailRecipientStatus
from ..utils import get_anymail_setting, timestamp
from .base_requests import AnymailRequestsBackend, RequestsPayload
from ..utils import get_anymail_setting, timestamp, update_deep
class SendGridBackend(AnymailRequestsBackend):
"""
SendGrid API Email Backend
SendGrid v3 API Email Backend
"""
def __init__(self, **kwargs):
"""Init options from Django settings"""
# Auth requires *either* SENDGRID_API_KEY or SENDGRID_USERNAME+SENDGRID_PASSWORD
esp_name = self.esp_name
self.api_key = get_anymail_setting('api_key', esp_name=esp_name, kwargs=kwargs,
default=None, allow_bare=True)
self.username = get_anymail_setting('username', esp_name=esp_name, kwargs=kwargs,
default=None, allow_bare=True)
self.password = get_anymail_setting('password', esp_name=esp_name, kwargs=kwargs,
default=None, allow_bare=True)
if self.api_key is None and (self.username is None or self.password is None):
# Warn if v2-only username or password settings found
username = get_anymail_setting('username', esp_name=esp_name, kwargs=kwargs, default=None, allow_bare=True)
password = get_anymail_setting('password', esp_name=esp_name, kwargs=kwargs, default=None, allow_bare=True)
if username or password:
raise AnymailConfigurationError(
"You must set either SENDGRID_API_KEY or both SENDGRID_USERNAME and "
"SENDGRID_PASSWORD in your Django ANYMAIL settings."
)
"SendGrid v3 API doesn't support username/password auth; Please change to API key.\n"
"(For legacy v2 API, use anymail.backends.sendgrid_v2.EmailBackend.)")
self.api_key = get_anymail_setting('api_key', esp_name=esp_name, kwargs=kwargs, allow_bare=True)
self.generate_message_id = get_anymail_setting('generate_message_id', esp_name=esp_name,
kwargs=kwargs, default=True)
self.merge_field_format = get_anymail_setting('merge_field_format', esp_name=esp_name,
kwargs=kwargs, default=None)
# This is SendGrid's Web API v2 (because the Web API v3 doesn't support sending)
# Undocumented setting to disable workaround for SendGrid display-name quoting bug (see below).
# If/when SendGrid fixes their API, recipient names will end up with extra double quotes
# until Anymail is updated to remove the workaround. In the meantime, you can disable it
# by adding `"SENDGRID_WORKAROUND_NAME_QUOTE_BUG": False` to your `ANYMAIL` settings.
self.workaround_name_quote_bug = get_anymail_setting('workaround_name_quote_bug', esp_name=esp_name,
kwargs=kwargs, default=True)
# This is SendGrid's newer Web API v3
api_url = get_anymail_setting('api_url', esp_name=esp_name, kwargs=kwargs,
default="https://api.sendgrid.com/api/")
default="https://api.sendgrid.com/v3/")
if not api_url.endswith("/"):
api_url += "/"
super(SendGridBackend, self).__init__(api_url, **kwargs)
@@ -46,18 +51,15 @@ class SendGridBackend(AnymailRequestsBackend):
def build_message_payload(self, message, defaults):
return SendGridPayload(message, defaults, self)
def raise_for_status(self, response, payload, message):
if response.status_code < 200 or response.status_code >= 300:
raise AnymailRequestsAPIError(email_message=message, payload=payload, response=response)
def parse_recipient_status(self, response, payload, message):
parsed_response = self.deserialize_json_response(response, payload, message)
try:
sendgrid_message = parsed_response["message"]
except (KeyError, TypeError):
raise AnymailRequestsAPIError("Invalid SendGrid API response format",
email_message=message, payload=payload, response=response)
if sendgrid_message != "success":
errors = parsed_response.get("errors", [])
raise AnymailRequestsAPIError("SendGrid send failed: '%s'" % "; ".join(errors),
email_message=message, payload=payload, response=response)
# Simulate a per-recipient status of "queued":
# If we get here, the send call was successful.
# (SendGrid uses a non-2xx response for any failures, caught in raise_for_status.)
# SendGrid v3 doesn't provide any information in the response for a successful send,
# so simulate a per-recipient status of "queued":
status = AnymailRecipientStatus(message_id=payload.message_id, status="queued")
return {recipient.email: status for recipient in payload.all_recipients}
@@ -67,78 +69,59 @@ class SendGridPayload(RequestsPayload):
def __init__(self, message, defaults, backend, *args, **kwargs):
self.all_recipients = [] # used for backend.parse_recipient_status
self.generate_message_id = backend.generate_message_id
self.workaround_name_quote_bug = backend.workaround_name_quote_bug
self.message_id = None # Message-ID -- assigned in serialize_data unless provided in headers
self.smtpapi = {} # SendGrid x-smtpapi field
self.to_list = [] # needed for build_merge_data
self.merge_field_format = backend.merge_field_format
self.merge_data = None # late-bound per-recipient data
self.merge_global_data = None
http_headers = kwargs.pop('headers', {})
query_params = kwargs.pop('params', {})
if backend.api_key is not None:
http_headers['Authorization'] = 'Bearer %s' % backend.api_key
else:
query_params['api_user'] = backend.username
query_params['api_key'] = backend.password
http_headers['Authorization'] = 'Bearer %s' % backend.api_key
http_headers['Content-Type'] = 'application/json'
http_headers['Accept'] = 'application/json'
super(SendGridPayload, self).__init__(message, defaults, backend,
params=query_params, headers=http_headers,
headers=http_headers,
*args, **kwargs)
def get_api_endpoint(self):
return "mail.send.json"
return "mail/send"
def init_payload(self):
self.data = { # becomes json
"personalizations": [{}],
"headers": CaseInsensitiveDict(),
}
def serialize_data(self):
"""Performs any necessary serialization on self.data, and returns the result."""
if self.generate_message_id:
self.ensure_message_id()
self.build_merge_data()
if self.merge_data is not None:
# Move the 'to' recipients to smtpapi, so SG does batch send
# (else all recipients would see each other's emails).
# Regular 'to' must still be a valid email (even though "ignored")...
# we use the from_email as recommended by SG support
# (See https://github.com/anymail/django-anymail/pull/14#issuecomment-220231250)
self.smtpapi['to'] = [email.address for email in self.to_list]
self.data['to'] = [self.data['from']]
self.data['toname'] = [self.data.get('fromname', " ")]
# Serialize x-smtpapi to json:
if len(self.smtpapi) > 0:
# If esp_extra was also used to set x-smtpapi, need to merge it
if "x-smtpapi" in self.data:
esp_extra_smtpapi = self.data["x-smtpapi"]
for key, value in esp_extra_smtpapi.items():
if key == "filters":
# merge filters (else it's difficult to mix esp_extra with other features)
self.smtpapi.setdefault(key, {}).update(value)
else:
# all other keys replace any current value
self.smtpapi[key] = value
self.data["x-smtpapi"] = self.serialize_json(self.smtpapi)
elif "x-smtpapi" in self.data:
self.data["x-smtpapi"] = self.serialize_json(self.data["x-smtpapi"])
# Serialize extra headers to json:
headers = self.data["headers"]
self.data["headers"] = self.serialize_json(dict(headers.items()))
if "Reply-To" in headers:
# Reply-To must be in its own param
reply_to = headers.pop('Reply-To')
self.set_reply_to([self.parsed_email(reply_to)])
if len(headers) > 0:
self.data["headers"] = dict(headers) # flatten to normal dict for json serialization
else:
del self.data["headers"] # don't send empty headers
return self.data
return self.serialize_json(self.data)
def ensure_message_id(self):
"""Ensure message has a known Message-ID for later event tracking"""
headers = self.data["headers"]
if "Message-ID" not in headers:
if "Message-ID" not in self.data["headers"]:
# Only make our own if caller hasn't already provided one
headers["Message-ID"] = self.make_message_id()
self.message_id = headers["Message-ID"]
self.data["headers"]["Message-ID"] = self.make_message_id()
self.message_id = self.data["headers"]["Message-ID"]
# Workaround for missing message ID (smtp-id) in SendGrid engagement events
# (click and open tracking): because unique_args get merged into the raw event
# record, we can supply the 'smtp-id' field for any events missing it.
self.smtpapi.setdefault('unique_args', {})['smtp-id'] = self.message_id
self.data.setdefault("custom_args", {})["smtp-id"] = self.message_id
def make_message_id(self):
"""Returns a Message-ID that could be used for this payload
@@ -146,20 +129,33 @@ class SendGridPayload(RequestsPayload):
Tries to use the from_email's domain as the Message-ID's domain
"""
try:
_, domain = self.data["from"].split("@")
_, domain = self.data["from"]["email"].split("@")
except (AttributeError, KeyError, TypeError, ValueError):
domain = None
return make_msgid(domain=domain)
def build_merge_data(self):
"""Set smtpapi['sub'] and ['section']"""
"""Set personalizations[...]['substitutions'] and data['sections']"""
merge_field_format = self.merge_field_format or '{}'
if self.merge_data is not None:
# Convert from {to1: {a: A1, b: B1}, to2: {a: A2}} (merge_data format)
# to {a: [A1, A2], b: [B1, ""]} ({field: [data in to-list order], ...})
# Burst apart each to-email in personalizations[0] into a separate
# personalization, and add merge_data for that recipient
assert len(self.data["personalizations"]) == 1
base_personalizations = self.data["personalizations"].pop()
to_list = base_personalizations.pop("to") # {email, name?} for each message.to
all_fields = set()
for recipient_data in self.merge_data.values():
all_fields = all_fields.union(recipient_data.keys())
recipients = [email.email for email in self.to_list]
for recipient in to_list:
personalization = base_personalizations.copy() # captures cc, bcc, and any esp_extra
personalization["to"] = [recipient]
try:
recipient_data = self.merge_data[recipient["email"]]
personalization["substitutions"] = {merge_field_format.format(field): data
for field, data in recipient_data.items()}
all_fields = all_fields.union(recipient_data.keys())
except KeyError:
pass # no merge_data for this recipient
self.data["personalizations"].append(personalization)
if self.merge_field_format is None and all(field.isalnum() for field in all_fields):
warnings.warn(
@@ -168,143 +164,172 @@ class SendGridPayload(RequestsPayload):
"Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
AnymailWarning)
sub_field_fmt = self.merge_field_format or '{}'
sub_fields = {field: sub_field_fmt.format(field) for field in all_fields}
self.smtpapi['sub'] = {
# If field data is missing for recipient, use (formatted) field as the substitution.
# (This allows default to resolve from global "section" substitutions.)
sub_fields[field]: [self.merge_data.get(recipient, {}).get(field, sub_fields[field])
for recipient in recipients]
for field in all_fields
}
if self.merge_global_data is not None:
section_field_fmt = self.merge_field_format or '{}'
self.smtpapi['section'] = {
section_field_fmt.format(field): data
# (merge into any existing 'sections' from esp_extra)
self.data.setdefault("sections", {}).update({
merge_field_format.format(field): data
for field, data in self.merge_global_data.items()
}
})
# Confusingly, "Section tags have to be contained within a Substitution tag"
# (https://sendgrid.com/docs/API_Reference/SMTP_API/section_tags.html),
# so we need to insert a "-field-": "-field-" identity fallback for each
# missing global field in the recipient substitutions...
global_fields = [merge_field_format.format(field)
for field in self.merge_global_data.keys()]
for personalization in self.data["personalizations"]:
substitutions = personalization.setdefault("substitutions", {})
substitutions.update({field: field for field in global_fields
if field not in substitutions})
if (self.merge_field_format is None and
all(field.isalnum() for field in self.merge_global_data.keys())):
warnings.warn(
"Your SendGrid global merge fields don't seem to have delimiters, "
"which can cause unexpected results with Anymail's merge_data. "
"Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
AnymailWarning)
#
# Payload construction
#
def init_payload(self):
self.data = {} # {field: [multiple, values]}
self.files = {}
self.data['headers'] = CaseInsensitiveDict() # headers keys are case-insensitive
@staticmethod
def email_object(email, workaround_name_quote_bug=False):
"""Converts ParsedEmail to SendGrid API {email, name} dict"""
obj = {"email": email.email}
if email.name:
# Work around SendGrid API bug: v3 fails to properly quote display-names
# containing commas or semicolons in personalizations (but not in from_email
# or reply_to). See https://github.com/sendgrid/sendgrid-python/issues/291.
# We can work around the problem by quoting the name for SendGrid.
if workaround_name_quote_bug:
obj["name"] = '"%s"' % rfc822_quote(email.name)
else:
obj["name"] = email.name
return obj
def set_from_email(self, email):
self.data["from"] = email.email
if email.name:
self.data["fromname"] = email.name
def set_to(self, emails):
self.to_list = emails # track for later use by build_merge_data
self.set_recipients('to', emails)
self.data["from"] = self.email_object(email)
def set_recipients(self, recipient_type, emails):
assert recipient_type in ["to", "cc", "bcc"]
if emails:
self.data[recipient_type] = [email.email for email in emails]
empty_name = " " # SendGrid API balks on complete empty name fields
self.data[recipient_type + "name"] = [email.name or empty_name for email in emails]
workaround_name_quote_bug = self.workaround_name_quote_bug
# Normally, exactly one "personalizations" entry for all recipients
# (Exception: with merge_data; will be burst apart later.)
self.data["personalizations"][0][recipient_type] = \
[self.email_object(email, workaround_name_quote_bug) for email in emails]
self.all_recipients += emails # used for backend.parse_recipient_status
def set_subject(self, subject):
self.data["subject"] = subject
if subject != "": # see note in set_text_body about template rendering
self.data["subject"] = subject
def set_reply_to(self, emails):
# Note: SendGrid mangles the 'replyto' API param: it drops
# all but the last email in a multi-address replyto, and
# drops all the display names. [tested 2016-03-10]
#
# To avoid those quirks, we provide a fully-formed Reply-To
# in the custom headers, which makes it through intact.
if emails:
reply_to = ", ".join([email.address for email in emails])
self.data["headers"]["Reply-To"] = reply_to
# SendGrid only supports a single address in the reply_to API param.
if len(emails) > 1:
self.unsupported_feature("multiple reply_to addresses")
if len(emails) > 0:
self.data["reply_to"] = self.email_object(emails[0])
def set_extra_headers(self, headers):
# SendGrid requires header values to be strings -- not integers.
# We'll stringify ints and floats; anything else is the caller's responsibility.
# (This field gets converted to json in self.serialize_data)
self.data["headers"].update({
k: str(v) if isinstance(v, (int, float)) else v
for k, v in headers.items()
})
def set_text_body(self, body):
self.data["text"] = body
# Empty strings (the EmailMessage default) can cause unexpected SendGrid
# template rendering behavior, such as ignoring the HTML template and
# rendering HTML from the plaintext template instead.
# Treat an empty string as a request to omit the body
# (which means use the template content if present.)
if body != "":
self.data.setdefault("content", []).append({
"type": "text/plain",
"value": body,
})
def set_html_body(self, body):
if "html" in self.data:
# second html body could show up through multiple alternatives, or html body + alternative
self.unsupported_feature("multiple html parts")
self.data["html"] = body
# SendGrid's API permits multiple html bodies
# "If you choose to include the text/plain or text/html mime types, they must be
# the first indices of the content array in the order text/plain, text/html."
if body != "": # see note in set_text_body about template rendering
self.data.setdefault("content", []).append({
"type": "text/html",
"value": body,
})
def add_alternative(self, content, mimetype):
# SendGrid is one of the few ESPs that supports arbitrary alternative parts in their API
self.data.setdefault("content", []).append({
"type": mimetype,
"value": content,
})
def add_attachment(self, attachment):
filename = attachment.name or ""
att = {
"content": attachment.b64content,
"type": attachment.mimetype,
"filename": attachment.name or '', # required -- submit empty string if unknown
}
if attachment.inline:
filename = filename or attachment.cid # must have non-empty name for the cid matching
content_field = "content[%s]" % filename
self.data[content_field] = attachment.cid
files_field = "files[%s]" % filename
if files_field in self.files:
# It's possible SendGrid could actually handle this case (needs testing),
# but requests doesn't seem to accept a list of tuples for a files field.
# (See the MailgunBackend version for a different approach that might work.)
self.unsupported_feature(
"multiple attachments with the same filename ('%s')" % filename if filename
else "multiple unnamed attachments")
self.files[files_field] = (filename, attachment.content, attachment.mimetype)
att["disposition"] = "inline"
att["content_id"] = attachment.cid
self.data.setdefault("attachments", []).append(att)
def set_metadata(self, metadata):
self.smtpapi['unique_args'] = metadata
# SendGrid requires custom_args values to be strings -- not integers.
# (And issues the cryptic error {"field": null, "message": "Bad Request", "help": null}
# if they're not.)
# We'll stringify ints and floats; anything else is the caller's responsibility.
self.data["custom_args"] = {
k: str(v) if isinstance(v, (int, float)) else v
for k, v in metadata.items()
}
def set_send_at(self, send_at):
# Backend has converted pretty much everything to
# a datetime by here; SendGrid expects unix timestamp
self.smtpapi["send_at"] = int(timestamp(send_at)) # strip microseconds
self.data["send_at"] = int(timestamp(send_at)) # strip microseconds
def set_tags(self, tags):
self.smtpapi["category"] = tags
def add_filter(self, filter_name, setting, val):
self.smtpapi.setdefault('filters', {})\
.setdefault(filter_name, {})\
.setdefault('settings', {})[setting] = val
self.data["categories"] = tags
def set_track_clicks(self, track_clicks):
self.add_filter('clicktrack', 'enable', int(track_clicks))
self.data.setdefault("tracking_settings", {})["click_tracking"] = {
"enable": track_clicks,
}
def set_track_opens(self, track_opens):
# SendGrid's opentrack filter also supports a "replace"
# parameter, which Anymail doesn't offer directly.
# (You could add it through esp_extra.)
self.add_filter('opentrack', 'enable', int(track_opens))
# SendGrid's open_tracking setting also supports a "substitution_tag" parameter,
# which Anymail doesn't offer directly. (You could add it through esp_extra.)
self.data.setdefault("tracking_settings", {})["open_tracking"] = {
"enable": track_opens,
}
def set_template_id(self, template_id):
self.add_filter('templates', 'enable', 1)
self.add_filter('templates', 'template_id', template_id)
# Must ensure text and html are non-empty, or template parts won't render.
# https://sendgrid.com/docs/API_Reference/Web_API_v3/Transactional_Templates/smtpapi.html#-Text-or-HTML-Templates
if not self.data.get("text", ""):
self.data["text"] = " "
if not self.data.get("html", ""):
self.data["html"] = " "
self.data["template_id"] = template_id
def set_merge_data(self, merge_data):
# Becomes smtpapi['sub'] in build_merge_data, after we know recipients and merge_field_format.
# Becomes personalizations[...]['substitutions'] in build_merge_data,
# after we know recipients and merge_field_format.
self.merge_data = merge_data
def set_merge_global_data(self, merge_global_data):
# Becomes smtpapi['section'] in build_merge_data, after we know merge_field_format.
# Becomes data['section'] in build_merge_data, after we know merge_field_format.
self.merge_global_data = merge_global_data
def set_esp_extra(self, extra):
self.merge_field_format = extra.pop('merge_field_format', self.merge_field_format)
self.data.update(extra)
self.merge_field_format = extra.pop("merge_field_format", self.merge_field_format)
if "x-smtpapi" in extra:
raise AnymailConfigurationError(
"You are attempting to use SendGrid v2 API-style x-smtpapi params "
"with the SendGrid v3 API. Please update your `esp_extra` to the new API, "
"or use 'anymail.backends.sendgrid_v2.EmailBackend' for the old API."
)
update_deep(self.data, extra)