Event-tracking webhooks

Closes #3

anymail/webhooks/base.py

@@ -0,0 +1,139 @@
+import base64
+import re
+import six
+import warnings
+
+from django.http import HttpResponse
+from django.utils.decorators import method_decorator
+from django.views.decorators.csrf import csrf_exempt
+from django.views.generic import View
+
+from ..exceptions import AnymailInsecureWebhookWarning, AnymailWebhookValidationFailure
+from ..utils import get_anymail_setting, collect_all_methods
+
+
+class AnymailBasicAuthMixin(object):
+    """Implements webhook basic auth as mixin to AnymailBaseWebhookView."""
+
+    # Whether to warn if basic auth is not configured.
+    # For most ESPs, basic auth is the only webhook security,
+    # so the default is True. Subclasses can set False if
+    # they enforce other security (like signed webhooks).
+    warn_if_no_basic_auth = True
+
+    # List of allowable HTTP basic-auth 'user:pass' strings.
+    basic_auth = None  # (Declaring class attr allows override by kwargs in View.as_view.)
+
+    def __init__(self, **kwargs):
+        self.basic_auth = get_anymail_setting('webhook_authorization', default=[],
+                                              kwargs=kwargs)  # no esp_name -- auth is shared between ESPs
+        # Allow a single string:
+        if isinstance(self.basic_auth, six.string_types):
+            self.basic_auth = [self.basic_auth]
+        if self.warn_if_no_basic_auth and len(self.basic_auth) < 1:
+            warnings.warn(
+                "Your Anymail webhooks are insecure and open to anyone on the web. "
+                "You should set WEBHOOK_AUTHORIZATION in your ANYMAIL settings. "
+                "See 'Securing webhooks' in the Anymail docs.",
+                AnymailInsecureWebhookWarning)
+        # noinspection PyArgumentList
+        super(AnymailBasicAuthMixin, self).__init__(**kwargs)
+
+    def validate_request(self, request):
+        """If configured for webhook basic auth, validate request has correct auth."""
+        if self.basic_auth:
+            valid = False
+            try:
+                authtype, authdata = request.META['HTTP_AUTHORIZATION'].split()
+                if authtype.lower() == "basic":
+                    auth = base64.b64decode(authdata).decode('utf-8')
+                    if auth in self.basic_auth:
+                        valid = True
+            except (IndexError, KeyError, TypeError, ValueError):
+                valid = False
+            if not valid:
+                # noinspection PyUnresolvedReferences
+                raise AnymailWebhookValidationFailure(
+                    "Missing or invalid basic auth in Anymail %s webhook" % self.esp_name)
+
+
+# Mixin note: Django's View.__init__ doesn't cooperate with chaining,
+# so all mixins that need __init__ must appear before View in MRO.
+class AnymailBaseWebhookView(AnymailBasicAuthMixin, View):
+    """Base view for processing ESP event webhooks
+
+    ESP-specific implementations should subclass
+    and implement parse_events. They may also
+    want to implement validate_request
+    if additional security is available.
+    """
+
+    def __init__(self, **kwargs):
+        super(AnymailBaseWebhookView, self).__init__(**kwargs)
+        self.validators = collect_all_methods(self.__class__, 'validate_request')
+
+    # Subclass implementation:
+
+    # Where to send events: either ..signals.inbound or ..signals.tracking
+    signal = None
+
+    def validate_request(self, request):
+        """Check validity of webhook post, or raise AnymailWebhookValidationFailure.
+
+        AnymailBaseWebhookView includes basic auth validation.
+        Subclasses can implement (or provide via mixins) if the ESP supports
+        additional validation (such as signature checking).
+
+        *All* definitions of this method in the class chain (including mixins)
+        will be called. There is no need to chain to the superclass.
+        (See self.run_validators and collect_all_methods.)
+        """
+        # if request.POST['signature'] != expected_signature:
+        #     raise AnymailWebhookValidationFailure("...message...")
+        # (else just do nothing)
+        pass
+
+    def parse_events(self, request):
+        """Return a list of normalized AnymailWebhookEvent extracted from ESP post data.
+
+        Subclasses must implement.
+        """
+        raise NotImplementedError()
+
+    # HTTP handlers (subclasses shouldn't need to override):
+
+    http_method_names = ["post", "head", "options"]
+
+    def head(self, request, *args, **kwargs):
+        # Some ESPs verify the webhook with a HEAD request at configuration time
+        return HttpResponse()
+
+    @method_decorator(csrf_exempt)
+    def post(self, request, *args, **kwargs):
+        # Normal Django exception handling will do the right thing:
+        # - AnymailWebhookValidationFailure will turn into an HTTP 400 response
+        #   (via Django SuspiciousOperation handling)
+        # - Any other errors (e.g., in signal dispatch) will turn into HTTP 500
+        #   responses (via normal Django error handling). ESPs generally
+        #   treat that as "try again later".
+        self.run_validators(request)
+        events = self.parse_events(request)
+        esp_name = self.esp_name
+        for event in events:
+            self.signal.send(sender=self.__class__, event=event, esp_name=esp_name)
+        return HttpResponse()
+
+    # Request validation (subclasses shouldn't need to override):
+
+    def run_validators(self, request):
+        for validator in self.validators:
+            validator(self, request)
+
+    @property
+    def esp_name(self):
+        """
+        Read-only name of the ESP for this webhook view.
+
+        (E.g., MailgunTrackingWebhookView will return "Mailgun")
+        """
+        return re.sub(r'(Tracking|Inbox)WebhookView$', "", self.__class__.__name__)

anymail/webhooks/mailgun.py

@@ -0,0 +1,133 @@
+import json
+from datetime import datetime
+
+import hashlib
+import hmac
+from django.utils.timezone import utc
+
+from .base import AnymailBaseWebhookView
+from ..exceptions import AnymailWebhookValidationFailure
+from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+from ..utils import get_anymail_setting, combine
+
+
+class MailgunBaseWebhookView(AnymailBaseWebhookView):
+    """Base view class for Mailgun webhooks"""
+
+    warn_if_no_basic_auth = False  # because we validate against signature
+
+    api_key = None  # (Declaring class attr allows override by kwargs in View.as_view.)
+
+    def __init__(self, **kwargs):
+        api_key = get_anymail_setting('api_key', esp_name=self.esp_name,
+                                      kwargs=kwargs, allow_bare=True)
+        self.api_key = api_key.encode('ascii')  # hmac.new requires bytes key in python 3
+        super(MailgunBaseWebhookView, self).__init__(**kwargs)
+
+    def validate_request(self, request):
+        super(MailgunBaseWebhookView, self).validate_request(request)  # first check basic auth if enabled
+        try:
+            token = request.POST['token']
+            timestamp = request.POST['timestamp']
+            signature = str(request.POST['signature'])  # force to same type as hexdigest() (for python2)
+        except KeyError:
+            raise AnymailWebhookValidationFailure("Mailgun webhook called without required security fields")
+        expected_signature = hmac.new(key=self.api_key, msg='{}{}'.format(timestamp, token).encode('ascii'),
+                                      digestmod=hashlib.sha256).hexdigest()
+        if not hmac.compare_digest(signature, expected_signature):
+            raise AnymailWebhookValidationFailure("Mailgun webhook called with incorrect signature")
+
+    def parse_events(self, request):
+        return [self.esp_to_anymail_event(request.POST)]
+
+    def esp_to_anymail_event(self, esp_event):
+        raise NotImplementedError()
+
+
+class MailgunTrackingWebhookView(MailgunBaseWebhookView):
+    """Handler for Mailgun delivery and engagement tracking webhooks"""
+
+    signal = tracking
+
+    event_types = {
+        # Map Mailgun event: Anymail normalized type
+        'delivered': EventType.DELIVERED,
+        'dropped': EventType.REJECTED,
+        'bounced': EventType.BOUNCED,
+        'complained': EventType.COMPLAINED,
+        'unsubscribed': EventType.UNSUBSCRIBED,
+        'opened': EventType.OPENED,
+        'clicked': EventType.CLICKED,
+        # Mailgun does not send events corresponding to QUEUED or DEFERRED
+    }
+
+    reject_reasons = {
+        # Map Mailgun (SMTP) error codes to Anymail normalized reject_reason.
+        # By default, we will treat anything 400-599 as REJECT_BOUNCED
+        # so only exceptions are listed here.
+        499: RejectReason.TIMED_OUT,  # unable to connect to MX (also covers invalid recipients)
+        # These 6xx codes appear to be Mailgun extensions to SMTP
+        # (and don't seem to be documented anywhere):
+        605: RejectReason.BOUNCED,  # previous bounce
+        607: RejectReason.SPAM,  # previous spam complaint
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        # esp_event is a Django QueryDict (from request.POST),
+        # which has multi-valued fields, but is *not* case-insensitive
+
+        event_type = self.event_types.get(esp_event['event'], EventType.UNKNOWN)
+        timestamp = datetime.fromtimestamp(int(esp_event['timestamp']), tz=utc)
+        # Message-Id is not documented for every event, but seems to always be included.
+        # (It's sometimes spelled as 'message-id', lowercase, and missing the <angle-brackets>.)
+        message_id = esp_event.get('Message-Id', esp_event.get('message-id', None))
+        if message_id and not message_id.startswith('<'):
+            message_id = "<{}>".format(message_id)
+
+        description = esp_event.get('description', None)
+        mta_response = esp_event.get('error', esp_event.get('notification', None))
+        reject_reason = None
+        try:
+            mta_status = int(esp_event['code'])
+        except (KeyError, TypeError):
+            pass
+        else:
+            reject_reason = self.reject_reasons.get(
+                mta_status,
+                RejectReason.BOUNCED if 400 <= mta_status < 600
+                else RejectReason.OTHER)
+
+        # Mailgun merges metadata fields with the other event fields.
+        # However, it also includes the original message headers,
+        # which have the metadata separately as X-Mailgun-Variables.
+        try:
+            headers = json.loads(esp_event['message-headers'])
+        except (KeyError, ):
+            metadata = None
+        else:
+            variables = [value for [field, value] in headers
+                         if field == 'X-Mailgun-Variables']
+            if len(variables) >= 1:
+                # Each X-Mailgun-Variables value is JSON. Parse and merge them all into single dict:
+                metadata = combine(*[json.loads(value) for value in variables])
+            else:
+                metadata = None
+
+        # tags are sometimes delivered as X-Mailgun-Tag fields, sometimes as tag
+        tags = esp_event.getlist('tag', esp_event.getlist('X-Mailgun-Tag', None))
+
+        return AnymailTrackingEvent(
+            event_type=event_type,
+            timestamp=timestamp,
+            message_id=message_id,
+            event_id=esp_event.get('token', None),
+            recipient=esp_event.get('recipient', None),
+            reject_reason=reject_reason,
+            description=description,
+            mta_response=mta_response,
+            tags=tags,
+            metadata=metadata,
+            click_url=esp_event.get('url', None),
+            user_agent=esp_event.get('user-agent', None),
+            esp_event=esp_event,
+        )

anymail/webhooks/mandrill.py

@@ -0,0 +1,140 @@
+import json
+from datetime import datetime
+
+import hashlib
+import hmac
+from base64 import b64encode
+from django.utils.timezone import utc
+
+from .base import AnymailBaseWebhookView
+from ..exceptions import AnymailWebhookValidationFailure, AnymailConfigurationError
+from ..signals import tracking, AnymailTrackingEvent, EventType
+from ..utils import get_anymail_setting, getfirst
+
+
+class MandrillSignatureMixin(object):
+    """Validates Mandrill webhook signature"""
+
+    # These can be set from kwargs in View.as_view, or pulled from settings in init:
+    webhook_key = None  # required
+    webhook_url = None  # optional; defaults to actual url used
+
+    def __init__(self, **kwargs):
+        # noinspection PyUnresolvedReferences
+        esp_name = self.esp_name
+        webhook_key = get_anymail_setting('webhook_key', esp_name=esp_name,
+                                          kwargs=kwargs, allow_bare=True)
+        self.webhook_key = webhook_key.encode('ascii')  # hmac.new requires bytes key in python 3
+        self.webhook_url = get_anymail_setting('webhook_url', esp_name=esp_name, default=None,
+                                               kwargs=kwargs, allow_bare=True)
+        # noinspection PyArgumentList
+        super(MandrillSignatureMixin, self).__init__(**kwargs)
+
+    def validate_request(self, request):
+        try:
+            signature = request.META["HTTP_X_MANDRILL_SIGNATURE"]
+        except KeyError:
+            raise AnymailWebhookValidationFailure("X-Mandrill-Signature header missing from webhook POST")
+
+        # Mandrill signs the exact URL plus the sorted POST params:
+        signed_data = self.webhook_url or request.build_absolute_uri()
+        params = request.POST.dict()
+        for key in sorted(params.keys()):
+            signed_data += key + params[key]
+
+        expected_signature = b64encode(hmac.new(key=self.webhook_key, msg=signed_data.encode('utf-8'),
+                                                digestmod=hashlib.sha1).digest())
+        if not hmac.compare_digest(signature, expected_signature):
+            raise AnymailWebhookValidationFailure("Mandrill webhook called with incorrect signature")
+
+
+class MandrillBaseWebhookView(MandrillSignatureMixin, AnymailBaseWebhookView):
+    """Base view class for Mandrill webhooks"""
+
+    warn_if_no_basic_auth = False  # because we validate against signature
+
+    def parse_events(self, request):
+        esp_events = json.loads(request.POST['mandrill_events'])
+        return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
+
+    def esp_to_anymail_event(self, esp_event):
+        raise NotImplementedError()
+
+
+class MandrillTrackingWebhookView(MandrillBaseWebhookView):
+
+    signal = tracking
+
+    event_types = {
+        # Message events:
+        'send': EventType.SENT,
+        'deferral': EventType.DEFERRED,
+        'hard_bounce': EventType.BOUNCED,
+        'soft_bounce': EventType.DEFERRED,
+        'open': EventType.OPENED,
+        'click': EventType.CLICKED,
+        'spam': EventType.COMPLAINED,
+        'unsub': EventType.UNSUBSCRIBED,
+        'reject': EventType.REJECTED,
+        # Sync events (we don't really normalize these well):
+        'whitelist': EventType.UNKNOWN,
+        'blacklist': EventType.UNKNOWN,
+        # Inbound events:
+        'inbound': EventType.INBOUND,
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        esp_type = getfirst(esp_event, ['event', 'type'], None)
+        event_type = self.event_types.get(esp_type, EventType.UNKNOWN)
+        if event_type == EventType.INBOUND:
+            raise AnymailConfigurationError(
+                "You seem to have set Mandrill's *inbound* webhook URL "
+                "to Anymail's Mandrill *tracking* webhook URL.")
+
+        try:
+            timestamp = datetime.fromtimestamp(esp_event['ts'], tz=utc)
+        except (KeyError, ValueError):
+            timestamp = None
+
+        try:
+            recipient = esp_event['msg']['email']
+        except KeyError:
+            try:
+                recipient = esp_event['reject']['email']  # sync events
+            except KeyError:
+                recipient = None
+
+        try:
+            mta_response = esp_event['msg']['diag']
+        except KeyError:
+            mta_response = None
+
+        try:
+            description = getfirst(esp_event['reject'], ['detail', 'reason'])
+        except KeyError:
+            description = None
+
+        try:
+            metadata = esp_event['msg']['metadata']
+        except KeyError:
+            metadata = None
+
+        try:
+            tags = esp_event['msg']['tags']
+        except KeyError:
+            tags = None
+
+        return AnymailTrackingEvent(
+            click_url=esp_event.get('url', None),
+            description=description,
+            esp_event=esp_event,
+            event_type=event_type,
+            message_id=esp_event.get('_id', None),
+            metadata=metadata,
+            mta_response=mta_response,
+            recipient=recipient,
+            reject_reason=None,  # probably map esp_event['msg']['bounce_description'], but insufficient docs
+            tags=tags,
+            timestamp=timestamp,
+            user_agent=esp_event.get('user_agent', None),
+        )

anymail/webhooks/postmark.py

@@ -0,0 +1,104 @@
+import json
+
+from django.utils.dateparse import parse_datetime
+
+from .base import AnymailBaseWebhookView
+from ..exceptions import AnymailConfigurationError
+from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+from ..utils import getfirst
+
+
+class PostmarkBaseWebhookView(AnymailBaseWebhookView):
+    """Base view class for Postmark webhooks"""
+
+    def parse_events(self, request):
+        esp_event = json.loads(request.body.decode('utf-8'))
+        return [self.esp_to_anymail_event(esp_event)]
+
+    def esp_to_anymail_event(self, esp_event):
+        raise NotImplementedError()
+
+
+class PostmarkTrackingWebhookView(PostmarkBaseWebhookView):
+    """Handler for Postmark delivery and engagement tracking webhooks"""
+
+    signal = tracking
+
+    event_types = {
+        # Map Postmark event type: Anymail normalized (event type, reject reason)
+        'HardBounce': (EventType.BOUNCED, RejectReason.BOUNCED),
+        'Transient': (EventType.DEFERRED, None),
+        'Unsubscribe': (EventType.UNSUBSCRIBED, RejectReason.UNSUBSCRIBED),
+        'Subscribe': (EventType.SUBSCRIBED, None),
+        'AutoResponder': (EventType.AUTORESPONDED, None),
+        'AddressChange': (EventType.AUTORESPONDED, None),
+        'DnsError': (EventType.DEFERRED, None),  # "temporary DNS error"
+        'SpamNotification': (EventType.COMPLAINED, RejectReason.SPAM),
+        'OpenRelayTest': (EventType.DEFERRED, None),  # Receiving MTA is testing Postmark
+        'Unknown': (EventType.UNKNOWN, None),
+        'SoftBounce': (EventType.DEFERRED, RejectReason.BOUNCED),  # until HardBounce later
+        'VirusNotification': (EventType.BOUNCED, RejectReason.OTHER),
+        'ChallengeVerification': (EventType.AUTORESPONDED, None),
+        'BadEmailAddress': (EventType.REJECTED, RejectReason.INVALID),
+        'SpamComplaint': (EventType.COMPLAINED, RejectReason.SPAM),
+        'ManuallyDeactivated': (EventType.REJECTED, RejectReason.BLOCKED),
+        'Unconfirmed': (EventType.REJECTED, None),
+        'Blocked': (EventType.REJECTED, RejectReason.BLOCKED),
+        'SMTPApiError': (EventType.FAILED, None),  # could occur if user also using Postmark SMTP directly
+        'InboundError': (EventType.INBOUND_FAILED, None),
+        'DMARCPolicy': (EventType.REJECTED, RejectReason.BLOCKED),
+        'TemplateRenderingFailed': (EventType.FAILED, None),
+        # Postmark does not report DELIVERED
+        # Postmark does not report CLICKED (because it doesn't implement click-tracking)
+        # OPENED doesn't have a Type field; detected separately below
+        # INBOUND doesn't have a Type field; should come in through different webhook
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        reject_reason = None
+        try:
+            esp_type = esp_event['Type']
+            event_type, reject_reason = self.event_types.get(esp_type, (EventType.UNKNOWN, None))
+        except KeyError:
+            if 'FirstOpen' in esp_event:
+                event_type = EventType.OPENED
+            elif 'From' in esp_event:
+                # This is an inbound event
+                raise AnymailConfigurationError(
+                    "You seem to have set Postmark's *inbound* webhook URL "
+                    "to Anymail's Postmark *tracking* webhook URL.")
+            else:
+                event_type = EventType.UNKNOWN
+
+        recipient = getfirst(esp_event, ['Email', 'Recipient'], None)  # Email for bounce; Recipient for open
+
+        try:
+            timestr = getfirst(esp_event, ['BouncedAt', 'ReceivedAt'])
+        except KeyError:
+            timestamp = None
+        else:
+            timestamp = parse_datetime(timestr)
+
+        try:
+            event_id = str(esp_event['ID'])  # only in bounce events
+        except KeyError:
+            event_id = None
+
+        try:
+            tags = [esp_event['Tag']]
+        except KeyError:
+            tags = None
+
+        return AnymailTrackingEvent(
+            description=esp_event.get('Description', None),
+            esp_event=esp_event,
+            event_id=event_id,
+            event_type=event_type,
+            message_id=esp_event.get('MessageID', None),
+            mta_response=esp_event.get('Details', None),
+            recipient=recipient,
+            reject_reason=reject_reason,
+            tags=tags,
+            timestamp=timestamp,
+            user_agent=esp_event.get('UserAgent', None),
+        )

anymail/webhooks/sendgrid.py

@@ -0,0 +1,123 @@
+import json
+from datetime import datetime
+
+from django.utils.timezone import utc
+
+from .base import AnymailBaseWebhookView
+from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+
+
+class SendGridBaseWebhookView(AnymailBaseWebhookView):
+    """Base view class for SendGrid webhooks"""
+
+    def parse_events(self, request):
+        esp_events = json.loads(request.body.decode('utf-8'))
+        return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
+
+    def esp_to_anymail_event(self, esp_event):
+        raise NotImplementedError()
+
+
+class SendGridTrackingWebhookView(SendGridBaseWebhookView):
+    """Handler for SendGrid delivery and engagement tracking webhooks"""
+
+    signal = tracking
+
+    event_types = {
+        # Map SendGrid event: Anymail normalized type
+        'bounce': EventType.BOUNCED,
+        'deferred': EventType.DEFERRED,
+        'delivered': EventType.DELIVERED,
+        'dropped': EventType.REJECTED,
+        'processed': EventType.QUEUED,
+        'click': EventType.CLICKED,
+        'open': EventType.OPENED,
+        'spamreport': EventType.COMPLAINED,
+        'unsubscribe': EventType.UNSUBSCRIBED,
+        'group_unsubscribe': EventType.UNSUBSCRIBED,
+        'group_resubscribe': EventType.SUBSCRIBED,
+    }
+
+    reject_reasons = {
+        # Map SendGrid reason/type strings (lowercased) to Anymail normalized reject_reason
+        'invalid': RejectReason.INVALID,
+        'unsubscribed address': RejectReason.UNSUBSCRIBED,
+        'bounce': RejectReason.BOUNCED,
+        'blocked': RejectReason.BLOCKED,
+        'expired': RejectReason.TIMED_OUT,
+    }
+
+    def esp_to_anymail_event(self, esp_event):
+        event_type = self.event_types.get(esp_event['event'], EventType.UNKNOWN)
+        try:
+            timestamp = datetime.fromtimestamp(esp_event['timestamp'], tz=utc)
+        except (KeyError, ValueError):
+            timestamp = None
+
+        if esp_event['event'] == 'dropped':
+            mta_response = None  # dropped at ESP before even getting to MTA
+            reason = esp_event.get('type', esp_event.get('reason', ''))  # cause could be in 'type' or 'reason'
+            reject_reason = self.reject_reasons.get(reason.lower(), RejectReason.OTHER)
+        else:
+            # MTA response is in 'response' for delivered; 'reason' for bounce
+            mta_response = esp_event.get('response', esp_event.get('reason', None))
+            reject_reason = None
+
+        # SendGrid merges metadata ('unique_args') with the event.
+        # We can (sort of) split metadata back out by filtering known
+        # SendGrid event params, though this can miss metadata keys
+        # that duplicate SendGrid params, and can accidentally include
+        # non-metadata keys if SendGrid modifies their event records.
+        metadata_keys = set(esp_event.keys()) - self.sendgrid_event_keys
+        if len(metadata_keys) > 0:
+            metadata = {key: esp_event[key] for key in metadata_keys}
+        else:
+            metadata = None
+
+        return AnymailTrackingEvent(
+            event_type=event_type,
+            timestamp=timestamp,
+            message_id=esp_event.get('smtp-id', None),
+            event_id=esp_event.get('sg_event_id', None),
+            recipient=esp_event.get('email', None),
+            reject_reason=reject_reason,
+            mta_response=mta_response,
+            tags=esp_event.get('category', None),
+            metadata=metadata,
+            click_url=esp_event.get('url', None),
+            user_agent=esp_event.get('useragent', None),
+            esp_event=esp_event,
+        )
+
+    # Known keys in SendGrid events (used to recover metadata above)
+    sendgrid_event_keys = {
+        'asm_group_id',
+        'attempt',  # MTA deferred count
+        'category',
+        'cert_err',
+        'email',
+        'event',
+        'ip',
+        'marketing_campaign_id',
+        'marketing_campaign_name',
+        'newsletter',  # ???
+        'nlvx_campaign_id',
+        'nlvx_campaign_split_id',
+        'nlvx_user_id',
+        'pool',
+        'post_type',
+        'reason',  # MTA bounce/drop reason; SendGrid suppression reason
+        'response',  # MTA deferred/delivered message
+        'send_at',
+        'sg_event_id',
+        'sg_message_id',
+        'smtp-id',
+        'status',  # SMTP status code
+        'timestamp',
+        'tls',
+        'type',  # suppression reject reason ("bounce", "blocked", "expired")
+        'url',  # click tracking
+        'url_offset',  # click tracking
+        'useragent',  # click/open tracking
+    }
+