SparkPost: initial open and AMP tracking events

* Add SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED boolean
  setting, default False, controlling whether to report
  SparkPost "Initial Open" events as Anymail "opened".
* Add mapping for SparkPost "AMP Click", "AMP Open",
  and "AMP Initial Open" events.
* Update outdated doc references to SparkPost site

Closes #206

CHANGELOG.rst

@@ -25,6 +25,25 @@ Release history
 ^^^^^^^^^^^^^^^
     ..  This extra heading level keeps the ToC from becoming unmanageably long
 
+vNext
+-----
+
+*Unreleased changes*
+
+Features
+~~~~~~~~
+
+* **SparkPost:** Add option for event tracking webhooks to map SparkPost's "Initial Open"
+  event to Anymail's normalized "opened" type. (By default, only SparkPost's "Open" is
+  reported as Anymail "opened", and "Initial Open" maps to "unknown" to avoid duplicates.
+  See `docs <https://anymail.readthedocs.io/en/latest/esps/sparkpost/#sparkpost-webhooks>`__.
+  Thanks to `@slinkymanbyday`_.)
+
+* **SparkPost:** In event tracking webhooks, map AMP open and click events to the
+  corresponding Anymail normalized event types. (Previously these were treated as
+  as "unknown" events.)
+
+
 v8.0
 ----
 
@@ -1170,6 +1189,7 @@ Features
 .. _@RignonNoel: https://github.com/RignonNoel
 .. _@sebashwa: https://github.com/sebashwa
 .. _@sebbacon: https://github.com/sebbacon
+.. _@slinkymanbyday: https://github.com/slinkymanbyday
 .. _@swrobel: https://github.com/swrobel
 .. _@Thorbenl: https://github.com/Thorbenl
 .. _@tcourtqtm: https://github.com/tcourtqtm

anymail/webhooks/sparkpost.py

@@ -8,6 +8,7 @@ from .base import AnymailBaseWebhookView
 from ..exceptions import AnymailConfigurationError
 from ..inbound import AnymailInboundMessage
 from ..signals import inbound, tracking, AnymailInboundEvent, AnymailTrackingEvent, EventType, RejectReason
+from ..utils import get_anymail_setting
 
 
 class SparkPostBaseWebhookView(AnymailBaseWebhookView):
@@ -64,12 +65,21 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
         'delay': EventType.DEFERRED,
         'click': EventType.CLICKED,
         'open': EventType.OPENED,
+        'amp_click': EventType.CLICKED,
+        'amp_open': EventType.OPENED,
         'generation_failure': EventType.FAILED,
         'generation_rejection': EventType.REJECTED,
         'list_unsubscribe': EventType.UNSUBSCRIBED,
         'link_unsubscribe': EventType.UNSUBSCRIBED,
     }
 
+    # Additional event_types mapping when Anymail setting
+    # SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED is enabled.
+    initial_open_event_types = {
+        'initial_open': EventType.OPENED,
+        'amp_initial_open': EventType.OPENED,
+    }
+
     reject_reasons = {
         # Map SparkPost event.bounce_class: Anymail normalized reject reason.
         # Can also supply (RejectReason, EventType) for bounce_class that affects our event_type.
@@ -96,6 +106,19 @@ class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
         '100': (RejectReason.OTHER, EventType.AUTORESPONDED),  # Challenge-Response
     }
 
+    def __init__(self, **kwargs):
+        # Set Anymail setting SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED True
+        # to report *both* "open" and "initial_open" as Anymail "opened" events.
+        # (Otherwise only "open" maps to "opened", matching the behavior of most
+        # other ESPs.) Handling "initial_open" is opt-in, to help avoid duplicate
+        # "opened" events on the same first open.
+        track_initial_open_as_opened = get_anymail_setting(
+            'track_initial_open_as_opened', default=False,
+            esp_name=self.esp_name, kwargs=kwargs)
+        if track_initial_open_as_opened:
+            self.event_types = {**self.event_types, **self.initial_open_event_types}
+        super().__init__(**kwargs)
+
     def esp_to_anymail_event(self, event_class, event, raw_event):
         if event_class == 'relay_message':
             # This is an inbound event

docs/esps/sparkpost.rst

@@ -103,6 +103,17 @@ You must specify the full, versioned API endpoint as shown above (not just the b
 .. _SparkPost API Endpoint: https://developers.sparkpost.com/api/index.html#header-api-endpoints
 
 
+.. setting:: ANYMAIL_SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED
+
+.. rubric:: SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED
+
+.. versionadded:: vNext
+
+Boolean, default ``False``. When using Anymail's tracking webhooks, whether to report
+SparkPost's "Initial Open" event as an Anymail normalized "opened" event.
+(SparkPost's "Open" event is always normalized to Anymail's "opened" event.
+See :ref:`sparkpost-webhooks` below.)
+
 .. _sparkpost-esp-extra:
 
 esp_extra support
@@ -268,33 +279,49 @@ Status tracking webhooks
 ------------------------
 
 If you are using Anymail's normalized :ref:`status tracking <event-tracking>`, set up the
-webhook in your `SparkPost account settings under "Webhooks"`_:
+webhook in your `SparkPost configuration under "Webhooks"`_:
 
 * Target URL: :samp:`https://{yoursite.example.com}/anymail/sparkpost/tracking/`
 * Authentication: choose "Basic Auth." For username and password enter the two halves of the
   *random:random* shared secret you created for your :setting:`ANYMAIL_WEBHOOK_SECRET`
   Django setting. (Anymail doesn't support OAuth webhook auth.)
-* Events: click "Select" and then *clear* the checkbox for "Relay Events" category (which is for
+* Events: you can leave "All events" selected, or choose "Select individual events"
-  inbound email). You can leave all the other categories of events checked, or disable
+  to pick the specific events you're interested in tracking.
-  any you aren't interested in tracking.
 
 SparkPost will report these Anymail :attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s:
 queued, rejected, bounced, deferred, delivered, opened, clicked, complained, unsubscribed,
 subscribed.
 
+By default, Anymail reports SparkPost's "Open"---but *not* its "Initial Open"---event
+as Anymail's normalized "opened" :attr:`~anymail.signals.AnymailTrackingEvent.event_type`.
+This avoids duplicate "opened" events when both SparkPost types are enabled.
+
+.. versionadded:: vNext
+
+    To receive SparkPost "Initial Open" events as Anymail's "opened", set
+    :setting:`"SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED": True <ANYMAIL_SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED>`
+    in your ANYMAIL settings dict. You will probably want to disable SparkPost "Open"
+    events when using this setting.
+
+.. versionchanged:: vNext
+
+    SparkPost's "AMP Click" and "AMP Open" are reported as Anymail's "clicked" and
+    "opened" events. If you enable the SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED setting,
+    "AMP Initial Open" will also map to "opened." (Earlier Anymail releases reported
+    all AMP events as "unknown".)
+
+
 The event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` field will be
 a single, raw `SparkPost event`_. (Although SparkPost calls webhooks with batches of events,
 Anymail will invoke your signal receiver separately for each event in the batch.)
-The esp_event is the raw, `wrapped json event structure`_ as provided by SparkPost:
+The esp_event is the raw, wrapped json event structure as provided by SparkPost:
 `{'msys': {'<event_category>': {...<actual event data>...}}}`.
 
 
-.. _SparkPost account settings under "Webhooks":
+.. _SparkPost configuration under "Webhooks":
-    https://app.sparkpost.com/account/webhooks
+    https://app.sparkpost.com/webhooks
 .. _SparkPost event:
-    https://support.sparkpost.com/customer/portal/articles/1976204-webhook-event-reference
+    https://developers.sparkpost.com/api/webhooks/#header-webhook-event-types
-.. _wrapped json event structure:
-    https://support.sparkpost.com/customer/en/portal/articles/2311698-comparing-webhook-and-message-event-data
 
 
 .. _sparkpost-inbound:

tests/test_sparkpost_webhooks.py

@@ -1,7 +1,7 @@
 import json
 from datetime import datetime
 
-from django.test import tag
+from django.test import override_settings, tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -273,9 +273,49 @@ class SparkPostDeliveryTestCase(WebhookTestCase):
         self.assertEqual(event.event_type, "opened")
         self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36")
 
+    @override_settings(ANYMAIL_SPARKPOST_TRACK_INITIAL_OPEN_AS_OPENED=True)
+    def test_initial_open_event_as_opened(self):
+        # Mapping SparkPost "initial_open" to Anymail normalized "opened" is opt-in via a setting,
+        # for backwards compatibility and to avoid reporting duplicate "opened" events when all
+        # SparkPost event types are enabled.
+        raw_events = [{"msys": {"track_event": {
+            "type": "initial_open",
+            "raw_rcpt_to": "recipient@example.com",
+            "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36",
+        }}}]
+        response = self.client.post('/anymail/sparkpost/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SparkPostTrackingWebhookView,
+                                                      event=ANY, esp_name='SparkPost')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.event_type, "opened")
+        self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36")
+
+    def test_initial_open_event_as_unknown(self):
+        # By default, SparkPost "initial_open" is *not* mapped to Anymail "opened".
+        raw_events = [{"msys": {"track_event": {
+            "type": "initial_open",
+            "raw_rcpt_to": "recipient@example.com",
+            "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36",
+        }}}]
+        response = self.client.post('/anymail/sparkpost/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        kwargs = self.assert_handler_called_once_with(self.tracking_handler, sender=SparkPostTrackingWebhookView,
+                                                      event=ANY, esp_name='SparkPost')
+        event = kwargs['event']
+        self.assertIsInstance(event, AnymailTrackingEvent)
+        self.assertEqual(event.event_type, "unknown")
+        # Here's how to get the raw SparkPost event type:
+        self.assertEqual(event.esp_event["msys"].get("track_event", {}).get("type"), "initial_open")
+        # Note that other Anymail normalized event properties are still available:
+        self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36")
+
     def test_click_event(self):
         raw_events = [{"msys": {"track_event": {
-            "type": "click",
+            "type": "amp_click",
             "raw_rcpt_to": "recipient@example.com",
             "target_link_name": "Example Link Name",
             "target_link_url": "http://example.com",
@@ -292,3 +332,20 @@ class SparkPostDeliveryTestCase(WebhookTestCase):
         self.assertEqual(event.recipient, "recipient@example.com")
         self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36")
         self.assertEqual(event.click_url, "http://example.com")
+
+    def test_amp_events(self):
+        raw_events = [{"msys": {"track_event": {
+            "type": "amp_open",
+        }}}, {"msys": {"track_event": {
+            "type": "amp_initial_open",
+        }}}, {"msys": {"track_event": {
+            "type": "amp_click",
+        }}}]
+        response = self.client.post('/anymail/sparkpost/tracking/',
+                                    content_type='application/json', data=json.dumps(raw_events))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(self.tracking_handler.call_count, 3)
+        events = [kwargs["event"] for (args, kwargs) in self.tracking_handler.call_args_list]
+        self.assertEqual(events[0].event_type, "opened")
+        self.assertEqual(events[1].event_type, "unknown")  # amp_initial_open is mapped to "unknown" by default
+        self.assertEqual(events[2].event_type, "clicked")