Test without optional packages

Tox:
* Add Tox factor for extras (all, none, individual ESP).
  For now, only break out ESPs that have specific extra
  dependencies (amazon_ses, sparkpost).
* Install most package dependencies (including extras)
  through the package itself.
* Use new runtests.py environment vars to limit test tags
  when Tox isn't installing all extras.

Travis:
* Rework matrix to request specific TOXENVs directly;
  drop tox-travis.

Test runner (runtests.py):
* Centralize RUN_LIVE_TESTS logic in runtests.py
* Add ANYMAIL_ONLY_TEST and ANYMAIL_SKIP_TESTS env vars
  (comma-separated lists of tags)

Test implementations:
* Tag all ESP-specific tests with ESP
* Tag live tests with "live"
* Don't import ESP-specific packages at test module level. 
  (Test discovery imports test modules before tag-based filtering.)

Closes #104

.travis.yml

@@ -10,9 +10,15 @@ branches:
     - master
     - /^v\d+\.\d+(\.\d+)?(-\S*)?$/
 
+env:
+  global:
+    # Let Travis report failures that tox.ini would normally ignore:
+    - TOX_FORCE_IGNORE_OUTCOME=false
+
 matrix:
   include:
-    - { env: LINT_AND_DOCS=true, python: 3.6 }
+    - python: 3.6
+      env: TOXENV="lint,docs"
 
     # Anymail supports the same Python versions as Django, plus PyPy.
     # https://docs.djangoproject.com/en/dev/faq/install/#what-python-version-can-i-use-with-django
@@ -21,36 +27,41 @@ matrix:
     # combinations, to avoid rapidly consuming the testing accounts' entire send allotments.
 
     # Django 1.11: Python 2.7, 3.4, 3.5, or 3.6
-    - { env: DJANGO=1.11 RUN_LIVE_TESTS=true, python: 2.7 }
-    - { env: DJANGO=1.11, python: 3.4 }
-    - { env: DJANGO=1.11, python: 3.5 }
-    - { env: DJANGO=1.11, python: 3.6 }
-    - { env: DJANGO=1.11, python: pypy2.7-6.0 }
+    - { env: TOXENV=django111-py27-all RUN_LIVE_TESTS=true, python: 2.7 }
+    - { env: TOXENV=django111-py34-all, python: 3.4 }
+    - { env: TOXENV=django111-py35-all, python: 3.5 }
+    - { env: TOXENV=django111-py36-all, python: 3.6 }
+    - { env: TOXENV=django111-pypy-all, python: pypy2.7-6.0 }
     # Django 2.0: Python 3.5+
-    - { env: DJANGO=2.0, python: 3.5 }
-    - { env: DJANGO=2.0, python: 3.6 }
-    - { env: DJANGO=2.0, python: pypy3.5-6.0 }
+    - { env: TOXENV=django20-py35-all, python: 3.5 }
+    - { env: TOXENV=django20-py36-all, python: 3.6 }
+    - { env: TOXENV=django20-pypy3-all, python: pypy3.5-6.0 }
     # Django 2.1: Python 3.5, 3.6, or 3.7
-    - { env: DJANGO=2.1, python: 3.5 }
-    - { env: DJANGO=2.1 RUN_LIVE_TESTS=true, python: 3.6 }
-    - { env: DJANGO=2.1, python: 3.7 }
-    - { env: DJANGO=2.1, python: pypy3.5-6.0 }
+    - { env: TOXENV=django21-py35-all, python: 3.5 }
+    - { env: TOXENV=django21-py36-all RUN_LIVE_TESTS=true, python: 3.6 }
+    - { env: TOXENV=django21-py37-all, python: 3.7 }
+    - { env: TOXENV=django21-pypy3-all, python: pypy3.5-6.0 }
     # Django 2.2: Python 3.5, 3.6, or 3.7
-    - { env: DJANGO=2.2, python: 3.5 }
-    - { env: DJANGO=2.2, python: 3.6 }
-    - { env: DJANGO=2.2, python: 3.7 }
-    - { env: DJANGO=2.2, python: pypy3.5-6.0 }
+    - { env: TOXENV=django22-py35-all, python: 3.5 }
+    - { env: TOXENV=django22-py36-all, python: 3.6 }
+    - { env: TOXENV=django22-py37-all, python: 3.7 }
+    - { env: TOXENV=django22-pypy3-all, python: pypy3.5-6.0 }
     # Django development master (direct from GitHub source):
-    - { env: DJANGO=master, python: 3.6 }
+    - { env: TOXENV=djangoMaster-py36-all, python: 3.6 }
+    # Install without optional extras (don't need to cover entire matrix)
+    - { env: TOXENV=django21-py37-none, python: 3.7 }
+    - { env: TOXENV=django21-py37-amazon_ses, python: 3.7 }
+    - { env: TOXENV=django21-py37-sparkpost, python: 3.7 }
 
   allow_failures:
-    - env: DJANGO=master
+    - env: TOXENV=djangoMaster-py36-all
       python: 3.6
 
 cache: pip
 
 install:
-  - pip install tox-travis
+  # pin tox to avoid https://github.com/tox-dev/tox/issues/1160
+  - pip install tox~=3.6.1
 
 script:
   - tox

Pipfile

@@ -14,12 +14,12 @@ six = "*"
 sparkpost = "*"
 
 [dev-packages]
-detox = "*"
+detox = "==0.18"
 flake8 = "*"
 mock = "*"
 sphinx = "*"
 sphinx-rtd-theme = "*"
-tox = "*"
+tox = "~=3.6.1"
 twine = "*"
 
 [requires]

docs/contributing.rst

@@ -71,7 +71,7 @@ and Python versions. Tests are run at least once a week, to check whether ESP AP
 and other dependencies have changed out from under Anymail.
 
 For local development, the recommended test command is
-:shell:`tox -e django20-py36,django18-py27,lint`, which tests a representative
+:shell:`tox -e django21-py36-all,django111-py27-all,lint`, which tests a representative
 combination of Python and Django versions. It also runs :pypi:`flake8` and other
 code-style checkers. Some other test options are covered below, but using this
 tox command catches most problems, and is a good pre-pull-request check.
@@ -104,10 +104,10 @@ with those, `pyenv`_ is a helpful way to install and manage multiple Python vers
     .. code-block:: console
 
         $ pip install tox  # (if you haven't already)
-        $ tox -e django20-py36,django18-py27,lint  # test recommended environments
+        $ tox -e django21-py36-all,django111-py27-all,lint  # test recommended environments
 
         ## you can also run just some test cases, e.g.:
-        $ tox -e django20-py36,django18-py27 tests.test_mailgun_backend tests.test_utils
+        $ tox -e django21-py36-all,django111-py27-all tests.test_mailgun_backend tests.test_utils
 
         ## to test more Python/Django versions:
         $ tox  # ALL 20+ envs! (grab a coffee, or use `detox` to run tests in parallel)
@@ -121,7 +121,7 @@ API keys or other settings. For example:
 
         $ export MAILGUN_TEST_API_KEY='your-Mailgun-API-key'
         $ export MAILGUN_TEST_DOMAIN='mail.example.com'  # sending domain for that API key
-        $ tox -e django20-py36 tests.test_mailgun_integration
+        $ tox -e django21-py36-all tests.test_mailgun_integration
 
 Check the ``*_integration_tests.py`` files in the `tests source`_ to see which variables
 are required for each ESP. Depending on the supported features, the integration tests for

runtests.py

@@ -4,7 +4,9 @@
 #   or
 # runtests.py [tests.test_x tests.test_y.SomeTestCase ...]
 
+from __future__ import print_function
 import sys
+from distutils.util import strtobool
 
 import django
 import os
@@ -17,6 +19,18 @@ def setup_and_run_tests(test_labels=None):
     """Discover and run project tests. Returns number of failures."""
     test_labels = test_labels or ['tests']
 
+    tags = envlist('ANYMAIL_ONLY_TEST')
+    exclude_tags = envlist('ANYMAIL_SKIP_TESTS')
+
+    # In automated testing, don't run live tests unless specifically requested
+    if envbool('CONTINUOUS_INTEGRATION') and not envbool('RUN_LIVE_TESTS'):
+        exclude_tags.append('live')
+
+    if tags:
+        print("Only running tests tagged: %r" % tags)
+    if exclude_tags:
+        print("Excluding tests tagged: %r" % exclude_tags)
+
     warnings.simplefilter('default')  # show DeprecationWarning and other default-ignored warnings
 
     # noinspection PyStringFormat
@@ -25,7 +39,7 @@ def setup_and_run_tests(test_labels=None):
     django.setup()
 
     TestRunner = get_runner(settings)
-    test_runner = TestRunner(verbosity=1)
+    test_runner = TestRunner(verbosity=1, tags=tags, exclude_tags=exclude_tags)
     return test_runner.run_tests(test_labels)
 
 
@@ -36,5 +50,30 @@ def runtests(test_labels=None):
     sys.exit(bool(failures))
 
 
+def envbool(var, default=False):
+    """Returns value of environment variable var as a bool, or default if not set.
+
+    Converts `'true'` to `True`, and `'false'` to `False`.
+    See :func:`~distutils.util.strtobool` for full list of allowable values.
+    """
+    val = os.getenv(var, None)
+    if val is None:
+        return default
+    else:
+        return strtobool(val)
+
+
+def envlist(var):
+    """Returns value of environment variable var split in a comma-separated list.
+
+    Returns an empty list if variable is empty or not set.
+    """
+    val = os.getenv(var, "").split(',')
+    if val == ['']:
+        # "Splitting an empty string with a specified separator returns ['']"
+        val = []
+    return val
+
+
 if __name__ == '__main__':
     runtests(test_labels=sys.argv[1:])

tests/test_amazon_ses_backend.py

@@ -5,13 +5,10 @@ import json
 from datetime import datetime
 from email.mime.application import MIMEApplication
 
-import botocore.config
-import botocore.exceptions
 import six
 from django.core import mail
 from django.core.mail import BadHeaderError
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 from mock import ANY, patch
 
 from anymail.exceptions import AnymailAPIError, AnymailUnsupportedFeature
@@ -20,6 +17,7 @@ from anymail.message import attach_inline_image_file, AnymailMessage
 from .utils import AnymailTestMixin, SAMPLE_IMAGE_FILENAME, sample_image_content, sample_image_path
 
 
+@tag('amazon_ses')
 @override_settings(EMAIL_BACKEND='anymail.backends.amazon_ses.EmailBackend')
 class AmazonSESBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin):
     """TestCase that uses the Amazon SES EmailBackend with a mocked boto3 client"""
@@ -61,8 +59,9 @@ class AmazonSESBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin):
         return mock_operation.return_value
 
     def set_mock_failure(self, response, operation_name="send_raw_email"):
+        from botocore.exceptions import ClientError
         mock_operation = getattr(self.mock_client_instance, operation_name)
-        mock_operation.side_effect = botocore.exceptions.ClientError(response, operation_name=operation_name)
+        mock_operation.side_effect = ClientError(response, operation_name=operation_name)
 
     def get_session_params(self):
         if self.mock_session.call_args is None:
@@ -111,6 +110,7 @@ class AmazonSESBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin):
             raise AssertionError(msg or "ESP API was called and shouldn't have been")
 
 
+@tag('amazon_ses')
 class AmazonSESBackendStandardEmailTests(AmazonSESBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -318,6 +318,7 @@ class AmazonSESBackendStandardEmailTests(AmazonSESBackendMockAPITestCase):
                               headers={"X-Header": "custom header value\r\ninjected"}).send()
 
 
+@tag('amazon_ses')
 class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -589,6 +590,7 @@ class AmazonSESBackendAnymailFeatureTests(AmazonSESBackendMockAPITestCase):
         self.assertEqual(self.message.anymail_status.esp_response, response_content)
 
 
+@tag('amazon_ses')
 class AmazonSESBackendConfigurationTests(AmazonSESBackendMockAPITestCase):
     """Test configuration options"""
 
@@ -635,7 +637,8 @@ class AmazonSESBackendConfigurationTests(AmazonSESBackendMockAPITestCase):
 
     def test_client_params_in_connection_init(self):
         """You can also supply credentials specifically for a particular EmailBackend connection instance"""
-        boto_config = botocore.config.Config(connect_timeout=30)
+        from botocore.config import Config
+        boto_config = Config(connect_timeout=30)
         conn = mail.get_connection(
             'anymail.backends.amazon_ses.EmailBackend',
             client_params={"aws_session_token": "test-session-token", "config": boto_config})

tests/test_amazon_ses_inbound.py

@@ -5,7 +5,7 @@ from base64 import b64encode
 from datetime import datetime
 from textwrap import dedent
 
-import botocore.exceptions
+from django.test import tag
 from django.utils.timezone import utc
 from mock import ANY, patch
 
@@ -18,6 +18,7 @@ from .test_amazon_ses_webhooks import AmazonSESWebhookTestsMixin
 from .webhook_cases import WebhookTestCase
 
 
+@tag('amazon_ses')
 class AmazonSESInboundTests(WebhookTestCase, AmazonSESWebhookTestsMixin):
 
     def setUp(self):
@@ -270,7 +271,8 @@ class AmazonSESInboundTests(WebhookTestCase, AmazonSESWebhookTestsMixin):
     def test_inbound_s3_failure_message(self):
         """Issue a helpful error when S3 download fails"""
         # Boto's error: "An error occurred (403) when calling the HeadObject operation: Forbidden")
-        self.mock_s3.download_fileobj.side_effect = botocore.exceptions.ClientError(
+        from botocore.exceptions import ClientError
+        self.mock_s3.download_fileobj.side_effect = ClientError(
             {'Error': {'Code': 403, 'Message': 'Forbidden'}}, operation_name='HeadObject')
 
         raw_ses_event = {
@@ -290,7 +292,7 @@ class AmazonSESInboundTests(WebhookTestCase, AmazonSESWebhookTestsMixin):
             "Anymail AmazonSESInboundWebhookView couldn't download S3 object 'YourBucket:inbound/the_object_key'"
         ) as cm:
             self.post_from_sns('/anymail/amazon_ses/inbound/', raw_sns_message)
-        self.assertIsInstance(cm.exception, botocore.exceptions.ClientError)  # both Boto and Anymail exception class
+        self.assertIsInstance(cm.exception, ClientError)  # both Boto and Anymail exception class
         self.assertIn("ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden",
                       str(cm.exception))  # original Boto message included
 

tests/test_amazon_ses_integration.py

@@ -5,13 +5,12 @@ import os
 import unittest
 import warnings
 
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 try:
     ResourceWarning
@@ -24,7 +23,6 @@ AMAZON_SES_TEST_SECRET_ACCESS_KEY = os.getenv("AMAZON_SES_TEST_SECRET_ACCESS_KEY
 AMAZON_SES_TEST_REGION_NAME = os.getenv("AMAZON_SES_TEST_REGION_NAME", "us-east-1")
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @unittest.skipUnless(AMAZON_SES_TEST_ACCESS_KEY_ID and AMAZON_SES_TEST_SECRET_ACCESS_KEY,
                      "Set AMAZON_SES_TEST_ACCESS_KEY_ID and AMAZON_SES_TEST_SECRET_ACCESS_KEY "
                      "environment variables to run Amazon SES integration tests")
@@ -43,6 +41,7 @@ AMAZON_SES_TEST_REGION_NAME = os.getenv("AMAZON_SES_TEST_REGION_NAME", "us-east-
         },
         "AMAZON_SES_CONFIGURATION_SET_NAME": "TestConfigurationSet",  # actual config set in Anymail test account
     })
+@tag('amazon_ses', 'live')
 class AmazonSESBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):
     """Amazon SES API integration tests
 

tests/test_amazon_ses_webhooks.py

@@ -2,8 +2,7 @@ import json
 import warnings
 from datetime import datetime
 
-import botocore.exceptions
-from django.test import override_settings
+from django.test import override_settings, tag
 from django.utils.timezone import utc
 from mock import ANY, patch
 
@@ -27,6 +26,7 @@ class AmazonSESWebhookTestsMixin(object):
             **kwargs)
 
 
+@tag('amazon_ses')
 class AmazonSESWebhookSecurityTests(WebhookTestCase, AmazonSESWebhookTestsMixin, WebhookBasicAuthTestsMixin):
     def call_webhook(self):
         return self.post_from_sns('/anymail/amazon_ses/tracking/',
@@ -43,6 +43,7 @@ class AmazonSESWebhookSecurityTests(WebhookTestCase, AmazonSESWebhookTestsMixin,
         self.assertEqual(response["WWW-Authenticate"], 'Basic realm="Anymail WEBHOOK_SECRET"')
 
 
+@tag('amazon_ses')
 class AmazonSESNotificationsTests(WebhookTestCase, AmazonSESWebhookTestsMixin):
     def test_bounce_event(self):
         # This test includes a complete Amazon SES example event. (Later tests omit some payload for brevity.)
@@ -404,6 +405,7 @@ class AmazonSESNotificationsTests(WebhookTestCase, AmazonSESWebhookTestsMixin):
             self.post_from_sns('/anymail/amazon_ses/tracking/', raw_sns_message)
 
 
+@tag('amazon_ses')
 class AmazonSESSubscriptionManagementTests(WebhookTestCase, AmazonSESWebhookTestsMixin):
     # Anymail will automatically respond to SNS subscription notifications
     # if Anymail is configured to require basic auth via WEBHOOK_SECRET.
@@ -450,7 +452,8 @@ class AmazonSESSubscriptionManagementTests(WebhookTestCase, AmazonSESWebhookTest
 
     def test_sns_subscription_confirmation_failure(self):
         """Auto-confirmation allows error through if confirm call fails"""
-        self.mock_client_instance.confirm_subscription.side_effect = botocore.exceptions.ClientError({
+        from botocore.exceptions import ClientError
+        self.mock_client_instance.confirm_subscription.side_effect = ClientError({
             'Error': {
                 'Type': 'Sender',
                 'Code': 'InternalError',
@@ -461,7 +464,7 @@ class AmazonSESSubscriptionManagementTests(WebhookTestCase, AmazonSESWebhookTest
                 'HTTPStatusCode': 500,
             }
         }, operation_name="confirm_subscription")
-        with self.assertRaisesMessage(botocore.exceptions.ClientError, "Gremlins!"):
+        with self.assertRaisesMessage(ClientError, "Gremlins!"):
             self.post_from_sns('/anymail/amazon_ses/tracking/', self.SNS_SUBSCRIPTION_CONFIRMATION)
         # didn't notify receivers:
         self.assertEqual(self.tracking_handler.call_count, 0)

tests/test_mailgun_backend.py

@@ -16,8 +16,7 @@ from email.mime.image import MIMEImage
 
 from django.core import mail
 from django.core.exceptions import ImproperlyConfigured
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 from django.utils.timezone import get_fixed_timezone, override as override_current_timezone
 
 from anymail.exceptions import (
@@ -30,6 +29,7 @@ from .utils import (AnymailTestMixin, sample_email_content,
                     sample_image_content, sample_image_path, SAMPLE_IMAGE_FILENAME)
 
 
+@tag('mailgun')
 @override_settings(EMAIL_BACKEND='anymail.backends.mailgun.EmailBackend',
                    ANYMAIL={'MAILGUN_API_KEY': 'test_api_key'})
 class MailgunBackendMockAPITestCase(RequestsBackendMockAPITestCase):
@@ -44,6 +44,7 @@ class MailgunBackendMockAPITestCase(RequestsBackendMockAPITestCase):
         self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', 'from@example.com', ['to@example.com'])
 
 
+@tag('mailgun')
 class MailgunBackendStandardEmailTests(MailgunBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -374,6 +375,7 @@ class MailgunBackendStandardEmailTests(MailgunBackendMockAPITestCase):
         self.assertEqual(sent, 0)
 
 
+@tag('mailgun')
 class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -563,6 +565,7 @@ class MailgunBackendAnymailFeatureTests(MailgunBackendMockAPITestCase):
     # (Anything that requests can serialize as a form field will work with Mailgun)
 
 
+@tag('mailgun')
 class MailgunBackendRecipientsRefusedTests(MailgunBackendMockAPITestCase):
     """Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid"""
 
@@ -594,11 +597,13 @@ class MailgunBackendRecipientsRefusedTests(MailgunBackendMockAPITestCase):
         self.assertEqual(sent, 0)
 
 
+@tag('mailgun')
 class MailgunBackendSessionSharingTestCase(SessionSharingTestCasesMixin, MailgunBackendMockAPITestCase):
     """Requests session sharing tests"""
     pass  # tests are defined in the mixin
 
 
+@tag('mailgun')
 @override_settings(EMAIL_BACKEND="anymail.backends.mailgun.EmailBackend")
 class MailgunBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin):
     """Test ESP backend without required settings in place"""

tests/test_mailgun_inbound.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from textwrap import dedent
 
 import six
-from django.test import override_settings
+from django.test import override_settings, tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -19,6 +19,7 @@ from .utils import sample_image_content, sample_email_content
 from .webhook_cases import WebhookTestCase
 
 
+@tag('mailgun')
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
 class MailgunInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):

tests/test_mailgun_integration.py

@@ -9,19 +9,18 @@ from datetime import datetime, timedelta
 from time import mktime, sleep
 
 import requests
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 MAILGUN_TEST_API_KEY = os.getenv('MAILGUN_TEST_API_KEY')
 MAILGUN_TEST_DOMAIN = os.getenv('MAILGUN_TEST_DOMAIN')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('mailgun', 'live')
 @unittest.skipUnless(MAILGUN_TEST_API_KEY and MAILGUN_TEST_DOMAIN,
                      "Set MAILGUN_TEST_API_KEY and MAILGUN_TEST_DOMAIN environment variables "
                      "to run Mailgun integration tests")

tests/test_mailgun_webhooks.py

@@ -4,7 +4,7 @@ from datetime import datetime
 import hashlib
 import hmac
 from django.core.exceptions import ImproperlyConfigured
-from django.test import override_settings
+from django.test import override_settings, tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -59,6 +59,7 @@ def querydict_to_postdict(qd):
     }
 
 
+@tag('mailgun')
 class MailgunWebhookSettingsTestCase(WebhookTestCase):
     def test_requires_api_key(self):
         with self.assertRaises(ImproperlyConfigured):
@@ -66,6 +67,7 @@ class MailgunWebhookSettingsTestCase(WebhookTestCase):
                              data=json.dumps(mailgun_sign_payload({'event-data': {'event': 'delivered'}})))
 
 
+@tag('mailgun')
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
 class MailgunWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     should_warn_if_no_auth = False  # because we check webhook signature
@@ -94,6 +96,7 @@ class MailgunWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin
         self.assertEqual(response.status_code, 400)
 
 
+@tag('mailgun')
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
 class MailgunTestCase(WebhookTestCase):
     # Tests for Mailgun's new webhooks (announced 2018-06-29)
@@ -445,6 +448,7 @@ class MailgunTestCase(WebhookTestCase):
         self.assertEqual(event.click_url, "https://example.com/test")
 
 
+@tag('mailgun')
 @override_settings(ANYMAIL_MAILGUN_API_KEY=TEST_API_KEY)
 class MailgunLegacyTestCase(WebhookTestCase):
     # Tests for Mailgun's "legacy" webhooks

tests/test_mailjet_backend.py

@@ -7,8 +7,7 @@ from email.mime.image import MIMEImage
 
 from django.core import mail
 from django.core.exceptions import ImproperlyConfigured
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import (AnymailAPIError, AnymailSerializationError,
                                 AnymailUnsupportedFeature,
@@ -19,6 +18,7 @@ from .mock_requests_backend import RequestsBackendMockAPITestCase, SessionSharin
 from .utils import sample_image_content, sample_image_path, SAMPLE_IMAGE_FILENAME, AnymailTestMixin, decode_att
 
 
+@tag('mailjet')
 @override_settings(EMAIL_BACKEND='anymail.backends.mailjet.EmailBackend',
                    ANYMAIL={
                        'MAILJET_API_KEY': '',
@@ -64,6 +64,7 @@ class MailjetBackendMockAPITestCase(RequestsBackendMockAPITestCase):
         ])
 
 
+@tag('mailjet')
 class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -354,6 +355,7 @@ class MailjetBackendStandardEmailTests(MailjetBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('mailjet')
 class MailjetBackendAnymailFeatureTests(MailjetBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -623,11 +625,13 @@ class MailjetBackendAnymailFeatureTests(MailjetBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('mailjet')
 class MailjetBackendSessionSharingTestCase(SessionSharingTestCasesMixin, MailjetBackendMockAPITestCase):
     """Requests session sharing tests"""
     pass  # tests are defined in the mixin
 
 
+@tag('mailjet')
 @override_settings(EMAIL_BACKEND="anymail.backends.mailjet.EmailBackend")
 class MailjetBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin):
     """Test ESP backend without required settings in place"""

tests/test_mailjet_inbound.py

@@ -1,6 +1,7 @@
 import json
 from base64 import b64encode
 
+from django.test import tag
 from mock import ANY
 
 from anymail.inbound import AnymailInboundMessage
@@ -11,6 +12,7 @@ from .utils import sample_image_content, sample_email_content
 from .webhook_cases import WebhookTestCase
 
 
+@tag('mailjet')
 class MailjetInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):
         raw_event = {

tests/test_mailjet_integration.py

@@ -1,19 +1,18 @@
 import os
 import unittest
 
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 MAILJET_TEST_API_KEY = os.getenv('MAILJET_TEST_API_KEY')
 MAILJET_TEST_SECRET_KEY = os.getenv('MAILJET_TEST_SECRET_KEY')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('mailjet', 'live')
 @unittest.skipUnless(MAILJET_TEST_API_KEY and MAILJET_TEST_SECRET_KEY,
                      "Set MAILJET_TEST_API_KEY and MAILJET_TEST_SECRET_KEY "
                      "environment variables to run Mailjet integration tests")

tests/test_mailjet_webhooks.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from django.test import tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -9,6 +10,7 @@ from anymail.webhooks.mailjet import MailjetTrackingWebhookView
 from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
 
 
+@tag('mailjet')
 class MailjetWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     def call_webhook(self):
         return self.client.post('/anymail/mailjet/tracking/',
@@ -17,6 +19,7 @@ class MailjetWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin
     # Actual tests are in WebhookBasicAuthTestsMixin
 
 
+@tag('mailjet')
 class MailjetDeliveryTestCase(WebhookTestCase):
 
     def test_sent_event(self):

tests/test_mandrill_backend.py

@@ -7,8 +7,7 @@ from email.mime.image import MIMEImage
 
 from django.core import mail
 from django.core.exceptions import ImproperlyConfigured
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 from django.utils.timezone import get_fixed_timezone, override as override_current_timezone
 
 from anymail.exceptions import (AnymailAPIError, AnymailRecipientsRefused,
@@ -19,6 +18,7 @@ from .mock_requests_backend import RequestsBackendMockAPITestCase, SessionSharin
 from .utils import sample_image_content, sample_image_path, SAMPLE_IMAGE_FILENAME, AnymailTestMixin, decode_att
 
 
+@tag('mandrill')
 @override_settings(EMAIL_BACKEND='anymail.backends.mandrill.EmailBackend',
                    ANYMAIL={'MANDRILL_API_KEY': 'test_api_key'})
 class MandrillBackendMockAPITestCase(RequestsBackendMockAPITestCase):
@@ -35,6 +35,7 @@ class MandrillBackendMockAPITestCase(RequestsBackendMockAPITestCase):
         self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', 'from@example.com', ['to@example.com'])
 
 
+@tag('mandrill')
 class MandrillBackendStandardEmailTests(MandrillBackendMockAPITestCase):
     """Test backend support for Django mail wrappers"""
 
@@ -267,6 +268,7 @@ class MandrillBackendStandardEmailTests(MandrillBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('mandrill')
 class MandrillBackendAnymailFeatureTests(MandrillBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -534,6 +536,7 @@ class MandrillBackendAnymailFeatureTests(MandrillBackendMockAPITestCase):
         self.assertRegex(str(err), r"Decimal.*is not JSON serializable")  # original message
 
 
+@tag('mandrill')
 class MandrillBackendRecipientsRefusedTests(MandrillBackendMockAPITestCase):
     """Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid"""
 
@@ -588,11 +591,13 @@ class MandrillBackendRecipientsRefusedTests(MandrillBackendMockAPITestCase):
         self.assertEqual(sent, 1)  # refused message is included in sent count
 
 
+@tag('mandrill')
 class MandrillBackendSessionSharingTestCase(SessionSharingTestCasesMixin, MandrillBackendMockAPITestCase):
     """Requests session sharing tests"""
     pass  # tests are defined in the mixin
 
 
+@tag('mandrill')
 @override_settings(EMAIL_BACKEND="anymail.backends.mandrill.EmailBackend")
 class MandrillBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin):
     """Test backend without required settings"""

tests/test_mandrill_djrill_features.py

@@ -1,12 +1,13 @@
 from datetime import date
 from django.core import mail
-from django.test import override_settings
+from django.test import override_settings, tag
 
 from anymail.exceptions import AnymailSerializationError
 
 from .test_mandrill_backend import MandrillBackendMockAPITestCase
 
 
+@tag('mandrill')
 class MandrillBackendDjrillFeatureTests(MandrillBackendMockAPITestCase):
     """Test backend support for deprecated features leftover from Djrill"""
 

tests/test_mandrill_inbound.py

@@ -1,6 +1,6 @@
 from textwrap import dedent
 
-from django.test import override_settings
+from django.test import override_settings, tag
 from mock import ANY
 
 from anymail.inbound import AnymailInboundMessage
@@ -11,6 +11,7 @@ from .test_mandrill_webhooks import TEST_WEBHOOK_KEY, mandrill_args
 from .webhook_cases import WebhookTestCase
 
 
+@tag('mandrill')
 @override_settings(ANYMAIL_MANDRILL_WEBHOOK_KEY=TEST_WEBHOOK_KEY)
 class MandrillInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):

tests/test_mandrill_integration.py

@@ -2,18 +2,17 @@ import os
 import unittest
 
 from django.core import mail
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError, AnymailRecipientsRefused
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 MANDRILL_TEST_API_KEY = os.getenv('MANDRILL_TEST_API_KEY')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('mandrill', 'live')
 @unittest.skipUnless(MANDRILL_TEST_API_KEY,
                      "Set MANDRILL_TEST_API_KEY environment variable to run integration tests")
 @override_settings(MANDRILL_API_KEY=MANDRILL_TEST_API_KEY,

tests/test_mandrill_webhooks.py

@@ -6,7 +6,7 @@ import hashlib
 import hmac
 from base64 import b64encode
 from django.core.exceptions import ImproperlyConfigured
-from django.test import override_settings
+from django.test import override_settings, tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -48,6 +48,7 @@ def mandrill_args(events=None,
     }
 
 
+@tag('mandrill')
 class MandrillWebhookSettingsTestCase(WebhookTestCase):
     def test_requires_webhook_key(self):
         with self.assertRaisesRegex(ImproperlyConfigured, r'MANDRILL_WEBHOOK_KEY'):
@@ -62,6 +63,7 @@ class MandrillWebhookSettingsTestCase(WebhookTestCase):
         self.assertEqual(response.status_code, 200)
 
 
+@tag('mandrill')
 @override_settings(ANYMAIL_MANDRILL_WEBHOOK_KEY=TEST_WEBHOOK_KEY)
 class MandrillWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     should_warn_if_no_auth = False  # because we check webhook signature
@@ -127,6 +129,7 @@ class MandrillWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixi
         self.assertEqual(response.status_code, 400)
 
 
+@tag('mandrill')
 @override_settings(ANYMAIL_MANDRILL_WEBHOOK_KEY=TEST_WEBHOOK_KEY)
 class MandrillTrackingTestCase(WebhookTestCase):
 

tests/test_postmark_backend.py

@@ -7,8 +7,7 @@ from email.mime.image import MIMEImage
 
 from django.core import mail
 from django.core.exceptions import ImproperlyConfigured
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import (
     AnymailAPIError, AnymailSerializationError,
@@ -19,6 +18,7 @@ from .mock_requests_backend import RequestsBackendMockAPITestCase, SessionSharin
 from .utils import sample_image_content, sample_image_path, SAMPLE_IMAGE_FILENAME, AnymailTestMixin, decode_att
 
 
+@tag('postmark')
 @override_settings(EMAIL_BACKEND='anymail.backends.postmark.EmailBackend',
                    ANYMAIL={'POSTMARK_SERVER_TOKEN': 'test_server_token'})
 class PostmarkBackendMockAPITestCase(RequestsBackendMockAPITestCase):
@@ -36,6 +36,7 @@ class PostmarkBackendMockAPITestCase(RequestsBackendMockAPITestCase):
         self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', 'from@example.com', ['to@example.com'])
 
 
+@tag('postmark')
 class PostmarkBackendStandardEmailTests(PostmarkBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -318,6 +319,7 @@ class PostmarkBackendStandardEmailTests(PostmarkBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('postmark')
 class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -605,6 +607,7 @@ class PostmarkBackendAnymailFeatureTests(PostmarkBackendMockAPITestCase):
         self.assertRegex(str(err), r"Decimal.*is not JSON serializable")  # original message
 
 
+@tag('postmark')
 class PostmarkBackendRecipientsRefusedTests(PostmarkBackendMockAPITestCase):
     """Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid"""
 
@@ -699,11 +702,13 @@ class PostmarkBackendRecipientsRefusedTests(PostmarkBackendMockAPITestCase):
         self.assertEqual(status.recipients['spam@example.com'].status, 'rejected')
 
 
+@tag('postmark')
 class PostmarkBackendSessionSharingTestCase(SessionSharingTestCasesMixin, PostmarkBackendMockAPITestCase):
     """Requests session sharing tests"""
     pass  # tests are defined in the mixin
 
 
+@tag('postmark')
 @override_settings(EMAIL_BACKEND="anymail.backends.postmark.EmailBackend")
 class PostmarkBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin):
     """Test ESP backend without required settings in place"""

tests/test_postmark_inbound.py

@@ -1,6 +1,7 @@
 import json
 from base64 import b64encode
 
+from django.test import tag
 from mock import ANY
 
 from anymail.exceptions import AnymailConfigurationError
@@ -12,6 +13,7 @@ from .utils import sample_image_content, sample_email_content
 from .webhook_cases import WebhookTestCase
 
 
+@tag('postmark')
 class PostmarkInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):
         raw_event = {

tests/test_postmark_integration.py

@@ -1,13 +1,12 @@
 import os
 import unittest
 
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 
 # For most integration tests, Postmark's sandboxed "POSTMARK_API_TEST" token is used.
@@ -16,7 +15,7 @@ POSTMARK_TEST_SERVER_TOKEN = os.getenv('POSTMARK_TEST_SERVER_TOKEN')
 POSTMARK_TEST_TEMPLATE_ID = os.getenv('POSTMARK_TEST_TEMPLATE_ID')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('postmark', 'live')
 @override_settings(ANYMAIL_POSTMARK_SERVER_TOKEN="POSTMARK_API_TEST",
                    EMAIL_BACKEND="anymail.backends.postmark.EmailBackend")
 class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):

tests/test_postmark_webhooks.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from django.test import tag
 from django.utils.timezone import get_fixed_timezone, utc
 from mock import ANY
 
@@ -10,6 +11,7 @@ from anymail.webhooks.postmark import PostmarkTrackingWebhookView
 from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
 
 
+@tag('postmark')
 class PostmarkWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     def call_webhook(self):
         return self.client.post('/anymail/postmark/tracking/',
@@ -18,6 +20,7 @@ class PostmarkWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixi
     # Actual tests are in WebhookBasicAuthTestsMixin
 
 
+@tag('postmark')
 class PostmarkDeliveryTestCase(WebhookTestCase):
     def test_bounce_event(self):
         raw_event = {

tests/test_sendgrid_backend.py

@@ -9,8 +9,7 @@ from email.mime.image import MIMEImage
 
 import six
 from django.core import mail
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 from django.utils.timezone import get_fixed_timezone, override as override_current_timezone
 
 from anymail.exceptions import (AnymailAPIError, AnymailConfigurationError, AnymailSerializationError,
@@ -24,6 +23,7 @@ from .utils import sample_image_content, sample_image_path, SAMPLE_IMAGE_FILENAM
 longtype = int if six.PY3 else long  # NOQA: F821
 
 
+@tag('sendgrid')
 @override_settings(EMAIL_BACKEND='anymail.backends.sendgrid.EmailBackend',
                    ANYMAIL={'SENDGRID_API_KEY': 'test_api_key'})
 class SendGridBackendMockAPITestCase(RequestsBackendMockAPITestCase):
@@ -36,6 +36,7 @@ class SendGridBackendMockAPITestCase(RequestsBackendMockAPITestCase):
         self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', 'from@example.com', ['to@example.com'])
 
 
+@tag('sendgrid')
 class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -328,6 +329,7 @@ class SendGridBackendStandardEmailTests(SendGridBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('sendgrid')
 class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -709,6 +711,7 @@ class SendGridBackendAnymailFeatureTests(SendGridBackendMockAPITestCase):
                          {"email": "from@example.com", "name": "Sender, Inc."})
 
 
+@tag('sendgrid')
 class SendGridBackendRecipientsRefusedTests(SendGridBackendMockAPITestCase):
     """Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid"""
 
@@ -718,11 +721,13 @@ class SendGridBackendRecipientsRefusedTests(SendGridBackendMockAPITestCase):
     pass  # not applicable to this backend
 
 
+@tag('sendgrid')
 class SendGridBackendSessionSharingTestCase(SessionSharingTestCasesMixin, SendGridBackendMockAPITestCase):
     """Requests session sharing tests"""
     pass  # tests are defined in the mixin
 
 
+@tag('sendgrid')
 @override_settings(EMAIL_BACKEND="anymail.backends.sendgrid.EmailBackend")
 class SendGridBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin):
     """Test ESP backend without required settings in place"""
@@ -732,6 +737,7 @@ class SendGridBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin)
             mail.send_mail('Subject', 'Message', 'from@example.com', ['to@example.com'])
 
 
+@tag('sendgrid')
 @override_settings(EMAIL_BACKEND="anymail.backends.sendgrid.EmailBackend")
 class SendGridBackendDisallowsV2Tests(SimpleTestCase, AnymailTestMixin):
     """Using v2-API-only features should cause errors with v3 backend"""

tests/test_sendgrid_inbound.py

@@ -2,6 +2,7 @@ import json
 from textwrap import dedent
 
 import six
+from django.test import tag
 from mock import ANY
 
 from anymail.inbound import AnymailInboundMessage
@@ -12,6 +13,7 @@ from .utils import sample_image_content, sample_email_content
 from .webhook_cases import WebhookTestCase
 
 
+@tag('sendgrid')
 class SendgridInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):
         raw_event = {

tests/test_sendgrid_integration.py

@@ -2,19 +2,18 @@ import os
 import unittest
 from datetime import datetime, timedelta
 
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 SENDGRID_TEST_API_KEY = os.getenv('SENDGRID_TEST_API_KEY')
 SENDGRID_TEST_TEMPLATE_ID = os.getenv('SENDGRID_TEST_TEMPLATE_ID')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('sendgrid', 'live')
 @unittest.skipUnless(SENDGRID_TEST_API_KEY,
                      "Set SENDGRID_TEST_API_KEY environment variable "
                      "to run SendGrid integration tests")

tests/test_sendgrid_webhooks.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from django.test import tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -9,6 +10,7 @@ from anymail.webhooks.sendgrid import SendGridTrackingWebhookView
 from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
 
 
+@tag('sendgrid')
 class SendGridWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     def call_webhook(self):
         return self.client.post('/anymail/sendgrid/tracking/',
@@ -17,6 +19,7 @@ class SendGridWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixi
     # Actual tests are in WebhookBasicAuthTestsMixin
 
 
+@tag('sendgrid')
 class SendGridDeliveryTestCase(WebhookTestCase):
 
     def test_processed_event(self):

tests/test_sendinblue_backend.py

@@ -10,8 +10,7 @@ from email.mime.image import MIMEImage
 
 import six
 from django.core import mail
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 from django.utils.timezone import get_fixed_timezone, override as override_current_timezone
 
 from anymail.exceptions import (AnymailAPIError, AnymailConfigurationError, AnymailSerializationError,
@@ -25,6 +24,7 @@ from .utils import sample_image_content, sample_image_path, SAMPLE_IMAGE_FILENAM
 longtype = int if six.PY3 else long  # NOQA: F821
 
 
+@tag('sendinblue')
 @override_settings(EMAIL_BACKEND='anymail.backends.sendinblue.EmailBackend',
                    ANYMAIL={'SENDINBLUE_API_KEY': 'test_api_key'})
 class SendinBlueBackendMockAPITestCase(RequestsBackendMockAPITestCase):
@@ -38,6 +38,7 @@ class SendinBlueBackendMockAPITestCase(RequestsBackendMockAPITestCase):
         self.message = mail.EmailMultiAlternatives('Subject', 'Text Body', 'from@example.com', ['to@example.com'])
 
 
+@tag('sendinblue')
 class SendinBlueBackendStandardEmailTests(SendinBlueBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -274,6 +275,7 @@ class SendinBlueBackendStandardEmailTests(SendinBlueBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('sendinblue')
 class SendinBlueBackendAnymailFeatureTests(SendinBlueBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -510,6 +512,7 @@ class SendinBlueBackendAnymailFeatureTests(SendinBlueBackendMockAPITestCase):
         self.assertRegex(str(err), r"Decimal.*is not JSON serializable")  # original message
 
 
+@tag('sendinblue')
 class SendinBlueBackendRecipientsRefusedTests(SendinBlueBackendMockAPITestCase):
     """Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid"""
 
@@ -519,11 +522,13 @@ class SendinBlueBackendRecipientsRefusedTests(SendinBlueBackendMockAPITestCase):
     pass  # not applicable to this backend
 
 
+@tag('sendinblue')
 class SendinBlueBackendSessionSharingTestCase(SessionSharingTestCasesMixin, SendinBlueBackendMockAPITestCase):
     """Requests session sharing tests"""
     pass  # tests are defined in the mixin
 
 
+@tag('sendinblue')
 @override_settings(EMAIL_BACKEND="anymail.backends.sendinblue.EmailBackend")
 class SendinBlueBackendImproperlyConfiguredTests(SimpleTestCase, AnymailTestMixin):
     """Test ESP backend without required settings in place"""

tests/test_sendinblue_integration.py

@@ -1,18 +1,17 @@
 import os
 import unittest
 
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin
 
 SENDINBLUE_TEST_API_KEY = os.getenv('SENDINBLUE_TEST_API_KEY')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('sendinblue', 'live')
 @unittest.skipUnless(SENDINBLUE_TEST_API_KEY,
                      "Set SENDINBLUE_TEST_API_KEY environment variable "
                      "to run SendinBlue integration tests")

tests/test_sendinblue_webhooks.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from django.test import tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -9,6 +10,7 @@ from anymail.webhooks.sendinblue import SendinBlueTrackingWebhookView
 from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
 
 
+@tag('sendinblue')
 class SendinBlueWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     def call_webhook(self):
         return self.client.post('/anymail/sendinblue/tracking/',
@@ -17,6 +19,7 @@ class SendinBlueWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMi
     # Actual tests are in WebhookBasicAuthTestsMixin
 
 
+@tag('sendinblue')
 class SendinBlueDeliveryTestCase(WebhookTestCase):
     # SendinBlue's webhook payload data doesn't seem to be documented anywhere.
     # There's a list of webhook events at https://apidocs.sendinblue.com/webhooks/#3.

tests/test_sparkpost_backend.py

@@ -8,11 +8,9 @@ import os
 import requests
 import six
 from django.core import mail
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 from django.utils.timezone import get_fixed_timezone, override as override_current_timezone, utc
 from mock import patch
-from sparkpost.exceptions import SparkPostAPIException
 
 from anymail.exceptions import (AnymailAPIError, AnymailUnsupportedFeature, AnymailRecipientsRefused,
                                 AnymailConfigurationError, AnymailInvalidAddress)
@@ -21,6 +19,7 @@ from anymail.message import attach_inline_image_file
 from .utils import AnymailTestMixin, decode_att, SAMPLE_IMAGE_FILENAME, sample_image_path, sample_image_content
 
 
+@tag('sparkpost')
 @override_settings(EMAIL_BACKEND='anymail.backends.sparkpost.EmailBackend',
                    ANYMAIL={'SPARKPOST_API_KEY': 'test_api_key'})
 class SparkPostBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin):
@@ -48,6 +47,7 @@ class SparkPostBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin):
         return self.mock_send.return_value
 
     def set_mock_failure(self, status_code=400, raw=b'{"errors":[{"message":"test error"}]}', encoding='utf-8'):
+        from sparkpost.exceptions import SparkPostAPIException
         # Need to build a real(-ish) requests.Response for SparkPostAPIException
         response = requests.Response()
         response.status_code = status_code
@@ -82,6 +82,7 @@ class SparkPostBackendMockAPITestCase(SimpleTestCase, AnymailTestMixin):
             raise AssertionError(msg or "ESP API was called and shouldn't have been")
 
 
+@tag('sparkpost')
 class SparkPostBackendStandardEmailTests(SparkPostBackendMockAPITestCase):
     """Test backend support for Django standard email features"""
 
@@ -325,6 +326,7 @@ class SparkPostBackendStandardEmailTests(SparkPostBackendMockAPITestCase):
             self.message.send()
 
 
+@tag('sparkpost')
 class SparkPostBackendAnymailFeatureTests(SparkPostBackendMockAPITestCase):
     """Test backend support for Anymail added features"""
 
@@ -545,6 +547,7 @@ class SparkPostBackendAnymailFeatureTests(SparkPostBackendMockAPITestCase):
     #   modify those errors.
 
 
+@tag('sparkpost')
 class SparkPostBackendRecipientsRefusedTests(SparkPostBackendMockAPITestCase):
     """Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid"""
 
@@ -586,6 +589,7 @@ class SparkPostBackendRecipientsRefusedTests(SparkPostBackendMockAPITestCase):
         self.assertEqual(sent, 1)  # refused message is included in sent count
 
 
+@tag('sparkpost')
 @override_settings(EMAIL_BACKEND="anymail.backends.sparkpost.EmailBackend")
 class SparkPostBackendConfigurationTests(SimpleTestCase, AnymailTestMixin):
     """Test various SparkPost client options"""

tests/test_sparkpost_inbound.py

@@ -2,6 +2,7 @@ import json
 from base64 import b64encode
 from textwrap import dedent
 
+from django.test import tag
 from mock import ANY
 
 from anymail.inbound import AnymailInboundMessage
@@ -12,6 +13,7 @@ from .utils import sample_image_content, sample_email_content
 from .webhook_cases import WebhookTestCase
 
 
+@tag('sparkpost')
 class SparkpostInboundTestCase(WebhookTestCase):
     def test_inbound_basics(self):
         event = {

tests/test_sparkpost_integration.py

@@ -2,18 +2,17 @@ import os
 import unittest
 from datetime import datetime, timedelta
 
-from django.test import SimpleTestCase
-from django.test.utils import override_settings
+from django.test import SimpleTestCase, override_settings, tag
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
+from .utils import AnymailTestMixin, sample_image_path
 
 SPARKPOST_TEST_API_KEY = os.getenv('SPARKPOST_TEST_API_KEY')
 
 
-@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
+@tag('sparkpost', 'live')
 @unittest.skipUnless(SPARKPOST_TEST_API_KEY,
                      "Set SPARKPOST_TEST_API_KEY environment variable "
                      "to run SparkPost integration tests")

tests/test_sparkpost_webhooks.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from django.test import tag
 from django.utils.timezone import utc
 from mock import ANY
 
@@ -10,6 +11,7 @@ from anymail.webhooks.sparkpost import SparkPostTrackingWebhookView
 from .webhook_cases import WebhookBasicAuthTestsMixin, WebhookTestCase
 
 
+@tag('sparkpost')
 class SparkPostWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMixin):
     def call_webhook(self):
         return self.client.post('/anymail/sparkpost/tracking/',
@@ -18,6 +20,7 @@ class SparkPostWebhookSecurityTestCase(WebhookTestCase, WebhookBasicAuthTestsMix
     # Actual tests are in WebhookBasicAuthTestsMixin
 
 
+@tag('sparkpost')
 class SparkPostDeliveryTestCase(WebhookTestCase):
 
     def test_ping_event(self):

tests/utils.py

@@ -8,32 +8,12 @@ import uuid
 import warnings
 from base64 import b64decode
 from contextlib import contextmanager
-from distutils.util import strtobool
 
 import six
 from django.test import Client
 from six.moves import StringIO
 
 
-def envbool(var, default=False):
-    """Returns value of environment variable var as a bool, or default if not set.
-
-    Converts `'true'` to `True`, and `'false'` to `False`.
-    See :func:`~distutils.util.strtobool` for full list of allowable values.
-    """
-    val = os.getenv(var, None)
-    if val is None:
-        return default
-    else:
-        return strtobool(val)
-
-
-# RUN_LIVE_TESTS: whether to run live API integration tests.
-# True by default, except in CONTINUOUS_INTEGRATION job.
-# (See comments and overrides in .travis.yml.)
-RUN_LIVE_TESTS = envbool('RUN_LIVE_TESTS', default=not envbool('CONTINUOUS_INTEGRATION'))
-
-
 def decode_att(att):
     """Returns the original data from base64-encoded attachment content"""
     return b64decode(att.encode('ascii'))

tox.ini

@@ -1,17 +1,20 @@
 [tox]
 envlist =
+    # Factors: django-python-extras
     # Test these environments first, to catch most errors early...
     lint
-    django21-py36
-    django111-py27
+    django21-py36-all
+    django111-py27-all
     docs
     # ... then test all the other supported combinations:
-    django21-py{35,37,py3}
-    django20-py{35,36,py3}
-    django111-py{34,35,36,py}
+    django21-py{35,37,py3}-all
+    django20-py{35,36,py3}-all
+    django111-py{34,35,36,py}-all
     # ... then prereleases (if available):
-    django22-py{35,36,37,py3}
-    djangoMaster-py{36,37}
+    django22-py{35,36,37,py3}-all
+    djangoMaster-py{36,37}-all
+    # ... then partial installation (limit extras):
+    django21-py37-{none,amazon_ses,sparkpost}
 
 [testenv]
 deps =
@@ -20,19 +23,27 @@ deps =
     django21: django~=2.1.0
     django22: django>=2.2a1
     djangoMaster: https://github.com/django/django/tarball/master
-    # testing dependencies (duplicates setup.py tests_require):
+    # testing dependencies (duplicates setup.py tests_require, less optional extras):
     mock
-    boto3
-    sparkpost
+extras =
+    all,amazon_ses: amazon_ses
+    all,sparkpost: sparkpost
+setenv =
+    # tell runtests.py to limit some test tags based on extras factor
+    none: ANYMAIL_SKIP_TESTS=amazon_ses,sparkpost
+    amazon_ses: ANYMAIL_ONLY_TEST=amazon_ses
+    sparkpost: ANYMAIL_ONLY_TEST=sparkpost
 ignore_outcome =
-    djangoMaster: True
-usedevelop = True
-args_are_paths = False
+    # CI that wants to handle errors itself can set TOX_FORCE_IGNORE_OUTCOME=false
+    djangoMaster: {env:TOX_FORCE_IGNORE_OUTCOME:true}
+args_are_paths = false
+commands_pre =
+    python -VV
 commands =
-    python --version
-    # pip install .[mailgun,...,sparkpost]  ## usedevelop=True + manual deps is much faster on repeat runs
     python runtests.py {posargs}
 passenv =
+    ANYMAIL_ONLY_TEST
+    ANYMAIL_SKIP_TESTS
     RUN_LIVE_TESTS
     CONTINUOUS_INTEGRATION
     AMAZON_SES_TEST_*
@@ -46,7 +57,7 @@ passenv =
 
 [testenv:lint]
 basepython = python3
-skip_install = True
+skip_install = true
 passenv =
     CONTINUOUS_INTEGRATION
 	# (but not any of the live test API keys)
@@ -59,7 +70,7 @@ commands =
 
 [testenv:docs]
 basepython = python3
-skip_install = True
+skip_install = true
 passenv =
     CONTINUOUS_INTEGRATION
 	# (but not any of the live test API keys)
@@ -78,20 +89,3 @@ commands =
     /bin/bash -c 'python setup.py --long-description \
                   | rst2html5.py --config=docs/_readme/docutils.cfg \
                   > {env:DOCS_BUILD_DIR}/readme.html'
-
-[travis]
-unignore_outcomes = True
-python =
-    3.6: py36, lint, docs
-
-[travis:env]
-DJANGO =
-    1.11: django111
-    2.0: django20
-    2.1: django21
-    2.2: django22
-    master: djangoMaster
-LINT_AND_DOCS =
-    true: lint, docs
-    docs: docs
-    lint: lint