From 0c5911ca349bc0158e958aaee7b06463026dd0bf Mon Sep 17 00:00:00 2001 From: medmunds Date: Thu, 23 Jun 2016 15:21:40 -0700 Subject: [PATCH] Tests: limit live API integration tests in Travis runs To conserve our ESP test accounts' send quotas, don't run the live API integration tests 13 times in every Travis run. Instead, just run them twice, on a representative set of Python/Django combinations: * Once on Python 2.7 (currently with Django 1.8) * Once on Python 3.x (currently 3.5 with Django 1.9) (Prep for running weekly tests on Travis cron.) The *non*-integration tests still run on all combos. * Introduce RUN_LIVE_TESTS environment var to control whether live API integration test cases should run. Default True, except in Travis-CI runs default False. * Enable RUN_LIVE_TESTS in .travis.yml matrix for the Python/Django combos listed above. --- .travis.yml | 8 ++++++-- tests/test_mailgun_integration.py | 3 ++- tests/test_mandrill_integration.py | 3 ++- tests/test_postmark_integration.py | 5 ++++- tests/test_sendgrid_integration.py | 4 +++- tests/test_sparkpost_integration.py | 3 ++- tests/utils.py | 20 ++++++++++++++++++++ 7 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 05c78da..f8d2e1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,12 @@ matrix: include: # Anymail supports the same python versions as Django, excluding Python 3.2, but adding pypy. # https://docs.djangoproject.com/en/dev/faq/install/#what-python-version-can-i-use-with-django + + # Live API integration tests are only run on a few, representative Python/Django version + # cominbations, to avoid rapidly consuming the testing accounts' entire send allotments. + # Django 1.8: Python 2.7, 3.2 (until the end of 2016), 3.3, 3.4, 3.5 - - { env: DJANGO=django==1.8, python: 2.7 } + - { env: DJANGO=django==1.8 RUN_LIVE_TESTS=true, python: 2.7 } - { env: DJANGO=django==1.8, python: 3.3 } - { env: DJANGO=django==1.8, python: 3.4 } - { env: DJANGO=django==1.8, python: 3.5 } @@ -13,7 +17,7 @@ matrix: # Django 1.9: Python 2.7, 3.4, 3.5 - { env: DJANGO=django==1.9, python: 2.7 } - { env: DJANGO=django==1.9, python: 3.4 } - - { env: DJANGO=django==1.9, python: 3.5 } + - { env: DJANGO=django==1.9 RUN_LIVE_TESTS=true, python: 3.5 } - { env: DJANGO=django==1.9, python: pypy } # Django 1.10 (prerelease): Python 2.7, 3.4, 3.5 - { env: DJANGO="--pre django", python: 2.7 } diff --git a/tests/test_mailgun_integration.py b/tests/test_mailgun_integration.py index 7bc1516..0c0b448 100644 --- a/tests/test_mailgun_integration.py +++ b/tests/test_mailgun_integration.py @@ -13,12 +13,13 @@ from django.test.utils import override_settings from anymail.exceptions import AnymailAPIError from anymail.message import AnymailMessage -from .utils import AnymailTestMixin, sample_image_path +from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS MAILGUN_TEST_API_KEY = os.getenv('MAILGUN_TEST_API_KEY') MAILGUN_TEST_DOMAIN = os.getenv('MAILGUN_TEST_DOMAIN') +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") @unittest.skipUnless(MAILGUN_TEST_API_KEY and MAILGUN_TEST_DOMAIN, "Set MAILGUN_TEST_API_KEY and MAILGUN_TEST_DOMAIN environment variables " "to run Mailgun integration tests") diff --git a/tests/test_mandrill_integration.py b/tests/test_mandrill_integration.py index d60c9ad..f6d131b 100644 --- a/tests/test_mandrill_integration.py +++ b/tests/test_mandrill_integration.py @@ -10,11 +10,12 @@ from django.test.utils import override_settings from anymail.exceptions import AnymailAPIError, AnymailRecipientsRefused from anymail.message import AnymailMessage -from .utils import AnymailTestMixin, sample_image_path +from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS MANDRILL_TEST_API_KEY = os.getenv('MANDRILL_TEST_API_KEY') +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") @unittest.skipUnless(MANDRILL_TEST_API_KEY, "Set MANDRILL_TEST_API_KEY environment variable to run integration tests") @override_settings(MANDRILL_API_KEY=MANDRILL_TEST_API_KEY, diff --git a/tests/test_postmark_integration.py b/tests/test_postmark_integration.py index f70dc09..e575f95 100644 --- a/tests/test_postmark_integration.py +++ b/tests/test_postmark_integration.py @@ -1,14 +1,17 @@ from __future__ import unicode_literals +import unittest + from django.test import SimpleTestCase from django.test.utils import override_settings from anymail.exceptions import AnymailAPIError from anymail.message import AnymailMessage -from .utils import AnymailTestMixin, sample_image_path +from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") @override_settings(ANYMAIL_POSTMARK_SERVER_TOKEN="POSTMARK_API_TEST", EMAIL_BACKEND="anymail.backends.postmark.PostmarkBackend") class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): diff --git a/tests/test_sendgrid_integration.py b/tests/test_sendgrid_integration.py index 15f3bf9..ce4f7de 100644 --- a/tests/test_sendgrid_integration.py +++ b/tests/test_sendgrid_integration.py @@ -11,7 +11,7 @@ from django.test.utils import override_settings from anymail.exceptions import AnymailAPIError from anymail.message import AnymailMessage -from .utils import AnymailTestMixin, sample_image_path +from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS # For API_KEY auth tests: SENDGRID_TEST_API_KEY = os.getenv('SENDGRID_TEST_API_KEY') @@ -21,6 +21,7 @@ SENDGRID_TEST_USERNAME = os.getenv('SENDGRID_TEST_USERNAME') SENDGRID_TEST_PASSWORD = os.getenv('SENDGRID_TEST_PASSWORD') +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") @unittest.skipUnless(SENDGRID_TEST_API_KEY, "Set SENDGRID_TEST_API_KEY environment variable " "to run SendGrid integration tests") @@ -118,6 +119,7 @@ class SendGridBackendIntegrationTests(SimpleTestCase, AnymailTestMixin): self.assertIn("authorization grant is invalid", str(err)) +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") @unittest.skipUnless(SENDGRID_TEST_USERNAME and SENDGRID_TEST_PASSWORD, "Set SENDGRID_TEST_USERNAME and SENDGRID_TEST_PASSWORD" "environment variables to run SendGrid integration tests") diff --git a/tests/test_sparkpost_integration.py b/tests/test_sparkpost_integration.py index 9bf1ff3..482c6f8 100644 --- a/tests/test_sparkpost_integration.py +++ b/tests/test_sparkpost_integration.py @@ -10,11 +10,12 @@ from django.test.utils import override_settings from anymail.exceptions import AnymailAPIError from anymail.message import AnymailMessage -from .utils import AnymailTestMixin, sample_image_path +from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS SPARKPOST_TEST_API_KEY = os.getenv('SPARKPOST_TEST_API_KEY') +@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment") @unittest.skipUnless(SPARKPOST_TEST_API_KEY, "Set SPARKPOST_TEST_API_KEY environment variable " "to run SparkPost integration tests") diff --git a/tests/utils.py b/tests/utils.py index 80fdb73..79cbdb4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,6 @@ # Anymail test utils import sys +from distutils.util import strtobool import os import re @@ -10,6 +11,25 @@ from contextlib import contextmanager from django.test import Client +def envbool(var, default=False): + """Returns value of environment variable var as a bool, or default if not set. + + Converts `'true'` to `True`, and `'false'` to `False`. + See :func:`~distutils.util.strtobool` for full list of allowable values. + """ + val = os.getenv(var, None) + if val is None: + return default + else: + return strtobool(val) + + +# RUN_LIVE_TESTS: whether to run live API integration tests. +# True by default, except in CONTINUOUS_INTEGRATION job. +# (See comments and overrides in .travis.yml.) +RUN_LIVE_TESTS = envbool('RUN_LIVE_TESTS', default=not envbool('CONTINUOUS_INTEGRATION')) + + def decode_att(att): """Returns the original data from base64-encoded attachment content""" return b64decode(att.encode('ascii'))