CI: get GitHub test matrix from tox envlist

Stop maintaining duplicate (and often not-quite-
in-sync) test matrices in tox.ini and test.yml.

.github/workflows/test.yml

@@ -13,95 +13,57 @@ on:
 
 
 jobs:
-  test:
+  get-envlist:
-    name: ${{ matrix.config.tox }} ${{ matrix.config.options }}
     runs-on: ubuntu-20.04
-    timeout-minutes: 15
+    outputs:
-    strategy:
+      envlist: ${{ steps.generate-envlist.outputs.envlist }}
-      fail-fast: false
-      matrix:
-        config:
-          - { tox: "lint", python: "3.10" }
-          - { tox: "docs", python: "3.10" }
-
-          # Anymail supports the same Python versions as Django, plus PyPy.
-          # https://docs.djangoproject.com/en/dev/faq/install/#what-python-version-can-i-use-with-django
-
-          # Django 2.0: Python 3.5+
-          - { tox: django20-py35-all, python: "3.5" }
-          - { tox: django20-py36-all, python: "3.6" }
-          - { tox: django20-pypy3-all, python: "pypy-3.6" }
-          # Django 2.1: Python 3.5, 3.6, or 3.7
-          - { tox: django21-py35-all, python: "3.5" }
-          - { tox: django21-py36-all, python: "3.6" }
-          - { tox: django21-py37-all, python: "3.7" }
-          - { tox: django21-pypy3-all, python: "pypy-3.7" }
-          # Django 2.2: Python 3.5, 3.6, 3.7, 3.8 (added in 2.2.8), 3.9 (added in 2.2.17)
-          - { tox: django22-py35-all, python: "3.5" }
-          - { tox: django22-py36-all, python: "3.6" }
-          - { tox: django22-py37-all, python: "3.7" }
-          - { tox: django22-py38-all, python: "3.8" }
-          - { tox: django22-py39-all, python: "3.9" }
-          - { tox: django22-pypy3-all, python: "pypy-3.8" }
-          # Django 3.0: Python 3.6, 3.7, 3.8, 3.9 (added in 3.0.11)
-          - { tox: django30-py36-all, python: "3.6" }
-          - { tox: django30-py37-all, python: "3.7" }
-          - { tox: django30-py38-all, python: "3.8" }
-          - { tox: django30-py39-all, python: "3.9" }
-          - { tox: django30-pypy3-all, python: "pypy-3.8" }
-          # Django 3.1: Python 3.6, 3.7, 3.8, 3.9 (added in 3.1.3)
-          - { tox: django31-py36-all, python: "3.6" }
-          - { tox: django31-py37-all, python: "3.7" }
-          - { tox: django31-py38-all, python: "3.8" }
-          - { tox: django31-py39-all, python: "3.9" }
-          - { tox: django31-pypy3-all, python: "pypy-3.8" }
-          # Django 3.2: Python 3.6, 3.7, 3.8, 3.9
-          - { tox: django32-py36-all, python: "3.6" }
-          - { tox: django32-py37-all, python: "3.7" }
-          - { tox: django32-py38-all, python: "3.8" }
-          - { tox: django32-py39-all, python: "3.9" }
-          - { tox: django32-pypy3-all, python: "pypy-3.8" }
-          # Django 4.0: Python 3.8, 3.9, 3.10
-          - { tox: django40-py38-all, python: "3.8" }
-          - { tox: django40-py39-all, python: "3.9" }
-          - { tox: django40-py310-all, python: "3.10" }
-          - { tox: django40-pypy3-all, python: "pypy-3.8" }
-          # Django current development (direct from GitHub source)
-          - { tox: djangoDev-py38-all, python: "3.8", options: allow-failures }
-          - { tox: djangoDev-py39-all, python: "3.9", options: allow-failures }
-          - { tox: djangoDev-py310-all, python: "3.10", options: allow-failures }
-          - { tox: djangoDev-py311-all, python: "3.11.0-alpha - 3.11", options: allow-failures }
-          # Install without optional extras (don't need to cover entire matrix)
-          - { tox: django40-py310-none, python: "3.10" }
-          - { tox: django40-py310-amazon_ses, python: "3.10" }
-          - { tox: django40-py310-postal, python: "3.10" }
-          # Test some specific older package versions
-          - { tox: django22-py37-all-old_urllib3, python: "3.7" }
-
     steps:
       - name: Get code
         uses: actions/checkout@v2
-      - name: Setup Python ${{ matrix.config.python }}
+      - name: Install tox-gh-matrix
+        run: |
+          python -m pip install tox tox-gh-matrix
+          python -m tox --version
+      - name: Generate tox envlist
+        id: generate-envlist
+        run: |
+          python -m tox --gh-matrix
+          python -m tox --gh-matrix-dump  # for debugging
+
+  test:
+    runs-on: ubuntu-20.04
+    needs: get-envlist
+    strategy:
+      matrix:
+        tox: ${{ fromJSON(needs.get-envlist.outputs.envlist) }}
+      fail-fast: false
+
+    name: ${{ matrix.tox.name }} ${{ matrix.tox.ignore_outcome && 'allow-failures' || '' }}
+    timeout-minutes: 15
+    steps:
+      - name: Get code
+        uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.tox.python.version }}
         # Ensure matrix Python version is installed and available for tox
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.config.python }}
+          python-version: ${{ matrix.tox.python.spec }}
       - name: Setup default Python
         # Change default Python version back to something consistent
         # for installing/running tox
         uses: actions/setup-python@v2
         with:
-          python-version: "3.10"
+          python-version: "3.8"
       - name: Install tox
         run: |
           set -x
-          python --version
+          python -VV
-          pip install tox
+          python -m pip install tox
-          tox --version
+          python -m tox --version
-      - name: Test ${{ matrix.config.tox }}
+      - name: Test ${{ matrix.tox.name }}
         run: |
-          tox -e ${{ matrix.config.tox }}
+          python -m tox -e ${{ matrix.tox.name }}
-        continue-on-error: ${{ contains( matrix.config.options, 'allow-failures' ) }}
+        continue-on-error: ${{ matrix.tox.ignore_outcome == true }}
         env:
           CONTINUOUS_INTEGRATION: true
-          TOX_FORCE_IGNORE_OUTCOME: false
+          TOX_OVERRIDE_IGNORE_OUTCOME: false

tox.ini

@@ -1,21 +1,30 @@
 [tox]
 envlist =
+    # Anymail supports the same Python versions as Django, plus PyPy.
+    # https://docs.djangoproject.com/en/dev/faq/install/#what-python-version-can-i-use-with-django
     # Factors: django-python-extras
-    # Test these environments first, to catch most errors early...
+    # Test lint, docs, earliest/latest Django first, to catch most errors early...
     lint
     django40-py310-all
     django20-py35-all
     docs
     # ... then test all the other supported combinations:
-    django40-py{38,39,py3}-all
+    # Django 4.0: Python 3.8, 3.9, 3.10
-    django32-py{36,37,38,39,py3}-all
+    django40-py{38,39,py38}-all
-    django31-py{36,37,38,39,py3}-all
+    # Django 3.2: Python 3.6, 3.7, 3.8, 3.9
-    django30-py{36,37,38,39,py3}-all
+    django32-py{36,37,38,39,py38}-all
-    django22-py{35,36,37,38,39,py3}-all
+    # Django 3.1: Python 3.6, 3.7, 3.8, 3.9 (added in 3.1.3)
-    django21-py{35,36,37,py3}-all
+    django31-py{36,37,38,39,py38}-all
-    django20-py{36,py3}-all
+    # Django 3.0: Python 3.6, 3.7, 3.8, 3.9 (added in 3.0.11)
-    # ... then prereleases (if available):
+    django30-py{36,37,38,39,py38}-all
-    djangoDev-py{38,39,310,py3}-all
+    # Django 2.2: Python 3.5, 3.6, 3.7, 3.8 (added in 2.2.8), 3.9 (added in 2.2.17)
+    django22-py{35,36,37,38,39,py38}-all
+    # Django 2.1: Python 3.5, 3.6, or 3.7
+    django21-py{35,36,37,py37}-all
+    # Django 2.0: Python 3.5+
+    django20-py{36,py36}-all
+    # ... then prereleases (if available) and current development:
+    djangoDev-py{38,39,310,py38}-all
     # ... then partial installation (limit extras):
     django40-py310-{none,amazon_ses,postal}
     # ... then older versions of some dependencies:
@@ -51,8 +60,8 @@ setenv =
     sendinblue: ANYMAIL_ONLY_TEST=sendinblue
     sparkpost: ANYMAIL_ONLY_TEST=sparkpost
 ignore_outcome =
-    # CI that wants to handle errors itself can set TOX_FORCE_IGNORE_OUTCOME=false
+    # CI that wants to handle errors itself can set TOX_OVERRIDE_IGNORE_OUTCOME=false
-    djangoDev: {env:TOX_FORCE_IGNORE_OUTCOME:true}
+    djangoDev: {env:TOX_OVERRIDE_IGNORE_OUTCOME:true}
 args_are_paths = false
 commands_pre =
     python -VV
@@ -67,7 +76,7 @@ passenv =
     ANYMAIL_TEST_*
 
 [testenv:lint]
-basepython = python3
+basepython = python3.8
 skip_install = true
 passenv =
     CONTINUOUS_INTEGRATION
@@ -81,7 +90,7 @@ commands =
     flake8
 
 [testenv:docs]
-basepython = python3
+basepython = python3.8
 skip_install = true
 passenv =
     CONTINUOUS_INTEGRATION