Rework execnet onto an async-native core (sync facade, trio or asyncio engine, uv-provisioned workers)

.github/workflows/test.yml

@@ -10,6 +10,8 @@ on:
     branches:
       - "main"
 
+  workflow_dispatch:
+
 # Cancel running jobs for the same workflow and branch.
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -24,6 +26,29 @@ jobs:
     - name: Build and Check Package
       uses: hynek/build-and-inspect-python-package@v2.18
 
+  # The docs build is `-W`, so a stale reference is an error, and the same
+  # env runs the doc examples as doctests -- they claim to be tested, and
+  # for years they silently were not.
+  docs:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v7
+      with:
+        python-version: "3.13"
+
+    - name: Install dependencies
+      run: pip install tox
+
+    - name: Build the docs and run the doc examples
+      run: tox run -e docs
+
   test:
 
     needs: [package]
@@ -51,7 +76,13 @@ jobs:
       uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python }}
-        allow-prereleases: true
+        # Only for the version that has no release yet: setup-python
+        # otherwise resolves released versions only, and a 3.15 job dies at
+        # setup ("version '3.15' with architecture 'x64' was not found").
+        # Scoped to 3.15 rather than switched on for the matrix, so the
+        # released versions keep testing what users actually install --
+        # move it to the next version when 3.15 ships.
+        allow-prereleases: ${{ matrix.python == '3.15' }}
 
     - name: Install dependencies
       run: pip install tox twine
@@ -63,5 +94,105 @@ jobs:
 
     - name: Test
       shell: bash
+      # The suite is installed from the sdist, which leaves a dev version
+      # with no source tree to build a provisioning wheel from -- so every
+      # test that needs a provisioned worker (ssh, foreign python, via)
+      # would skip.  Hand it the wheel from the same build instead: the
+      # workers then run exactly the artifact under test.
       run: |
+        export EXECNET_PROVISION_WHEEL="$PWD/$(find dist -name '*.whl' | head -1)"
         tox run -e py --installpkg `find dist/*.tar.gz`
+
+  # pytest-xdist drives execnet's local parallel testing: popen gateways,
+  # the main-thread worker profile, crashed-worker replacement, and the
+  # report/warning serialization.  Our own suite runs xdist as a *tool*,
+  # which does not exercise any of that -- so run xdist's own test suite
+  # against the execnet built from this branch.
+  #
+  # Two targets:
+  #
+  # * `release` is a pinned combination known to be green against the last
+  #   *released* execnet, so a failure there means we broke something.  It
+  #   blocks.  Bump `ref`/`pytest` together, and only after checking the new
+  #   pair is green against released execnet -- otherwise the signal is
+  #   gone.  (xdist 3.8.0 needs pytest<9: with pytest 9 two of its tests
+  #   fail against released execnet too.)
+  # * `default-branch` is early warning for drift on both sides.  It is
+  #   allowed to fail, because it can go red for reasons that are xdist's
+  #   to fix.  An empty `ref` follows whatever xdist's default branch is
+  #   (`master` today) rather than hardcoding a name that could change.
+  #
+  # To reproduce locally:
+  #   git clone https://github.com/pytest-dev/pytest-xdist
+  #   git -C pytest-xdist checkout v3.8.0     # or stay on the default branch
+  #   python -m venv .xdist-venv
+  #   .xdist-venv/bin/pip install ./pytest-xdist[testing] "pytest<9" .
+  #   cd pytest-xdist && ../.xdist-venv/bin/python -m pytest
+  xdist:
+
+    needs: [package]
+
+    runs-on: ubuntu-latest
+
+    name: xdist (${{ matrix.name }})
+
+    continue-on-error: ${{ matrix.experimental }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: release
+            ref: "v3.8.0"
+            pytest: "pytest<9"
+            experimental: false
+          - name: default-branch
+            ref: ""
+            pytest: "pytest"
+            experimental: true
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        path: execnet
+
+    - uses: actions/checkout@v7
+      with:
+        repository: pytest-dev/pytest-xdist
+        ref: ${{ matrix.ref }}
+        path: pytest-xdist
+
+    - name: Download Package
+      uses: actions/download-artifact@v8
+      with:
+        name: Packages
+        path: dist
+
+    - name: Set up Python
+      uses: actions/setup-python@v7
+      with:
+        python-version: "3.13"
+
+    - name: Install pytest-xdist and this execnet
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install "./pytest-xdist[testing]" "${{ matrix.pytest }}"
+        # after xdist, so its execnet>=2.1 requirement does not win
+        python -m pip install --force-reinstall `find dist/*.tar.gz`
+        python -c "import execnet, xdist; print('execnet', execnet.__version__)"
+
+    - name: Run the pytest-xdist test suite
+      shell: bash
+      working-directory: pytest-xdist
+      run: |
+        deselect=()
+        # `|| [ -n "$line" ]` so a file without a trailing newline still
+        # contributes its last entry
+        while read -r line || [ -n "$line" ]; do
+          line="${line%%#*}"
+          line="$(echo "$line" | xargs)"
+          [ -n "$line" ] && deselect+=(--deselect "$line")
+        done < ../execnet/.github/xdist-known-failures.txt
+        printf 'deselecting %d known failure(s)\n' $(( ${#deselect[@]} / 2 ))
+        python -m pytest -ra "${deselect[@]}"

.github/xdist-known-failures.txt

@@ -0,0 +1,19 @@
+# pytest-xdist tests that are expected to fail against this execnet, with
+# the reason.  Everything not listed here must pass -- the whole point of
+# the `xdist (release)` CI job is that this list stays short and justified.
+#
+# Blank lines and `#` comments are ignored; every other line is one pytest
+# node id, passed to `pytest --deselect`.  The same list is used for both
+# the pinned release and the `main` target; a node id that does not exist
+# in one of them is silently ignored, so an entry may name a test that
+# only one version has.
+#
+# Do not add an entry to make CI green.  Add one only for a test that is
+# asserting behaviour execnet deliberately changed, and say so.
+
+# Asserts `sys.argv == ["-c"]`, and its own docstring says it is
+# documenting "the behavior due to execnet using `python -c`".  execnet no
+# longer ships source over the wire: workers are launched through the CLI
+# (`python -m execnet worker --protocol-... --config ...`), so sys.argv
+# legitimately differs.  Needs an xdist-side update, not an execnet fix.
+testing/test_remote.py::test_remote_inner_argv

.pre-commit-config.yaml

@@ -31,5 +31,6 @@ repos:
     -   id: mypy
         additional_dependencies:
           - pytest
+          - trio
+          - hypothesis
           - types-pywin32
-          - types-gevent