Skip to content

Rerun only the failed tests on CI retries #3178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
antonwolfy wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Rerun only the failed tests on CI retries #3178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b3144ad
Rerun only the failed tests on CI retries
antonwolfy Nov 8, 2026
103bcde
Continue past collection errors in serial runs a --lf retry follows
antonwolfy Nov 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .github/workflows/check-onemath.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ env:
test-pkg-env: 'environments/test.yml'
test-env-name: 'test_onemath'
rerun-tests-on-failure: 'true'
# A retry reruns only the failed tests (--lf), or all of them after a timeout kill
# Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests
rerun-tests-max-attempts: 2
rerun-tests-timeout: 30
rerun-tensor-tests-timeout: 10
Expand Down Expand Up @@ -188,7 +190,7 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs dpnp.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests
env:
SKIP_TENSOR_TESTS: 1
SYCL_CACHE_PERSISTENT: 1
Expand All @@ -206,7 +208,7 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs dpnp.tests.tensor
python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests.tensor
env:
SYCL_CACHE_PERSISTENT: 1

Expand Down Expand Up @@ -324,7 +326,7 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs dpnp.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests
env:
SKIP_TENSOR_TESTS: 1
SYCL_CACHE_PERSISTENT: 1
Expand All @@ -342,6 +344,6 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs dpnp.tests.tensor
python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests.tensor
env:
SYCL_CACHE_PERSISTENT: 1
8 changes: 5 additions & 3 deletions .github/workflows/conda-build-test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ env:
fetch-depth: 1
python-ver-test-all-dtypes: '3.14'
test-env-name: 'test'
# A retry reruns only the failed tests (--lf), or all of them after a timeout kill
# Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests
rerun-tests-max-attempts: 2
rerun-tests-timeout: 45
rerun-tensor-tests-timeout: 15
Expand Down Expand Up @@ -282,9 +284,9 @@ jobs:

if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then
export DPNP_TEST_ALL_INT_TYPES=1
python -m pytest -ra --pyargs ${{ env.module-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.module-name }}.tests
else
python -m pytest -n auto -ra --pyargs ${{ env.module-name }}.tests
python -m pytest -n auto -ra --lf --pyargs ${{ env.module-name }}.tests
fi

- name: Run tensor tests
Expand All @@ -300,4 +302,4 @@ jobs:
. "$CONDA_UNIX/etc/profile.d/conda.sh"
conda activate ${{ env.test-env-name }}

python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
14 changes: 8 additions & 6 deletions .github/workflows/conda-package.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ env:
python-ver-test-all-dtypes: '3.14'
test-env-name: 'test'
rerun-tests-on-failure: 'true'
# A retry reruns only the failed tests (--lf), or all of them after a timeout kill
# Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests
rerun-tests-max-attempts: 2
rerun-tests-timeout: 45
rerun-tensor-tests-timeout: 15
Expand Down Expand Up @@ -295,9 +297,9 @@ jobs:

if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then
export DPNP_TEST_ALL_INT_TYPES=1
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
else
python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests
fi

- name: Run tensor tests
Expand All @@ -318,7 +320,7 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor

test_examples_linux:
name: Test examples
Expand Down Expand Up @@ -603,9 +605,9 @@ jobs:
$env:SKIP_TENSOR_TESTS=1
if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) {
$env:DPNP_TEST_ALL_INT_TYPES=1
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
} else {
python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests
}

- name: Run tensor tests
Expand All @@ -624,7 +626,7 @@ jobs:
retry_on: any
shell: pwsh
command: |
python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor

upload:
name: Upload
Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/cron-run-tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ env:
channels-list: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels'
test-env-name: test
test-packages: 'pytest scipy'
# A retry reruns only the failed tests (--lf), or all of them after a timeout kill
# Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests
rerun-tests-max-attempts: 2
rerun-tests-timeout: 20

Expand Down Expand Up @@ -138,7 +140,7 @@ jobs:
id: run_tests
continue-on-error: true
run: |
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
env:
SYCL_CACHE_PERSISTENT: 1

Expand All @@ -155,7 +157,7 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
env:
SYCL_CACHE_PERSISTENT: 1

Expand All @@ -168,6 +170,6 @@ jobs:
max_attempts: ${{ env.rerun-tests-max-attempts }}
retry_on: any
command: |
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
env:
SYCL_CACHE_PERSISTENT: 1
Loading