Merge pull request #34 from webknjaz/features/xfail-support

Add support for xfail

Author: RonnyPfannschmidt

src/pytest_forked/__init__.py

@@ -1,4 +1,6 @@
 import os
+import warnings
+
 import py
 # we know this bit is bad, but we cant help it with the current pytest setup
 from _pytest import runner
@@ -88,4 +90,24 @@ def report_process_crash(item, result):
         rep.sections.append(("captured stdout", result.out))
     if result.err:
         rep.sections.append(("captured stderr", result.err))
+
+    xfail_marker = item.get_closest_marker('xfail')
+    if not xfail_marker:
+        return rep
+
+    rep.outcome = "skipped"
+    rep.wasxfail = (
+        "reason: {xfail_reason}; "
+        "pytest-forked reason: {crash_info}".
+        format(
+            xfail_reason=xfail_marker.kwargs['reason'],
+            crash_info=info,
+        )
+    )
+    warnings.warn(
+        'pytest-forked xfail support is incomplete at the moment and may '
+        'output a misleading reason message',
+        RuntimeWarning,
+    )
+
     return rep

testing/test_xfail_behavior.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+"""Tests for xfail support."""
+import os
+import signal
+
+import pytest
+
+IS_PYTEST4_PLUS = int(pytest.__version__[0]) >= 4  # noqa: WPS609
+FAILED_WORD = 'FAILED' if IS_PYTEST4_PLUS else 'FAIL'
+
+pytestmark = pytest.mark.skipif(  # pylint: disable=invalid-name
+    not hasattr(os, 'fork'),  # noqa: WPS421
+    reason='os.fork required',
+)
+
+
+@pytest.mark.parametrize(
+    ('is_crashing', 'is_strict'),
+    (
+        pytest.param(True, True, id='strict xfail'),
+        pytest.param(False, True, id='strict xpass'),
+        pytest.param(True, False, id='non-strict xfail'),
+        pytest.param(False, False, id='non-strict xpass'),
+    ),
+)
+def test_xfail(is_crashing, is_strict, testdir):
+    """Test xfail/xpass/strict permutations."""
+    # pylint: disable=possibly-unused-variable
+    sig_num = signal.SIGTERM.numerator
+
+    test_func_body = (
+        'os.kill(os.getpid(), signal.SIGTERM)'
+        if is_crashing
+        else 'assert True'
+    )
+
+    if is_crashing:
+        # marked xfailed and crashing, no matter strict or not
+        expected_letter = 'x'  # XFAILED
+        expected_lowercase = 'xfailed'
+        expected_word = 'XFAIL'
+    elif is_strict:
+        # strict and not failing as expected should cause failure
+        expected_letter = 'F'  # FAILED
+        expected_lowercase = 'failed'
+        expected_word = FAILED_WORD
+    elif not is_strict:
+        # non-strict and not failing as expected should cause xpass
+        expected_letter = 'X'  # XPASS
+        expected_lowercase = 'xpassed'
+        expected_word = 'XPASS'
+
+    session_start_title = '*==== test session starts ====*'
+    loaded_pytest_plugins = 'plugins: forked*'
+    collected_tests_num = 'collected 1 item'
+    expected_progress = 'test_xfail.py {expected_letter!s}'.format(**locals())
+    failures_title = '*==== FAILURES ====*'
+    failures_test_name = '*____ test_function ____*'
+    failures_test_reason = '[XPASS(strict)] The process gets terminated'
+    short_test_summary_title = '*==== short test summary info ====*'
+    short_test_summary = (
+        '{expected_word!s} test_xfail.py::test_function'.
+        format(**locals())
+    )
+    if expected_lowercase == 'xpassed':
+        # XPASS wouldn't have the crash message from
+        # pytest-forked because the crash doesn't happen
+        short_test_summary = ' '.join((
+            short_test_summary, 'The process gets terminated',
+        ))
+    reason_string = (
+        '  reason: The process gets terminated; '
+        'pytest-forked reason: '
+        '*:*: running the test CRASHED with signal {sig_num:d}'.
+        format(**locals())
+    )
+    total_summary_line = (
+        '*==== 1 {expected_lowercase!s} in 0.*s* ====*'.
+        format(**locals())
+    )
+
+    expected_lines = (
+        session_start_title,
+        loaded_pytest_plugins,
+        collected_tests_num,
+        expected_progress,
+    )
+    if expected_word == FAILED_WORD:
+        # XPASS(strict)
+        expected_lines += (
+            failures_title,
+            failures_test_name,
+            failures_test_reason,
+        )
+    expected_lines += (
+        short_test_summary_title,
+        short_test_summary,
+    )
+    if expected_lowercase == 'xpassed' and expected_word == FAILED_WORD:
+        # XPASS(strict)
+        expected_lines += (
+            reason_string,
+        )
+    expected_lines += (
+        total_summary_line,
+    )
+
+    test_module = testdir.makepyfile(
+        """
+        import os
+        import signal
+
+        import pytest
+
+        @pytest.mark.xfail(
+            reason='The process gets terminated',
+            strict={is_strict!s},
+        )
+        @pytest.mark.forked
+        def test_function():
+            {test_func_body!s}
+        """.
+        format(**locals())
+    )
+
+    pytest_run_result = testdir.runpytest(
+        test_module,
+        '-ra',
+        '-p', 'no:warnings',  # the current implementation emits RuntimeWarning
+    )
+    pytest_run_result.stdout.fnmatch_lines(expected_lines)