From 069dc56359dfeb13f63e45aa1589692778c4f05a Mon Sep 17 00:00:00 2001 From: Maryna Shevchuk Date: Wed, 8 Jul 2026 10:15:55 -0700 Subject: [PATCH 1/4] add auth submetrics --- configs/agents/airline_agent.yaml | 2 +- src/eva/__init__.py | 4 +- .../diagnostic/authentication_success.py | 49 ++++ tests/fixtures/metric_signatures.json | 2 +- .../metrics/test_authentication_success.py | 210 ++++++++++++++++++ 5 files changed, 263 insertions(+), 4 deletions(-) diff --git a/configs/agents/airline_agent.yaml b/configs/agents/airline_agent.yaml index 1992f055..05f3d9c8 100644 --- a/configs/agents/airline_agent.yaml +++ b/configs/agents/airline_agent.yaml @@ -192,7 +192,7 @@ tools: - id: get_reservation name: get_reservation description: Retrieve flight reservation using confirmation number and passenger last name. This is typically the first tool called to authenticate the caller and load their flight numbers. - tool_type: read + tool_type: auth domain: flight required_parameters: - name: confirmation_number diff --git a/src/eva/__init__.py b/src/eva/__init__.py index eb2eaa6a..1995823a 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,8 +7,8 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.1" +simulation_version = "2.0.2" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.2.0" +metrics_version = "2.2.1" diff --git a/src/eva/metrics/diagnostic/authentication_success.py b/src/eva/metrics/diagnostic/authentication_success.py index 0662231b..96f2eba8 100644 --- a/src/eva/metrics/diagnostic/authentication_success.py +++ b/src/eva/metrics/diagnostic/authentication_success.py @@ -4,6 +4,8 @@ final evaluation scores. """ +from collections import Counter + from eva.metrics.base import CodeMetric, MetricContext from eva.metrics.registry import register_metric from eva.models.results import MetricScore @@ -68,6 +70,10 @@ async def compute(self, context: MetricContext) -> MetricScore: mismatches = compute_session_auth_mismatches(context.expected_scenario_db, context.final_scenario_db) success = len(mismatches) == 0 + sub_metrics = _build_authentication_success_sub_metrics( + self.name, context.tool_responses, context.agent_tools, success + ) + return MetricScore( name=self.name, score=1.0 if success else 0.0, @@ -80,7 +86,50 @@ async def compute(self, context: MetricContext) -> MetricScore: if success else f"Session mismatch on keys: {list(mismatches)}", }, + sub_metrics=sub_metrics or None, ) except Exception as e: return self._handle_error(e, context) + + +def _build_authentication_success_sub_metrics( + parent_name: str, + tool_responses: list[dict], + agent_tools: list[dict], + auth_success: bool, +) -> dict[str, MetricScore]: + """Build sub-metrics for authentication tool call behaviour.""" + auth_tool_names = {t.get("name") for t in agent_tools if t.get("tool_type") == "auth" and t.get("name")} + auth_calls = [r for r in tool_responses if r.get("tool_name") in auth_tool_names] + if not auth_calls: + return {} + + sub_metrics: dict[str, MetricScore] = {} + + counts_per_tool = Counter(r.get("tool_name") for r in auth_calls) + avg_calls = sum(counts_per_tool.values()) / len(counts_per_tool) if counts_per_tool else 0.0 + + if auth_success: + first_try_success = all(count == 1 for count in counts_per_tool.values()) + sub_metrics["auth_first_try_success"] = MetricScore( + name=f"{parent_name}.auth_first_try_success", + score=1.0 if first_try_success else 0.0, + normalized_score=1.0 if first_try_success else 0.0, + details={"calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + ) + sub_metrics["auth_num_calls"] = MetricScore( + name=f"{parent_name}.auth_num_calls", + score=round(avg_calls, 4), + normalized_score=None, + details={"calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + ) + else: + sub_metrics["auth_num_calls_on_failure"] = MetricScore( + name=f"{parent_name}.auth_num_calls_on_failure", + score=round(avg_calls, 4), + normalized_score=None, + details={"calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + ) + + return sub_metrics diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 3b95ed18..a356d02e 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -2,7 +2,7 @@ "AuthenticationSuccessMetric": { "name": "authentication_success", "prompt_hash": null, - "source_hash": "ecce31785d0d", + "source_hash": "5021c8eaa095", "version": "v0.1" }, "ConcisenessJudgeMetric": { diff --git a/tests/unit/metrics/test_authentication_success.py b/tests/unit/metrics/test_authentication_success.py index 5d5cecdc..2ce1b16c 100644 --- a/tests/unit/metrics/test_authentication_success.py +++ b/tests/unit/metrics/test_authentication_success.py @@ -6,6 +6,9 @@ from .conftest import make_metric_context +SUCCESS_RESPONSE = {"status": "success"} +FAILURE_RESPONSE = {"status": "error"} + @pytest.fixture def metric(): @@ -92,3 +95,210 @@ async def test_no_expected_session(metric): assert result.score is None assert result.skipped is True assert "skipping" in result.details["reason"] + + +@pytest.mark.asyncio +async def test_auth_first_try_success_true(metric): + """Auth succeeding with every auth tool called exactly once should score auth_first_try_success=1.0.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[{"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_first_try_success"] + assert sub.score == 1.0 + assert sub.normalized_score == 1.0 + assert sub.skipped is False + + +@pytest.mark.asyncio +async def test_auth_first_try_success_false(metric): + """Auth succeeding but only after a retry on an auth tool should score auth_first_try_success=0.0.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_first_try_success"] + assert sub.score == 0.0 + assert sub.normalized_score == 0.0 + assert sub.skipped is False + + +@pytest.mark.asyncio +async def test_auth_first_try_success_multiple_tools_one_retried(metric): + """Auth succeeding with one of several auth tools needing a retry should score auth_first_try_success=0.0.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[ + {"name": "get_reservation", "tool_type": "auth"}, + {"name": "verify", "tool_type": "auth"}, + ], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": SUCCESS_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_first_try_success"] + assert sub.score == 0.0 + assert sub.normalized_score == 0.0 + assert sub.skipped is False + + +@pytest.mark.asyncio +async def test_auth_num_calls_success_single_tool(metric): + """Auth succeeding with a single auth tool called once should score auth_num_calls=1.0.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[{"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_num_calls"] + assert sub.score == 1.0 + assert sub.normalized_score is None + assert sub.skipped is False + assert sub.details["calls_per_tool"] == {"get_reservation": 1} + assert sub.details["num_auth_tools"] == 1 + + assert "auth_num_calls_on_failure" not in result.sub_metrics + + +@pytest.mark.asyncio +async def test_auth_num_calls_success_multiple_calls(metric): + """Auth succeeding after multiple calls to the same tool should average those calls.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_num_calls"] + assert sub.score == 3.0 + assert sub.normalized_score is None + assert sub.skipped is False + assert sub.details["calls_per_tool"] == {"get_reservation": 3} + assert sub.details["num_auth_tools"] == 1 + + assert "auth_num_calls_on_failure" not in result.sub_metrics + + +@pytest.mark.asyncio +async def test_auth_num_calls_success_multiple_tools(metric): + """Auth succeeding with multiple distinct auth tools should average calls across tools.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[ + {"name": "get_reservation", "tool_type": "auth"}, + {"name": "verify", "tool_type": "auth"}, + ], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": SUCCESS_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_num_calls"] + assert sub.score == 3.0 + assert sub.normalized_score is None + assert sub.skipped is False + assert sub.details["calls_per_tool"] == {"get_reservation": 2, "verify": 4} + assert sub.details["num_auth_tools"] == 2 + + assert "auth_num_calls_on_failure" not in result.sub_metrics + + +@pytest.mark.asyncio +async def test_auth_num_calls_on_failure_single_tool(metric): + """Auth failing with a single auth tool called once should score auth_num_calls_on_failure=1.0.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "WRONG1"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[{"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_num_calls_on_failure"] + assert sub.score == 1.0 + assert sub.normalized_score is None + assert sub.skipped is False + assert sub.details["calls_per_tool"] == {"get_reservation": 1} + assert sub.details["num_auth_tools"] == 1 + + assert "auth_num_calls" not in result.sub_metrics + assert "auth_first_try_success" not in result.sub_metrics + + +@pytest.mark.asyncio +async def test_auth_num_calls_on_failure_multiple_calls(metric): + """Auth failing after multiple calls to the same tool should average those calls.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "WRONG1"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["auth_num_calls_on_failure"] + assert sub.score == 3.0 + assert sub.normalized_score is None + assert sub.skipped is False + assert sub.details["calls_per_tool"] == {"get_reservation": 3} + assert sub.details["num_auth_tools"] == 1 + + assert "auth_num_calls" not in result.sub_metrics + assert "auth_first_try_success" not in result.sub_metrics + + +@pytest.mark.asyncio +async def test_no_auth_tools_no_sub_metrics(metric): + """No auth-type tools on the agent should produce no sub-metrics.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_flight_status", "tool_type": "read"}], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is None From 57fe80120f27164268e481632ac47a2c44cdd84e Mon Sep 17 00:00:00 2001 From: Maryna Shevchuk Date: Wed, 22 Jul 2026 13:44:50 -0700 Subject: [PATCH 2/4] apply review changes --- src/eva/__init__.py | 4 +- .../diagnostic/authentication_success.py | 27 ++++++------ .../metrics/test_authentication_success.py | 44 +++++++++---------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 1995823a..0f7bc3b9 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,8 +7,8 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.2" +simulation_version = "2.0.3" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.2.1" +metrics_version = "2.2.2" diff --git a/src/eva/metrics/diagnostic/authentication_success.py b/src/eva/metrics/diagnostic/authentication_success.py index 96f2eba8..9d958a9e 100644 --- a/src/eva/metrics/diagnostic/authentication_success.py +++ b/src/eva/metrics/diagnostic/authentication_success.py @@ -100,15 +100,16 @@ def _build_authentication_success_sub_metrics( auth_success: bool, ) -> dict[str, MetricScore]: """Build sub-metrics for authentication tool call behaviour.""" - auth_tool_names = {t.get("name") for t in agent_tools if t.get("tool_type") == "auth" and t.get("name")} - auth_calls = [r for r in tool_responses if r.get("tool_name") in auth_tool_names] - if not auth_calls: - return {} + auth_tool_names = {t["name"] for t in agent_tools if t.get("tool_type") == "auth"} + + counts_per_tool = Counter(r.get("tool_name") for r in tool_responses if r.get("tool_name") in auth_tool_names) sub_metrics: dict[str, MetricScore] = {} - counts_per_tool = Counter(r.get("tool_name") for r in auth_calls) - avg_calls = sum(counts_per_tool.values()) / len(counts_per_tool) if counts_per_tool else 0.0 + if not counts_per_tool: + return sub_metrics + + avg_calls = sum(counts_per_tool.values()) / len(counts_per_tool) if auth_success: first_try_success = all(count == 1 for count in counts_per_tool.values()) @@ -116,20 +117,20 @@ def _build_authentication_success_sub_metrics( name=f"{parent_name}.auth_first_try_success", score=1.0 if first_try_success else 0.0, normalized_score=1.0 if first_try_success else 0.0, - details={"calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, ) - sub_metrics["auth_num_calls"] = MetricScore( - name=f"{parent_name}.auth_num_calls", + sub_metrics["num_auth_calls"] = MetricScore( + name=f"{parent_name}.num_auth_calls", score=round(avg_calls, 4), normalized_score=None, - details={"calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, ) else: - sub_metrics["auth_num_calls_on_failure"] = MetricScore( - name=f"{parent_name}.auth_num_calls_on_failure", + sub_metrics["num_auth_calls_on_failure"] = MetricScore( + name=f"{parent_name}.num_auth_calls_on_failure", score=round(avg_calls, 4), normalized_score=None, - details={"calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, ) return sub_metrics diff --git a/tests/unit/metrics/test_authentication_success.py b/tests/unit/metrics/test_authentication_success.py index 2ce1b16c..6dd57bbd 100644 --- a/tests/unit/metrics/test_authentication_success.py +++ b/tests/unit/metrics/test_authentication_success.py @@ -162,8 +162,8 @@ async def test_auth_first_try_success_multiple_tools_one_retried(metric): @pytest.mark.asyncio -async def test_auth_num_calls_success_single_tool(metric): - """Auth succeeding with a single auth tool called once should score auth_num_calls=1.0.""" +async def test_num_auth_calls_success_single_tool(metric): + """Auth succeeding with a single auth tool called once should score num_auth_calls=1.0.""" ctx = make_metric_context( expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, final_scenario_db={"session": {"confirmation_number": "ABC123"}}, @@ -173,18 +173,18 @@ async def test_auth_num_calls_success_single_tool(metric): result = await metric.compute(ctx) assert result.sub_metrics is not None - sub = result.sub_metrics["auth_num_calls"] + sub = result.sub_metrics["num_auth_calls"] assert sub.score == 1.0 assert sub.normalized_score is None assert sub.skipped is False - assert sub.details["calls_per_tool"] == {"get_reservation": 1} + assert sub.details["num_auth_calls_per_tool"] == {"get_reservation": 1} assert sub.details["num_auth_tools"] == 1 - assert "auth_num_calls_on_failure" not in result.sub_metrics + assert "num_auth_calls_on_failure" not in result.sub_metrics @pytest.mark.asyncio -async def test_auth_num_calls_success_multiple_calls(metric): +async def test_num_auth_calls_success_multiple_calls(metric): """Auth succeeding after multiple calls to the same tool should average those calls.""" ctx = make_metric_context( expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, @@ -199,18 +199,18 @@ async def test_auth_num_calls_success_multiple_calls(metric): result = await metric.compute(ctx) assert result.sub_metrics is not None - sub = result.sub_metrics["auth_num_calls"] + sub = result.sub_metrics["num_auth_calls"] assert sub.score == 3.0 assert sub.normalized_score is None assert sub.skipped is False - assert sub.details["calls_per_tool"] == {"get_reservation": 3} + assert sub.details["num_auth_calls_per_tool"] == {"get_reservation": 3} assert sub.details["num_auth_tools"] == 1 - assert "auth_num_calls_on_failure" not in result.sub_metrics + assert "num_auth_calls_on_failure" not in result.sub_metrics @pytest.mark.asyncio -async def test_auth_num_calls_success_multiple_tools(metric): +async def test_num_auth_calls_success_multiple_tools(metric): """Auth succeeding with multiple distinct auth tools should average calls across tools.""" ctx = make_metric_context( expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, @@ -231,19 +231,19 @@ async def test_auth_num_calls_success_multiple_tools(metric): result = await metric.compute(ctx) assert result.sub_metrics is not None - sub = result.sub_metrics["auth_num_calls"] + sub = result.sub_metrics["num_auth_calls"] assert sub.score == 3.0 assert sub.normalized_score is None assert sub.skipped is False - assert sub.details["calls_per_tool"] == {"get_reservation": 2, "verify": 4} + assert sub.details["num_auth_calls_per_tool"] == {"get_reservation": 2, "verify": 4} assert sub.details["num_auth_tools"] == 2 - assert "auth_num_calls_on_failure" not in result.sub_metrics + assert "num_auth_calls_on_failure" not in result.sub_metrics @pytest.mark.asyncio -async def test_auth_num_calls_on_failure_single_tool(metric): - """Auth failing with a single auth tool called once should score auth_num_calls_on_failure=1.0.""" +async def test_num_auth_calls_on_failure_single_tool(metric): + """Auth failing with a single auth tool called once should score num_auth_calls_on_failure=1.0.""" ctx = make_metric_context( expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, final_scenario_db={"session": {"confirmation_number": "WRONG1"}}, @@ -253,19 +253,19 @@ async def test_auth_num_calls_on_failure_single_tool(metric): result = await metric.compute(ctx) assert result.sub_metrics is not None - sub = result.sub_metrics["auth_num_calls_on_failure"] + sub = result.sub_metrics["num_auth_calls_on_failure"] assert sub.score == 1.0 assert sub.normalized_score is None assert sub.skipped is False - assert sub.details["calls_per_tool"] == {"get_reservation": 1} + assert sub.details["num_auth_calls_per_tool"] == {"get_reservation": 1} assert sub.details["num_auth_tools"] == 1 - assert "auth_num_calls" not in result.sub_metrics + assert "num_auth_calls" not in result.sub_metrics assert "auth_first_try_success" not in result.sub_metrics @pytest.mark.asyncio -async def test_auth_num_calls_on_failure_multiple_calls(metric): +async def test_num_auth_calls_on_failure_multiple_calls(metric): """Auth failing after multiple calls to the same tool should average those calls.""" ctx = make_metric_context( expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, @@ -280,14 +280,14 @@ async def test_auth_num_calls_on_failure_multiple_calls(metric): result = await metric.compute(ctx) assert result.sub_metrics is not None - sub = result.sub_metrics["auth_num_calls_on_failure"] + sub = result.sub_metrics["num_auth_calls_on_failure"] assert sub.score == 3.0 assert sub.normalized_score is None assert sub.skipped is False - assert sub.details["calls_per_tool"] == {"get_reservation": 3} + assert sub.details["num_auth_calls_per_tool"] == {"get_reservation": 3} assert sub.details["num_auth_tools"] == 1 - assert "auth_num_calls" not in result.sub_metrics + assert "num_auth_calls" not in result.sub_metrics assert "auth_first_try_success" not in result.sub_metrics From 84ef7811333bca70fee77a96249f22422ff61c28 Mon Sep 17 00:00:00 2001 From: Maryna Shevchuk Date: Fri, 24 Jul 2026 14:01:03 -0700 Subject: [PATCH 3/4] add authentication_success_rate submetric --- src/eva/__init__.py | 4 +- .../diagnostic/authentication_success.py | 7 ++ .../metrics/test_authentication_success.py | 85 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 0f7bc3b9..1995823a 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,8 +7,8 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.3" +simulation_version = "2.0.2" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.2.2" +metrics_version = "2.2.1" diff --git a/src/eva/metrics/diagnostic/authentication_success.py b/src/eva/metrics/diagnostic/authentication_success.py index 9d958a9e..1749cab5 100644 --- a/src/eva/metrics/diagnostic/authentication_success.py +++ b/src/eva/metrics/diagnostic/authentication_success.py @@ -125,6 +125,13 @@ def _build_authentication_success_sub_metrics( normalized_score=None, details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, ) + success_rate = len(counts_per_tool) / sum(counts_per_tool.values()) + sub_metrics["authentication_success_rate"] = MetricScore( + name=f"{parent_name}.authentication_success_rate", + score=round(success_rate, 4), + normalized_score=round(success_rate, 4), + details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)}, + ) else: sub_metrics["num_auth_calls_on_failure"] = MetricScore( name=f"{parent_name}.num_auth_calls_on_failure", diff --git a/tests/unit/metrics/test_authentication_success.py b/tests/unit/metrics/test_authentication_success.py index 6dd57bbd..3e56ce58 100644 --- a/tests/unit/metrics/test_authentication_success.py +++ b/tests/unit/metrics/test_authentication_success.py @@ -241,6 +241,91 @@ async def test_num_auth_calls_success_multiple_tools(metric): assert "num_auth_calls_on_failure" not in result.sub_metrics +@pytest.mark.asyncio +async def test_authentication_success_rate_single_tool_first_try(metric): + """Auth succeeding with a single tool called once should score authentication_success_rate=1.0.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[{"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["authentication_success_rate"] + assert sub.score == 1.0 + assert sub.normalized_score == 1.0 + assert sub.skipped is False + assert sub.details["num_auth_calls_per_tool"] == {"get_reservation": 1} + assert sub.details["num_auth_tools"] == 1 + + +@pytest.mark.asyncio +async def test_authentication_success_rate_single_tool_with_retries(metric): + """Auth succeeding after 3 calls to one tool should score authentication_success_rate=1/3.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["authentication_success_rate"] + assert sub.score == pytest.approx(0.3333, abs=1e-4) + assert sub.normalized_score == pytest.approx(0.3333, abs=1e-4) + assert sub.skipped is False + + +@pytest.mark.asyncio +async def test_authentication_success_rate_multiple_tools(metric): + """Auth succeeding across 2 tools with 6 total calls should score authentication_success_rate=2/6.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "ABC123"}}, + agent_tools=[ + {"name": "get_reservation", "tool_type": "auth"}, + {"name": "verify", "tool_type": "auth"}, + ], + tool_responses=[ + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + {"tool_name": "get_reservation", "tool_response": SUCCESS_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": FAILURE_RESPONSE}, + {"tool_name": "verify", "tool_response": SUCCESS_RESPONSE}, + ], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + sub = result.sub_metrics["authentication_success_rate"] + assert sub.score == pytest.approx(0.3333, abs=1e-4) + assert sub.normalized_score == pytest.approx(0.3333, abs=1e-4) + assert sub.skipped is False + + +@pytest.mark.asyncio +async def test_authentication_success_rate_absent_on_failure(metric): + """Auth failing should not produce an authentication_success_rate sub-metric.""" + ctx = make_metric_context( + expected_scenario_db={"session": {"confirmation_number": "ABC123"}}, + final_scenario_db={"session": {"confirmation_number": "WRONG1"}}, + agent_tools=[{"name": "get_reservation", "tool_type": "auth"}], + tool_responses=[{"tool_name": "get_reservation", "tool_response": FAILURE_RESPONSE}], + ) + result = await metric.compute(ctx) + + assert result.sub_metrics is not None + assert "authentication_success_rate" not in result.sub_metrics + + @pytest.mark.asyncio async def test_num_auth_calls_on_failure_single_tool(metric): """Auth failing with a single auth tool called once should score num_auth_calls_on_failure=1.0.""" From 0f1c4924f485d5ae8a8d9af8a9c087f403acb586 Mon Sep 17 00:00:00 2001 From: maryna-shevchuk Date: Tue, 28 Jul 2026 11:22:57 -0700 Subject: [PATCH 4/4] Update versions Co-authored-by: Joseph Marinier --- src/eva/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 1995823a..23260438 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.2" +simulation_version = "2.0.1" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor).