Add auth submetrics

configs/agents/airline_agent.yaml

@@ -192,7 +192,7 @@ tools:
 - id: get_reservation
   name: get_reservation
   description: Retrieve flight reservation using confirmation number and passenger last name. This is typically the first tool called to authenticate the caller and load their flight numbers.
-  tool_type: read
+  tool_type: auth
   domain: flight
   required_parameters:
   - name: confirmation_number

src/eva/__init__.py

@@ -11,4 +11,4 @@
 
 # Bump metrics_version when changes affect metric computation (metrics code,
 # judge prompts, pricing tables, postprocessor).
-metrics_version = "2.2.0"
+metrics_version = "2.2.1"

src/eva/metrics/diagnostic/authentication_success.py

@@ -4,6 +4,8 @@
 final evaluation scores.
 """
 
+from collections import Counter
+
 from eva.metrics.base import CodeMetric, MetricContext
 from eva.metrics.registry import register_metric
 from eva.models.results import MetricScore
@@ -68,6 +70,10 @@ async def compute(self, context: MetricContext) -> MetricScore:
             mismatches = compute_session_auth_mismatches(context.expected_scenario_db, context.final_scenario_db)
             success = len(mismatches) == 0
 
+            sub_metrics = _build_authentication_success_sub_metrics(
+                self.name, context.tool_responses, context.agent_tools, success
+            )
+
             return MetricScore(
                 name=self.name,
                 score=1.0 if success else 0.0,
@@ -80,7 +86,58 @@ async def compute(self, context: MetricContext) -> MetricScore:
                     if success
                     else f"Session mismatch on keys: {list(mismatches)}",
                 },
+                sub_metrics=sub_metrics or None,
             )
 
         except Exception as e:
             return self._handle_error(e, context)
+
+
+def _build_authentication_success_sub_metrics(
+    parent_name: str,
+    tool_responses: list[dict],
+    agent_tools: list[dict],
+    auth_success: bool,
+) -> dict[str, MetricScore]:
+    """Build sub-metrics for authentication tool call behaviour."""
+    auth_tool_names = {t["name"] for t in agent_tools if t.get("tool_type") == "auth"}
+
+    counts_per_tool = Counter(r.get("tool_name") for r in tool_responses if r.get("tool_name") in auth_tool_names)
+
+    sub_metrics: dict[str, MetricScore] = {}
+
+    if not counts_per_tool:
+        return sub_metrics
+
+    avg_calls = sum(counts_per_tool.values()) / len(counts_per_tool)
+
+    if auth_success:
+        first_try_success = all(count == 1 for count in counts_per_tool.values())
+        sub_metrics["auth_first_try_success"] = MetricScore(
+            name=f"{parent_name}.auth_first_try_success",
+            score=1.0 if first_try_success else 0.0,
+            normalized_score=1.0 if first_try_success else 0.0,
+            details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)},
+        )
+        sub_metrics["num_auth_calls"] = MetricScore(
+            name=f"{parent_name}.num_auth_calls",
+            score=round(avg_calls, 4),
+            normalized_score=None,
+            details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)},
+        )
+        success_rate = len(counts_per_tool) / sum(counts_per_tool.values())
+        sub_metrics["authentication_success_rate"] = MetricScore(
+            name=f"{parent_name}.authentication_success_rate",
+            score=round(success_rate, 4),
+            normalized_score=round(success_rate, 4),
+            details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)},
+        )
+    else:
+        sub_metrics["num_auth_calls_on_failure"] = MetricScore(
+            name=f"{parent_name}.num_auth_calls_on_failure",
+            score=round(avg_calls, 4),
+            normalized_score=None,
+            details={"num_auth_calls_per_tool": counts_per_tool, "num_auth_tools": len(counts_per_tool)},
+        )
+
+    return sub_metrics

tests/fixtures/metric_signatures.json

@@ -2,7 +2,7 @@
   "AuthenticationSuccessMetric": {
     "name": "authentication_success",
     "prompt_hash": null,
-    "source_hash": "ecce31785d0d",
+    "source_hash": "5021c8eaa095",
     "version": "v0.1"
   },
   "ConcisenessJudgeMetric": {