From 79e906bca9eb1583f8b23735ac8d010b6c237cf4 Mon Sep 17 00:00:00 2001 From: Katrina Date: Wed, 22 Jul 2026 12:03:13 -0400 Subject: [PATCH 1/6] add pretool speech to turn taking metrics --- docs/metrics/turn_taking.md | 22 +- src/eva/metrics/experience/turn_taking.py | 77 +++++- tests/fixtures/metric_signatures.json | 4 +- tests/unit/metrics/test_turn_taking.py | 275 ++++++++++++++++++++++ 4 files changed, 374 insertions(+), 4 deletions(-) diff --git a/docs/metrics/turn_taking.md b/docs/metrics/turn_taking.md index 468a1422..8bc7377f 100644 --- a/docs/metrics/turn_taking.md +++ b/docs/metrics/turn_taking.md @@ -19,6 +19,7 @@ Code-based metric (no LLM) that scores each user→assistant transition on a con - `latency_assistant_turns` — per-turn latency (`first_asst_start - last_user_end`) in seconds. Drives the latency curve. - `assistant_interrupted_turns` / `user_interrupted_turns` — turn-level interruption flags set by the processor. - `conversation_trace` — used to detect which turns contain a tool call (`type == "tool_call"`). Tool-call turns get a more lenient upper end of the latency curve and a higher `late` threshold, since tool execution adds inherent latency. +- `output_dir` — read directly (not via `conversation_trace`) to load `audit_log.json` for the `pretoolspeech_rate` sub-metric (see below). ## Per-turn Score @@ -189,7 +190,26 @@ Sub-metric aggregation reads the raw per-turn values from `per_turn_evidence` ra | `user_interruption.mean_yield_ms` | no | rate > 0 | Arithmetic mean of `yield_ms` across user-interrupt turns. | | `user_interruption.mean_yield_score` | yes | rate > 0 | Mean of the per-turn yield scores that feed the main score. | -Rate sub-metrics are emitted as `normalized_score` (they already live on `[0, 1]`). Raw-ms sub-metrics have `normalized_score = None` so they don't corrupt cross-metric averages. +**Turn-end fallback nudges** + +| Key | Normalized? | When present | Meaning | +| --- | --- | --- | --- | +| `fallback_nudge.rate` | yes | at least one assistant turn | `len(fallback_turn_ids) / total_assistant_turns`. Denominator is all assistant turns (`audio_timestamps_assistant_turns`), not the evaluable-turn count, since a fallback nudge produces an assistant turn with no paired user turn and so is never in the evaluable set. | +| `fallback_nudge.count` | no (raw count) | always, but `score=None` when zero | Total number of fallback-nudge turns. `None` on clean runs so cross-record aggregates exclude them rather than averaging in zeros. | + +**Pre-tool-speech lead-in rate** + +| Key | Normalized? | When present | Meaning | +| --- | --- | --- | --- | +| `pretoolspeech_rate` | yes | at least one tool-call group and `audit_log.json` is readable | Fraction of tool-call groups preceded by a non-empty assistant utterance since the previous group ended (or since the conversation started). Measures how often the agent gives a spoken lead-in (per `agent.pre_tool_speech` in `configs/prompts/simulation.yaml`) before invoking a tool. | + +Computed by `_compute_pre_tool_speech_groups`, which reads `audit_log.json` directly from `context.output_dir` **instead of** `conversation_trace`. A "tool-call group" is a maximal contiguous run of `tool_call`/`tool_response` entries in the raw audit-log transcript (sorted by `timestamp`) — consecutive tool calls fired off one lead-in (e.g. two tools called back-to-back with no intervening speech) count as a single group, matching the prompt's "one lead-in per batch" instruction. Assistant speech only counts if it has non-empty, non-whitespace content; a `user` event resets the "spoke since last group" flag so a lead-in from a previous turn can't be credited to a later turn's tool calls. Event types other than `assistant`/`user`/`tool_call`/`tool_response` (e.g. `llm_call`) are ignored rather than treated as group-breaking — see the regression test guarding this. + +**Why not `conversation_trace`?** For S2S, `conversation_trace` assistant entries come from the user simulator's STT transcription of the spoken audio, timestamped only after the audio finishes playing — not the audit log's true (early) LLM-generation timestamp used to build the trace for cascade/audio-LLM. That transcription can sort *after* a subsequent tool call in the timestamp-ordered trace even when the agent actually spoke first (confirmed against a real S2S transcript), so trace order can't be trusted for this signal on S2S. Reading `audit_log.json` directly sidesteps the issue and gives one code path across cascade, S2S, and audio-LLM. + +`pretoolspeech_rate` is omitted (not emitted) when there are no tool-call groups, or when `audit_log.json` is missing/unreadable/empty at `context.output_dir` — treated as "unknown", not "no lead-ins". + +Rate sub-metrics are emitted as `normalized_score` (they already live on `[0, 1]`). Raw-ms/count sub-metrics have `normalized_score = None` so they don't corrupt cross-metric averages. ## Tunable Constants diff --git a/src/eva/metrics/experience/turn_taking.py b/src/eva/metrics/experience/turn_taking.py index fc310e60..fef7b76e 100644 --- a/src/eva/metrics/experience/turn_taking.py +++ b/src/eva/metrics/experience/turn_taking.py @@ -35,13 +35,19 @@ user_interruption.mean_yield_ms, user_interruption.mean_yield_score (the latter two only when rate > 0) + Pre-tool speech: pretoolspeech_rate (lead-in tool-call groups / total tool-call groups; + computed from audit_log.json directly so it works uniformly across + cascade/S2S/audio-LLM; omitted when there are no tool calls or the + audit log is unavailable — see ``_compute_pre_tool_speech_groups``) All reported sub-metrics are consistent with the main score: ``mean_overlap_score``, ``mean_count_score``, and ``mean_yield_score`` aggregate exactly the per-turn scores that feed into ``turn_taking.score``. """ +import json import statistics +from pathlib import Path from typing import Any from eva.metrics.base import CodeMetric, MetricContext @@ -59,7 +65,7 @@ class TurnTakingMetric(CodeMetric): description = "Turn-taking evaluation based on per-turn latency and interruption behavior" category = "experience" pass_at_k_threshold = 0.8 - version = "v0.1" + version = "v0.2" # --- Latency curve (piecewise linear). 0 outside [LATENCY_HARD_EARLY_MS, LATENCY_HARD_LATE_MS]. --- # Ramp up 0 → 1 from LATENCY_HARD_EARLY_MS to LATENCY_SWEET_SPOT_LOW_MS. @@ -235,6 +241,64 @@ def _compute_yield_ms(context: MetricContext, turn_id: int) -> float | None: agent_stopped = prev_a_segs[-1][1] return max(0.0, agent_stopped - user_barge_in) * 1000 + @staticmethod + def _compute_pre_tool_speech_groups(context: MetricContext) -> list[bool] | None: + """Return one bool per contiguous run ("group") of tool_call/tool_response entries. + + True when the assistant spoke (non-empty content) since the previous group ended (or + since the start of the conversation) — i.e. it gave a pre-tool-speech lead-in before this + group of tool calls. Consecutive tool calls with no intervening speech (e.g. two tools + invoked back-to-back off one lead-in) count as a single group, matching the + "one lead-in per batch" prompt instruction (``agent.pre_tool_speech`` in + configs/prompts/simulation.yaml) — a multi-tool turn with one lead-in isn't penalized for + the tools that didn't get their own. + + Reads ``audit_log.json`` directly from ``context.output_dir`` instead of + ``context.conversation_trace``. The audit log carries every event's true, original + timestamp (when the assistant actually generated/spoke the text), whereas for S2S + ``conversation_trace`` assistant entries come from the user simulator's STT transcription + of the spoken audio — timestamped only after the audio finishes playing. That transcription + can sort *after* a subsequent tool_call in the timestamp-ordered trace even when the agent + actually spoke first (confirmed against a real S2S transcript), so ``conversation_trace`` + order cannot be trusted for this signal on S2S. Reading the audit log directly sidesteps + that entirely and works uniformly across cascade, S2S, and audio-LLM. + + Returns None when ``audit_log.json`` is missing, unreadable, or has no transcript — callers + should treat that as "unknown" rather than "no lead-ins". + """ + audit_log_path = Path(context.output_dir) / "audit_log.json" + try: + with open(audit_log_path) as f: + audit_log = json.load(f) + except (OSError, json.JSONDecodeError): + return None + transcript = audit_log.get("transcript") + if not transcript: + return None + + groups: list[bool] = [] + in_group = False + spoke_since_group = False + for entry in sorted(transcript, key=lambda e: int(e["timestamp"])): + message_type = entry.get("message_type") + if message_type in ("tool_call", "tool_response"): + if not in_group: + groups.append(spoke_since_group) + in_group = True + continue + # Ignore other event types (e.g. llm_call) without breaking the current group — + # only assistant/user speech should reset or extend the "spoke since group" state. + if message_type not in ("assistant", "user"): + continue + in_group = False + if message_type == "assistant": + content = entry.get("value", "") + if isinstance(content, str) and content.strip(): + spoke_since_group = True + else: + spoke_since_group = False + return groups + @classmethod def _per_turn_score_and_reason( cls, @@ -430,6 +494,17 @@ def _pct(p: float) -> float: "user_interruption.mean_yield_score", round(statistics.mean(yield_scores), 4), True ) + # --- Pre-tool-speech lead-in rate --- + # None (omitted) for S2S — see _compute_pre_tool_speech_groups docstring for why trace + # order can't be trusted there yet. + pre_tool_groups = cls._compute_pre_tool_speech_groups(context) + if pre_tool_groups: + sub["pretoolspeech_rate"] = _wrap( + "pretoolspeech_rate", + round(sum(pre_tool_groups) / len(pre_tool_groups), 4), + True, + ) + # Token usage (from agent_perf_stats.csv) mean_output_tokens = mean_agent_perf_stat(context.output_dir, "output_tokens") mean_reasoning_tokens = mean_agent_perf_stat(context.output_dir, "reasoning_tokens") diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 4dc4639f..7ded1093 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -92,8 +92,8 @@ "TurnTakingMetric": { "name": "turn_taking", "prompt_hash": null, - "source_hash": "fee8caa7adc7", - "version": "v0.1" + "source_hash": "7a2893c52748", + "version": "v0.2" }, "UserBehavioralFidelityMetric": { "name": "user_behavioral_fidelity", diff --git a/tests/unit/metrics/test_turn_taking.py b/tests/unit/metrics/test_turn_taking.py index 052742f2..5dd6bc8b 100644 --- a/tests/unit/metrics/test_turn_taking.py +++ b/tests/unit/metrics/test_turn_taking.py @@ -26,11 +26,13 @@ user_interruption.mean_yield_score (only when rate > 0) """ +import json import logging import pytest from eva.metrics.experience.turn_taking import TurnTakingMetric +from eva.models.config import PipelineType from .conftest import make_metric_context @@ -778,3 +780,276 @@ async def test_dual_interrupt_sub_metrics_populated_for_both_interrupt_types(sel assert sub["user_interruption.rate"].score == pytest.approx(1 / 3, abs=1e-4) assert sub["user_interruption.mean_yield_ms"].score == pytest.approx(500, abs=1) assert sub["user_interruption.mean_yield_score"].score == pytest.approx(0.75, abs=1e-3) + + +# ---------- Pre-tool-speech lead-in rate ---------- + + +def _audit_assistant(ts, content): + return {"message_type": "assistant", "timestamp": ts, "value": content} + + +def _audit_user(ts, content="hi"): + return {"message_type": "user", "timestamp": ts, "value": content} + + +def _audit_tool_call(ts, tool="lookup"): + return {"message_type": "tool_call", "timestamp": ts, "value": {"tool": tool, "parameters": {}}} + + +def _audit_tool_response(ts, tool="lookup"): + return {"message_type": "tool_response", "timestamp": ts, "value": {"tool": tool, "response": {}}} + + +def _audit_llm_call(ts, response=""): + """An llm_call event — present in real audit logs but must not affect grouping.""" + return {"message_type": "llm_call", "timestamp": ts, "value": {"agent": "Test Agent", "response": response}} + + +def _write_audit_log(tmp_path, transcript): + (tmp_path / "audit_log.json").write_text(json.dumps({"transcript": transcript})) + return str(tmp_path) + + +class TestPreToolSpeechGroups: + def test_no_lead_in_before_tool_call(self, metric, tmp_path): + """Empty assistant speech immediately before a tool call → group is False.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, ""), + _audit_tool_call(3), + _audit_tool_response(3), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [False] + + def test_lead_in_before_tool_call(self, metric, tmp_path): + """Non-empty assistant speech before a tool call → group is True.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, "Let me pull that up."), + _audit_tool_call(3), + _audit_tool_response(3), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [True] + + def test_multiple_tools_from_one_lead_in_count_as_one_group(self, metric, tmp_path): + """Two tool_call/tool_response pairs with no intervening speech → single group.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, ""), + _audit_tool_call(3, "confirm_swap"), + _audit_tool_response(3, "confirm_swap"), + _audit_tool_call(4, "notify_manager"), + _audit_tool_response(4, "notify_manager"), + _audit_assistant(5, "All done."), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [False] + + def test_llm_call_events_do_not_break_a_group(self, metric, tmp_path): + """Split llm_call events should not break a tool call group. + + llm_call entries (present in real audit logs) between two tool batches must be ignored, + not treated as a group-breaking event — regression guard for the bug where an intervening + llm_call incorrectly split one contiguous tool-call group into two. + """ + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_llm_call(2, ""), + _audit_tool_call(2, "confirm_swap"), + _audit_tool_response(2, "confirm_swap"), + _audit_llm_call(3, ""), + _audit_tool_call(3, "notify_manager"), + _audit_tool_response(3, "notify_manager"), + _audit_llm_call(4, "All done."), + _audit_assistant(4, "All done."), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [False] + + def test_user_speech_resets_lead_in_flag(self, metric, tmp_path): + """Assistant speech in an earlier turn doesn't count as a lead-in once the user speaks again.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_assistant(1, "Hello, how can I help?"), + _audit_user(2), + _audit_assistant(3, ""), + _audit_tool_call(4), + _audit_tool_response(4), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [False] + + def test_multiple_turns_multiple_groups(self, metric, tmp_path): + """Mix of lead-in and no-lead-in tool groups across turns.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, ""), + _audit_tool_call(3), + _audit_tool_response(3), + _audit_assistant(4, "Verified."), + _audit_user(5), + _audit_assistant(6, "One moment."), + _audit_tool_call(7), + _audit_tool_response(7), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [False, True] + + def test_out_of_order_transcript_is_sorted_by_timestamp(self, metric, tmp_path): + """Transcript entries out of file order are re-sorted by timestamp before grouping.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_tool_call(3), + _audit_user(1), + _audit_tool_response(3), + _audit_assistant(2, "Let me check."), + ], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [True] + + def test_no_tool_calls_returns_empty_list(self, metric, tmp_path): + output_dir = _write_audit_log( + tmp_path, + [_audit_user(1), _audit_assistant(2, "Just chatting, no tools needed.")], + ) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [] + + def test_missing_audit_log_returns_none(self, metric, tmp_path): + context = make_metric_context(output_dir=str(tmp_path)) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups is None + + def test_empty_transcript_returns_none(self, metric, tmp_path): + output_dir = _write_audit_log(tmp_path, []) + context = make_metric_context(output_dir=output_dir) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups is None + + def test_works_for_s2s_pipeline_type(self, metric, tmp_path): + """Audit-log-based computation works the same regardless of pipeline_type.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, "Let me pull that up."), + _audit_tool_call(3), + _audit_tool_response(3), + ], + ) + context = make_metric_context(output_dir=output_dir, pipeline_type=PipelineType.S2S) + groups = metric._compute_pre_tool_speech_groups(context) + assert groups == [True] + + +class TestPreToolSpeechSubMetric: + @pytest.mark.asyncio + async def test_sub_metric_reflects_lead_in_rate(self, metric, tmp_path): + """2 of 4 tool-call groups have a lead-in → pretoolspeech_rate = 0.5.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, ""), + _audit_tool_call(3, "a"), + _audit_tool_response(3, "a"), + _audit_assistant(4, "Got it."), + _audit_user(5), + _audit_assistant(6, "Let me check."), + _audit_tool_call(7, "b"), + _audit_tool_response(7, "b"), + _audit_user(8), + _audit_assistant(9, "One sec."), + _audit_tool_call(10, "c"), + _audit_tool_response(10, "c"), + _audit_user(11), + _audit_assistant(12, ""), + _audit_tool_call(13, "d"), + _audit_tool_response(13, "d"), + ], + ) + context = make_metric_context( + output_dir=output_dir, + audio_timestamps_user_turns={1: [(0.0, 1.0)]}, + audio_timestamps_assistant_turns={1: [(1.5, 2.0)]}, + ) + result = await metric.compute(context) + sub = result.sub_metrics + assert sub["pretoolspeech_rate"].score == pytest.approx(0.5) + assert sub["pretoolspeech_rate"].normalized_score == pytest.approx(0.5) + + @pytest.mark.asyncio + async def test_sub_metric_omitted_when_no_tool_calls(self, metric, tmp_path): + output_dir = _write_audit_log( + tmp_path, + [_audit_user(1), _audit_assistant(2, "No tools here.")], + ) + context = make_metric_context( + output_dir=output_dir, + audio_timestamps_user_turns={1: [(0.0, 1.0)]}, + audio_timestamps_assistant_turns={1: [(1.5, 2.0)]}, + ) + result = await metric.compute(context) + assert "pretoolspeech_rate" not in result.sub_metrics + + @pytest.mark.asyncio + async def test_sub_metric_omitted_when_audit_log_missing(self, metric, tmp_path): + context = make_metric_context( + output_dir=str(tmp_path), + audio_timestamps_user_turns={1: [(0.0, 1.0)]}, + audio_timestamps_assistant_turns={1: [(1.5, 2.0)]}, + ) + result = await metric.compute(context) + assert "pretoolspeech_rate" not in result.sub_metrics + + @pytest.mark.asyncio + async def test_sub_metric_populated_for_s2s(self, metric, tmp_path): + """Unlike the old trace-based approach, S2S now gets a real pretoolspeech_rate.""" + output_dir = _write_audit_log( + tmp_path, + [ + _audit_user(1), + _audit_assistant(2, "Let me check."), + _audit_tool_call(3), + _audit_tool_response(3), + ], + ) + context = make_metric_context( + output_dir=output_dir, + audio_timestamps_user_turns={1: [(0.0, 1.0)]}, + audio_timestamps_assistant_turns={1: [(1.5, 2.0)]}, + pipeline_type=PipelineType.S2S, + ) + result = await metric.compute(context) + sub = result.sub_metrics + assert sub["pretoolspeech_rate"].score == pytest.approx(1.0) From 8a8a3763ee1a78a3f3465abf19ce127bb440518e Mon Sep 17 00:00:00 2001 From: Katrina Date: Tue, 21 Jul 2026 15:37:17 -0400 Subject: [PATCH 2/6] add pretool speech --- src/eva/assistant/agentic/audio_llm_system.py | 5 +++++ src/eva/assistant/pipecat_server.py | 1 + src/eva/assistant/pipeline/audio_llm_processor.py | 2 ++ src/eva/models/config.py | 11 +++++------ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/eva/assistant/agentic/audio_llm_system.py b/src/eva/assistant/agentic/audio_llm_system.py index 9ff227a3..819784c6 100644 --- a/src/eva/assistant/agentic/audio_llm_system.py +++ b/src/eva/assistant/agentic/audio_llm_system.py @@ -39,6 +39,7 @@ def __init__( audit_log: AuditLog, alm_client: BaseALMClient, output_dir: Path | None = None, + pre_tool_speech: str = "off", llm_streaming: bool = False, full_audio_context: bool = False, ): @@ -49,6 +50,7 @@ def __init__( audit_log=audit_log, llm_client=alm_client, output_dir=output_dir, + pre_tool_speech=pre_tool_speech, llm_streaming=llm_streaming, ) self.alm_client: BaseALMClient = alm_client @@ -64,6 +66,9 @@ def __init__( agent_instructions=agent.instructions, datetime=current_date_time, ) + # Reuse the shared pre-tool lead-in prompt appended to the system prompt. + if self.pre_tool_speech == "auto": + self.system_prompt += "\n\n" + self.prompt_manager.get_prompt("agent.pre_tool_speech") # Per-turn audio history: list of (audio_bytes, sample_rate) self._turn_audio_history: list[tuple[bytes, int]] = [] diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index a326a8df..4d0f231a 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -368,6 +368,7 @@ async def _realtime_tool_handler(params) -> None: alm_client=alm_client, audio_collector=audio_llm_audio_collector, output_dir=self.output_dir, + pre_tool_speech=self.pipeline_config.pre_tool_speech, llm_streaming=self.pipeline_config.llm_streaming, full_audio_context=self.pipeline_config.audio_llm_params.get("full_audio_context", False), ) diff --git a/src/eva/assistant/pipeline/audio_llm_processor.py b/src/eva/assistant/pipeline/audio_llm_processor.py index 3088438a..8255d957 100644 --- a/src/eva/assistant/pipeline/audio_llm_processor.py +++ b/src/eva/assistant/pipeline/audio_llm_processor.py @@ -190,6 +190,7 @@ def __init__( alm_client: BaseALMClient, audio_collector: AudioLLMUserAudioCollector, output_dir: Path | None = None, + pre_tool_speech: str = "off", llm_streaming: bool = False, full_audio_context: bool = False, **kwargs, @@ -207,6 +208,7 @@ def __init__( audit_log=audit_log, alm_client=alm_client, output_dir=output_dir, + pre_tool_speech=pre_tool_speech, llm_streaming=llm_streaming, full_audio_context=full_audio_context, ) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 5a4cf055..4f80812a 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -295,13 +295,12 @@ def _validate_latency_optimizations(self) -> "ModelConfig": allowed = {"off", "auto"} if self.pre_tool_speech not in allowed: raise ValueError(f"pre_tool_speech must be one of {sorted(allowed)}, got '{self.pre_tool_speech}'") - # llm_streaming is honored by AUDIO_LLM too (via BaseALMClient.complete_stream); only - # pre_tool_speech / parallel_tool_calls remain CASCADE-only. - cascade_only_set = self.pre_tool_speech != "off" or self.parallel_tool_calls is not None - if cascade_only_set and self.pipeline_type != PipelineType.CASCADE: + # pre_tool_speech is honored by both CASCADE and AUDIO_LLM; llm_streaming by both + # (via BaseALMClient.complete_stream); only parallel_tool_calls remains CASCADE-only. + if self.parallel_tool_calls is not None and self.pipeline_type != PipelineType.CASCADE: logger.warning( - "Cascade LLM flags (pre_tool_speech / parallel_tool_calls) apply only to the CASCADE " - f"pipeline; they will be ignored for pipeline_type={self.pipeline_type}." + "parallel_tool_calls applies only to the CASCADE pipeline; it will be ignored " + f"for pipeline_type={self.pipeline_type}." ) return self From 83f2dcba5b8b2cbc571005e0d72f4675e53d1976 Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 23 Jul 2026 11:34:18 -0400 Subject: [PATCH 3/6] remove fallback nudging info from turn_taking.md --- docs/metrics/turn_taking.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/metrics/turn_taking.md b/docs/metrics/turn_taking.md index 8bc7377f..91830a64 100644 --- a/docs/metrics/turn_taking.md +++ b/docs/metrics/turn_taking.md @@ -190,13 +190,6 @@ Sub-metric aggregation reads the raw per-turn values from `per_turn_evidence` ra | `user_interruption.mean_yield_ms` | no | rate > 0 | Arithmetic mean of `yield_ms` across user-interrupt turns. | | `user_interruption.mean_yield_score` | yes | rate > 0 | Mean of the per-turn yield scores that feed the main score. | -**Turn-end fallback nudges** - -| Key | Normalized? | When present | Meaning | -| --- | --- | --- | --- | -| `fallback_nudge.rate` | yes | at least one assistant turn | `len(fallback_turn_ids) / total_assistant_turns`. Denominator is all assistant turns (`audio_timestamps_assistant_turns`), not the evaluable-turn count, since a fallback nudge produces an assistant turn with no paired user turn and so is never in the evaluable set. | -| `fallback_nudge.count` | no (raw count) | always, but `score=None` when zero | Total number of fallback-nudge turns. `None` on clean runs so cross-record aggregates exclude them rather than averaging in zeros. | - **Pre-tool-speech lead-in rate** | Key | Normalized? | When present | Meaning | From 7aeb7e11c12cd796e8054bb66b8f18bbbebd720c Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 23 Jul 2026 11:44:09 -0400 Subject: [PATCH 4/6] improve tests --- tests/unit/metrics/test_turn_taking.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/unit/metrics/test_turn_taking.py b/tests/unit/metrics/test_turn_taking.py index 5dd6bc8b..fbd9ce1d 100644 --- a/tests/unit/metrics/test_turn_taking.py +++ b/tests/unit/metrics/test_turn_taking.py @@ -892,9 +892,8 @@ def test_user_speech_resets_lead_in_flag(self, metric, tmp_path): [ _audit_assistant(1, "Hello, how can I help?"), _audit_user(2), - _audit_assistant(3, ""), - _audit_tool_call(4), - _audit_tool_response(4), + _audit_tool_call(3), + _audit_tool_response(3), ], ) context = make_metric_context(output_dir=output_dir) @@ -907,14 +906,13 @@ def test_multiple_turns_multiple_groups(self, metric, tmp_path): tmp_path, [ _audit_user(1), - _audit_assistant(2, ""), - _audit_tool_call(3), - _audit_tool_response(3), - _audit_assistant(4, "Verified."), - _audit_user(5), - _audit_assistant(6, "One moment."), - _audit_tool_call(7), - _audit_tool_response(7), + _audit_tool_call(2), + _audit_tool_response(2), + _audit_assistant(3, "Verified."), + _audit_user(4), + _audit_assistant(5, "One moment."), + _audit_tool_call(6), + _audit_tool_response(6), ], ) context = make_metric_context(output_dir=output_dir) From 1d42e6db921a48ceb23278f88b054cace9b7d168 Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 23 Jul 2026 11:48:05 -0400 Subject: [PATCH 5/6] update comment for s2s --- src/eva/metrics/experience/turn_taking.py | 3 +-- tests/fixtures/metric_signatures.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/eva/metrics/experience/turn_taking.py b/src/eva/metrics/experience/turn_taking.py index fef7b76e..aa691bd9 100644 --- a/src/eva/metrics/experience/turn_taking.py +++ b/src/eva/metrics/experience/turn_taking.py @@ -495,8 +495,7 @@ def _pct(p: float) -> float: ) # --- Pre-tool-speech lead-in rate --- - # None (omitted) for S2S — see _compute_pre_tool_speech_groups docstring for why trace - # order can't be trusted there yet. + # Reads audit_log.json directly so it works uniformly across cascade, S2S, and audio-LLM pre_tool_groups = cls._compute_pre_tool_speech_groups(context) if pre_tool_groups: sub["pretoolspeech_rate"] = _wrap( diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 7ded1093..64016bf1 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -92,7 +92,7 @@ "TurnTakingMetric": { "name": "turn_taking", "prompt_hash": null, - "source_hash": "7a2893c52748", + "source_hash": "1db130ae1aa6", "version": "v0.2" }, "UserBehavioralFidelityMetric": { From b69c2dd8438c4b3820145180a099bd17c202f384 Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 23 Jul 2026 12:31:48 -0400 Subject: [PATCH 6/6] force pretoolspeech to be .lower() --- src/eva/models/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 4f80812a..8bd9130b 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -197,6 +197,12 @@ class ModelConfig(BaseModel): "off", description="Prompt a model-generated lead-in before tool calls: 'off' or 'auto'.", ) + + @field_validator("pre_tool_speech", mode="before") + @classmethod + def _normalize_pre_tool_speech(cls, value: str) -> str: + return value.lower() if isinstance(value, str) else value + llm_streaming: bool = Field( False, description="Stream Chat Completions output to TTS sentence-by-sentence.",