fix(twilio): keep the lines a relay session says on its own in the history

src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs

@@ -61,11 +61,36 @@ public class AgentTurnCallbacks
     public string? LastReplyLine { get; set; }
     public string? LastReplyHeard { get; set; }
 
+    private readonly List<(string Text, DateTime SaidUtc)> _fillers = [];
+
+    /// <summary>
+    /// Notes a filler said while the agent works on the reply ("One moment, please."), to be kept in the history out of
+    /// the agent's context, so what the caller heard is all there
+    /// </summary>
+    public void NoteFillerSaid(string text)
+    {
+        lock (_fillers)
+        {
+            _fillers.Add((text, DateTime.UtcNow));
+        }
+    }
+
     /// <summary>
-    /// Set by the turn: called with each filler said while the agent works on the reply ("One moment, please."), which
-    /// the turn keeps in the history out of the agent's context, so what the caller heard is all there
+    /// The fillers said and not kept yet, oldest first; each is taken once
     /// </summary>
-    public Action<string>? OnFillerSaid { get; set; }
+    /// <remarks>
+    /// The turn takes them as it stores its reply, to keep them just ahead of it. The flow takes what is left once the
+    /// turn is over: the fillers of a turn that failed, ran past its time or stored no reply, lost otherwise.
+    /// </remarks>
+    public List<(string Text, DateTime SaidUtc)> TakeFillers()
+    {
+        lock (_fillers)
+        {
+            var taken = _fillers.ToList();
+            _fillers.Clear();
+            return taken;
+        }
+    }
 
     /// <summary>
     /// Called on the turn once it has read the conversation's states (<see cref="KeepUninterrupted"/> is set), before the

src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayLabels.cs

@@ -0,0 +1,35 @@
+namespace BotSharp.Plugin.Twilio.Models;
+
+/// <summary>
+/// What a line kept only as a record of what the caller heard was, kept as its dialog's MessageLabel: such lines look
+/// alike otherwise (assistant, out of the agent's context), and a flow may treat them differently
+/// </summary>
+public static class ConversationRelayLabels
+{
+    /// <summary>
+    /// Said while the agent worked on a reply ("One moment, please.")
+    /// </summary>
+    public const string Filler = "twilio_relay_filler";
+
+    /// <summary>
+    /// Said to a caller gone quiet: the last question asked again ("Sorry, I didn't catch that. ..."), "Are you still
+    /// there?", or "Sorry, can you say it again?". It checks the caller is there, so it is no line of the agent's to say
+    /// again when they ask what was said.
+    /// </summary>
+    public const string IdlePrompt = "twilio_relay_idle_prompt";
+
+    /// <summary>
+    /// Said as the session hangs up on a caller who stayed quiet
+    /// </summary>
+    public const string IdleGoodbye = "twilio_relay_idle_goodbye";
+
+    /// <summary>
+    /// Said as the session hangs up on a turn that failed or ran past its time
+    /// </summary>
+    public const string Apology = "twilio_relay_apology";
+
+    /// <summary>
+    /// A line the caller cut into, said again once nothing followed the cut-in (ResumeAfterCutInSeconds)
+    /// </summary>
+    public const string SaidAgain = "twilio_relay_said_again";
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs

@@ -39,6 +39,7 @@ public class ConversationRelaySession
     private readonly ConversationRelaySetting _relay;
     private readonly Func<CallerMessage, AgentTurnCallbacks, Task<AssistantMessage?>> _sendCallerMessage;
     private readonly ILogger _logger;
+    private readonly Func<string, string, DateTime, Task>? _keepSaidLine;
 
     private readonly SemaphoreSlim _sendLock = new(1, 1);
     private readonly object _sync = new();
@@ -140,13 +141,20 @@ public class ConversationRelaySession
     /// Sends the caller's words to the agent and returns its reply; the callbacks receive tool-call indications
     /// and the streamed reply
     /// </param>
+    /// <param name="keepSaidLine">
+    /// Keeps a line the session said on its own in the conversation history, with what it was
+    /// (<see cref="ConversationRelayLabels"/>) and when it was said, so the history shows what the caller heard: the
+    /// apology of a failed turn, an idle prompt and its goodbye, a line said again after a cut-in, and the fillers of a
+    /// turn that kept none. A turn keeps its own reply, and a tool-call indication is the flow's to keep.
+    /// </param>
     public ConversationRelaySession(
         WebSocket socket,
         string agentId,
         string conversationId,
         TwilioSetting settings,
         Func<CallerMessage, AgentTurnCallbacks, Task<AssistantMessage?>> sendCallerMessage,
-        ILogger logger)
+        ILogger logger,
+        Func<string, string, DateTime, Task>? keepSaidLine = null)
     {
         _socket = socket;
         _agentId = agentId;
@@ -155,6 +163,7 @@ public ConversationRelaySession(
         _relay = settings.ConversationRelay.For(agentId);
         _sendCallerMessage = sendCallerMessage;
         _logger = logger;
+        _keepSaidLine = keepSaidLine;
     }
 
     public async Task RunAsync(CancellationToken cancellationToken)
@@ -522,7 +531,8 @@ private async Task ProcessPendingAsync()
                 // A line said before a slow tool ("Okay, that's everything I need") is spoken now, and the watchdog
                 // counts the silence after it instead of waiting for the next response to close it
                 OnStreamEndedWithToolCall = stream ? CloseUtteranceBeforeToolAsync : null,
-                RewriteReply = stream ? MarkWhatTheCallerCutOff : null
+                // Streamed or not: a reply the call ended before is marked as never said
+                RewriteReply = MarkWhatTheCallerCutOff
             };
 
             CallerMessage message;
@@ -604,15 +614,15 @@ private async Task ProcessPendingAsync()
                     var timeoutSeconds = _relay.AiResponseTimeoutSeconds;
                     _logger.LogWarning($"Agent did not reply within {timeoutSeconds}s in conversation {_conversationId}.");
                     _timedOutTurn = turn;
-                    await FailAsync();
+                    await FailAsync(callbacks);
                     return;
                 }
                 reply = await turn;
             }
             catch (Exception ex)
             {
                 _logger.LogError(ex, $"Agent turn failed in conversation {_conversationId}.");
-                await FailAsync();
+                await FailAsync(callbacks);
                 return;
             }
             finally
@@ -624,6 +634,7 @@ private async Task ProcessPendingAsync()
                 }
                 watchdogCts.Cancel();
                 await SwallowAsync(watchdog);
+                await KeepFillersLeftAsync(callbacks);
             }
 
             await CloseUtteranceAsync();
@@ -673,7 +684,8 @@ await SpeakAsync(unstreamed, canSpeak: () =>
     /// While the agent works on a reply, says a filler each time the line has been silent for the next
     /// ResponseWatchdogIntervals value, so the caller never sits in a long silence while the model thinks or a tool runs.
     /// Silence counts from the end of whatever was said last (the caller, a filler, an indication), and no filler is
-    /// said twice in a turn while unused ones remain. The turn is told of each one said (OnFillerSaid).
+    /// said twice in a turn while unused ones remain. Each one said is noted on the turn's callbacks (NoteFillerSaid), to be
+    /// kept in the history.
     /// </summary>
     private async Task WatchResponseAsync(int seqNumber, AgentTurnCallbacks callbacks, CancellationToken cancellationToken)
     {
@@ -698,16 +710,15 @@ private async Task WatchResponseAsync(int seqNumber, AgentTurnCallbacks callback
 
             var text = PickFillerText(step == 0 ? relay.ResponseWatchdogPrompts : relay.ResponseWatchdogLongPrompts,
                 step == 0 ? DefaultWatchdogPrompts : DefaultWatchdogLongPrompts, said);
-            if (await SpeakFillerAsync(seqNumber, text, interval))
+            if (await SpeakFillerAsync(seqNumber, text, interval, callbacks))
             {
                 said.Add(text);
                 step++;
-                callbacks.OnFillerSaid?.Invoke(text);
             }
         }
     }
 
-    private async Task<bool> SpeakFillerAsync(int seqNumber, string text, TimeSpan interval)
+    private async Task<bool> SpeakFillerAsync(int seqNumber, string text, TimeSpan interval, AgentTurnCallbacks callbacks)
     {
         // Decided again once the send lock is held: while waiting for it the reply may have come in, or a streamed
         // line may have started or been closed, and then the silence this filler was meant for is over
@@ -718,6 +729,9 @@ private async Task<bool> SpeakFillerAsync(int seqNumber, string text, TimeSpan i
                     return false;
                 }
                 RememberUtterance(text, _saidCount + 1);
+                // Noted as it goes out, under the send lock: a turn that fails meanwhile finds every filler said before
+                // its apology
+                callbacks.NoteFillerSaid(text);
                 return true;
             },
             new { type = "text", token = text, last = true }, EstimateSpeech(text.Length));
@@ -840,25 +854,39 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken)
             bool StillSilent() => !_isProcessing && _activity == activity;
             if (askAgain && question != null)
             {
-                await SpeakAsync(PromptText("say-it-again-question", DefaultAskAgainText).Replace("{0}", question), StillSilent);
+                await SayOwnLineAsync(PromptText("say-it-again-question", DefaultAskAgainText).Replace("{0}", question),
+                    ConversationRelayLabels.IdlePrompt, StillSilent);
             }
-            else if (askAgain && relay.IdleAsksAgain && stage > 0)
+            // Nothing heard from the caller yet (no words, no keys): there is nothing for them to say again, only whether
+            // they are there. Why: silence after the greeting got "Sorry, can you say it again?" (local call simulation)
+            else if (askAgain && (relay.IdleAsksAgain && stage > 0 || activity == 0))
             {
-                await SpeakPromptAsync("still-there", DefaultStillThereText, StillSilent);
+                await SayOwnLineAsync(PromptText("still-there", DefaultStillThereText), ConversationRelayLabels.IdlePrompt, StillSilent);
             }
             else if (askAgain)
             {
-                await SpeakPromptAsync($"say-it-again-{Random.Shared.Next(1, 5)}", DefaultSayItAgainText, StillSilent);
+                await SayOwnLineAsync(PromptText($"say-it-again-{Random.Shared.Next(1, 5)}", DefaultSayItAgainText),
+                    ConversationRelayLabels.IdlePrompt, StillSilent);
             }
             else if (hangUp)
             {
                 _logger.LogInformation($"Caller stayed silent, hanging up conversation {_conversationId}.");
+                string? goodbye = null;
+                var goodbyeUtc = DateTime.MinValue;
                 if (relay.IdleAsksAgain)
                 {
-                    // The session is marked ended already, so nothing else can be said after it
-                    await SendAsync(TextFrame(PromptText("idle-goodbye", DefaultIdleGoodbyeText), last: true, keepUninterrupted: false));
+                    // The session is marked ended already, so nothing else can be said after it, and nothing the caller
+                    // says over it is answered: it plays to the end
+                    goodbye = PromptText("idle-goodbye", DefaultIdleGoodbyeText);
+                    await SendAsync(TextFrame(goodbye, last: true, keepUninterrupted: true));
+                    goodbyeUtc = DateTime.UtcNow;
                 }
                 await SendEndAsync(ConversationRelayHandoff.HangUp);
+                // Kept once the call is ended, so the storage doesn't hold up the hang-up
+                if (goodbye != null)
+                {
+                    await KeepSaidAsync(goodbye, ConversationRelayLabels.IdleGoodbye, goodbyeUtc);
+                }
             }
         }
     }
@@ -922,8 +950,7 @@ private async Task ResumeIfSilentAsync()
             _resumeLine = null;
         }
 
-        var said = false;
-        await SpeakAsync(line, canSpeak: () =>
+        var said = await SpeakAsync(line, canSpeak: () =>
         {
             if (_isProcessing || _activity != activity)
             {
@@ -938,25 +965,83 @@ await SpeakAsync(line, canSpeak: () =>
             NoteLine(line);
             // Set as it goes out: a cut-in into it, however soon, doesn't get it said a third time
             _lastWasResume = true;
-            said = true;
             return true;
         });
         if (said)
         {
             _logger.LogInformation($"Nothing followed the caller's cut-in in conversation {_conversationId}; said the line again.");
+            await KeepSaidAsync(line, ConversationRelayLabels.SaidAgain, DateTime.UtcNow);
         }
     }
 
-    private async Task FailAsync()
+    private async Task FailAsync(AgentTurnCallbacks callbacks)
     {
         // A turn that timed out may still be streaming; keep its words out of the apology
         lock (_sync)
         {
             _dropStream = true;
         }
         await CloseUtteranceAsync();
-        await SpeakPromptAsync("error", DefaultErrorText);
+        var apology = PromptText("error", DefaultErrorText);
+        // The call ends after it, so nothing the caller says over it is answered: it plays to the end
+        var apologized = await SpeakAsync(apology, keepUninterrupted: true);
+        var apologyUtc = DateTime.UtcNow;
         await EndAsync(ConversationRelayHandoff.HangUp);
+
+        // Kept once the call is ended, so the storage doesn't hold up the hang-up and no filler can come after the
+        // apology. The turn kept none of the fillers it took to fail: they go first, as they were said first.
+        await KeepFillersLeftAsync(callbacks);
+        if (apologized)
+        {
+            await KeepSaidAsync(apology, ConversationRelayLabels.Apology, apologyUtc);
+        }
+    }
+
+    /// <summary>
+    /// Keeps the fillers of a turn that it didn't keep itself: it failed, ran past its time, or stored no reply. Call once
+    /// the turn can say nothing more.
+    /// </summary>
+    private async Task KeepFillersLeftAsync(AgentTurnCallbacks callbacks)
+    {
+        foreach (var (text, saidUtc) in callbacks.TakeFillers())
+        {
+            await KeepSaidAsync(text, ConversationRelayLabels.Filler, saidUtc);
+        }
+    }
+
+    /// <summary>
+    /// Says a line of the session's own, not the agent's, and keeps it in the history once it went out
+    /// </summary>
+    /// <param name="label">What the line is (<see cref="ConversationRelayLabels"/>)</param>
+    /// <param name="canSpeak">Extra condition, checked under <see cref="_sync"/> once the send lock is held</param>
+    private async Task SayOwnLineAsync(string text, string label, Func<bool>? canSpeak = null)
+    {
+        if (await SpeakAsync(text, canSpeak))
+        {
+            await KeepSaidAsync(text, label, DateTime.UtcNow);
+        }
+    }
+
+    /// <summary>
+    /// Keeps a line the session said on its own in the conversation history, so the history shows what the caller heard
+    /// </summary>
+    /// <param name="label">What the line is (<see cref="ConversationRelayLabels"/>)</param>
+    private async Task KeepSaidAsync(string text, string label, DateTime saidUtc)
+    {
+        if (_keepSaidLine == null || string.IsNullOrWhiteSpace(text))
+        {
+            return;
+        }
+
+        try
+        {
+            await _keepSaidLine(text.Trim(), label, saidUtc);
+        }
+        catch (Exception ex)
+        {
+            // Only the record of the line is lost: the call goes on
+            _logger.LogWarning(ex, $"Could not keep a line said in conversation {_conversationId}.");
+        }
     }
 
     /// <summary>
@@ -1047,7 +1132,9 @@ await SendIfAsync(() =>
     /// Text goes out faster than it is spoken, so the cut is placed where Twilio says playback stopped; when Twilio
     /// doesn't say, or what it says is not the start of what went out, the cut falls where the sending stopped. A reply
     /// dropped because the caller went on talking first is marked as never said, so the next turn doesn't take it as
-    /// heard.
+    /// heard, and so is one the call ended before: the caller hung up while the agent worked, or the turn ran past
+    /// AiResponseTimeoutSeconds and the session apologized and hung up (it came back after the apology, and the history
+    /// showed it as said).
     /// </remarks>
     private string MarkWhatTheCallerCutOff(string content)
     {
@@ -1062,6 +1149,11 @@ private string MarkWhatTheCallerCutOff(string content)
                     ? "[the caller went on talking before anything was said]"
                     : $"[the caller went on talking before this was said; they did not hear: \"{final}\"]";
             }
+            // Nothing of it went out before the call ended, and nothing is said once it has (SpeakAsync)
+            if (_ended && !_streamedThisTurn && final.Length > 0)
+            {
+                return $"[the call ended before this was said; the caller did not hear: \"{final}\"]";
+            }
             if (!_cutInMidReply || final.Length == 0)
             {
                 return content;
@@ -1203,14 +1295,15 @@ await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true),
 
     /// <param name="canSpeak">Extra condition, checked under <see cref="_sync"/> once the send lock is held</param>
     /// <param name="keepUninterrupted">The caller must hear all of it, so it plays to the end whoever speaks over it</param>
-    private async Task SpeakAsync(string? text, Func<bool>? canSpeak = null, bool keepUninterrupted = false)
+    /// <returns>Whether it went out</returns>
+    private async Task<bool> SpeakAsync(string? text, Func<bool>? canSpeak = null, bool keepUninterrupted = false)
     {
         if (string.IsNullOrWhiteSpace(text))
         {
-            return;
+            return false;
         }
 
-        await SpeakIfAsync(() =>
+        return await SpeakIfAsync(() =>
             {
                 if (_ended || !(canSpeak?.Invoke() ?? true))
                 {