From b71e22cd6608d2c9f06aa02efccfb2549e69656d Mon Sep 17 00:00:00 2001 From: "mars.yu" <18671933+yuyixg@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:21:30 +0800 Subject: [PATCH] feat(twilio): a flow can make the relay wait quietly while the caller is busy on their phone A phone flow that asks the caller to do something on their phone (check a text arrived, fill in a form it texted them) expects silence, but the idle prompts read it as a dead line: "Sorry, I didn't catch that." five seconds in, a hang-up some twenty-five seconds in, before the flow could say what was still due (a confirmation number). Two conversation states, read as each turn's reply comes in (a reply with nothing to say still updates them): - twilio_relay_quiet_wait: "true" while silence is expected. The idle watcher then says only a soft check-in (prompt quiet-wait-check-in, "I'm still here whenever you're ready.") each QuietWaitSeconds (60), and hangs up after MaxQuietWaitPrompts (4) of them; both settings can be set per agent. - twilio_relay_goodbye_on_silence: what is said as the session hangs up on a caller who stayed quiet, in place of the idle-goodbye prompt, and said even when IdleAsksAgain is off. Without either state the idle prompts are as before. Co-Authored-By: Claude Opus 5.5 --- .../Models/AssistantMessage.cs | 12 +++++ .../Models/ConversationRelayLabels.cs | 5 +- .../Models/ConversationRelayStates.cs | 15 ++++++ .../Services/ConversationRelaySession.cs | 48 ++++++++++++++++--- .../Services/TwilioConversationProcessor.cs | 5 +- .../Settings/ConversationRelayAgentSetting.cs | 2 + .../Settings/ConversationRelaySetting.cs | 14 ++++++ 7 files changed, 91 insertions(+), 10 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 87f3e4377..ed497a3e3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -13,5 +13,17 @@ public class AssistantMessage /// What of the reply is said once the turn is over must play to the end, as the flow asked when the reply came in /// public bool KeepUninterrupted { get; set; } + + /// + /// The caller is busy on their phone and silence is expected (), as + /// the flow had it when the reply came in + /// + public bool QuietWait { get; set; } + + /// + /// What to say when hanging up on a caller who stays quiet from now on, in place of the idle-goodbye prompt + /// (); null for the prompt + /// + public string? GoodbyeOnSilence { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayLabels.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayLabels.cs index 9214331a1..1ffae7995 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayLabels.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayLabels.cs @@ -13,8 +13,9 @@ public static class ConversationRelayLabels /// /// Said to a caller gone quiet: the last question asked again ("Sorry, I didn't catch that. ..."), "Are you still - /// there?", or "Sorry, can you say it again?". It checks the caller is there, so it is no line of the agent's to say - /// again when they ask what was said. + /// there?", or "Sorry, can you say it again?"; while the flow waits quietly, the soft check-in ("I'm still here + /// whenever you're ready."). It checks the caller is there, so it is no line of the agent's to say again when they ask + /// what was said. /// public const string IdlePrompt = "twilio_relay_idle_prompt"; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs index a742862b7..07e57503f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs @@ -17,4 +17,19 @@ public static class ConversationRelayStates /// that text whole, so this is the only sign. Set by the session at the start of every turn. /// public const string LastReplyCutOff = "twilio_relay_last_reply_cut_off"; + + /// + /// "true" while the caller is busy with something the agent asked them to do on their phone, such as filling in a form + /// it texted them, and the agent waits quietly: silence is expected, so instead of the idle prompts the caller hears a + /// soft check-in (prompt quiet-wait-check-in) each QuietWaitSeconds, and the call is hung up only after + /// MaxQuietWaitPrompts of them. Read when a turn's reply is in; a flow clears it once the wait is over. + /// + public const string QuietWait = "twilio_relay_quiet_wait"; + + /// + /// What is said as the session hangs up on a caller who stayed quiet, in place of the idle-goodbye prompt: a flow sets + /// it while something is still to be told, such as the confirmation number of a request already made, and clears it + /// once that is said. Read when a turn's reply is in. + /// + public const string GoodbyeOnSilence = "twilio_relay_goodbye_on_silence"; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index a323d7abc..c5f7a689c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -20,6 +20,7 @@ public class ConversationRelaySession private const string DefaultAskAgainText = "Sorry, I didn't catch that. {0}"; private const string DefaultStillThereText = "Are you still there?"; private const string DefaultIdleGoodbyeText = "It sounds like we got disconnected. Please call back anytime."; + private const string DefaultQuietWaitCheckInText = "I'm still here whenever you're ready."; private const double SpokenCharsPerSecond = 15; private static readonly List DefaultWatchdogPrompts = ["hold-on-1", "hold-on-2", "hold-on-3", "hold-on-4", "hold-on-5"]; private static readonly List DefaultWatchdogLongPrompts = Enumerable.Range(1, 10).Select(i => $"hold-on-long-{i}").ToList(); @@ -95,6 +96,11 @@ public class ConversationRelaySession // turns: until a turn has read it, the last turn's answer is the best guess. private bool _keptUninterrupted; + // What the flow said, as the last reply came in, about a caller going quiet (ConversationRelayStates.QuietWait and + // GoodbyeOnSilence): kept until a later reply says otherwise, as a turn that says nothing still reads them + private bool _quietWait; + private string? _goodbyeOnSilence; + // The caller went on talking before the current turn said anything: its reply is dropped, and their words so far // are answered together in the next turn private bool _superseded; @@ -638,6 +644,16 @@ private async Task ProcessPendingAsync() } await CloseUtteranceAsync(); + if (reply != null) + { + // Before a reply with nothing to say is passed over: a flow waiting quietly answers the caller's asides + // with nothing + lock (_sync) + { + _quietWait = reply.QuietWait; + _goodbyeOnSilence = reply.GoodbyeOnSilence; + } + } if (reply == null || string.IsNullOrWhiteSpace(reply.Content) && !reply.ConversationEnd && !reply.HumanIntervationNeeded) { @@ -823,22 +839,26 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) var activity = 0; var stage = 0; string? question = null; + var quietWait = false; + string? goodbyeOnSilence = null; lock (_sync) { var quietFor = DateTime.UtcNow - _quietSinceUtc; if (_ended || _isProcessing || _quietSinceUtc == DateTime.MaxValue || _resumeLine != null - || quietFor < IdleTimeout(_idlePrompts)) + || quietFor < IdleTimeout(_idlePrompts, _quietWait)) { continue; } stage = _idlePrompts; - if (_idlePrompts < relay.MaxIdlePrompts) + quietWait = _quietWait; + goodbyeOnSilence = _goodbyeOnSilence; + if (_idlePrompts < (quietWait ? relay.MaxQuietWaitPrompts : relay.MaxIdlePrompts)) { _idlePrompts++; askAgain = true; activity = _activity; - question = relay.IdleAsksAgain && stage == 0 ? ConversationRelayLines.LastQuestion(_lastLine) : null; + question = !quietWait && relay.IdleAsksAgain && stage == 0 ? ConversationRelayLines.LastQuestion(_lastLine) : null; } else { @@ -852,7 +872,15 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) // Rechecked once the send lock is held: caller input arriving meanwhile starts a turn, and the prompt must not // then be said over or after that turn's reply bool StillSilent() => !_isProcessing && _activity == activity; - if (askAgain && question != null) + // The caller is busy on their phone: nothing to say again and no "Are you still there?", only that the agent + // is still on the line. Why: the previous build read that silence as a dead call and hung up on callers + // filling in the form it had texted them + if (askAgain && quietWait) + { + await SayOwnLineAsync(PromptText("quiet-wait-check-in", DefaultQuietWaitCheckInText), ConversationRelayLabels.IdlePrompt, + StillSilent); + } + else if (askAgain && question != null) { await SayOwnLineAsync(PromptText("say-it-again-question", DefaultAskAgainText).Replace("{0}", question), ConversationRelayLabels.IdlePrompt, StillSilent); @@ -873,11 +901,13 @@ await SayOwnLineAsync(PromptText($"say-it-again-{Random.Shared.Next(1, 5)}", Def _logger.LogInformation($"Caller stayed silent, hanging up conversation {_conversationId}."); string? goodbye = null; var goodbyeUtc = DateTime.MinValue; - if (relay.IdleAsksAgain) + // The flow's own goodbye is said whether or not the idle prompts say one: it tells what the caller still has + // to hear, such as a confirmation number + if (goodbyeOnSilence != null || relay.IdleAsksAgain) { // The session is marked ended already, so nothing else can be said after it, and nothing the caller // says over it is answered: it plays to the end - goodbye = PromptText("idle-goodbye", DefaultIdleGoodbyeText); + goodbye = goodbyeOnSilence ?? PromptText("idle-goodbye", DefaultIdleGoodbyeText); await SendAsync(TextFrame(goodbye, last: true, keepUninterrupted: true)); goodbyeUtc = DateTime.UtcNow; } @@ -892,8 +922,12 @@ await SayOwnLineAsync(PromptText($"say-it-again-{Random.Shared.Next(1, 5)}", Def } // How long the caller may stay silent before the idle prompt after this many (the last is before the hang-up) - private TimeSpan IdleTimeout(int prompts) + private TimeSpan IdleTimeout(int prompts, bool quietWait) { + if (quietWait) + { + return TimeSpan.FromSeconds(_relay.QuietWaitSeconds); + } var timeouts = _relay.IdleTimeouts; return TimeSpan.FromSeconds(timeouts.Count > 0 ? timeouts[Math.Min(prompts, timeouts.Count - 1)] : _relay.IdleTimeoutSeconds); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs index 3681ee0ae..8705b29c7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs @@ -167,7 +167,10 @@ await KeepSaidLines(sp, message.AgentId, message.ConversationId, inputMsg.Messag HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), Content = spoken, MessageId = msg.MessageId, - KeepUninterrupted = callbacks?.KeepUninterrupted?.Invoke() ?? false + KeepUninterrupted = callbacks?.KeepUninterrupted?.Invoke() ?? false, + QuietWait = conv.States.GetState(ConversationRelayStates.QuietWait) == "true", + GoodbyeOnSilence = conv.States.GetState(ConversationRelayStates.GoodbyeOnSilence) is { Length: > 0 } goodbye + ? goodbye : null }; } ); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs index 3e6140ae7..67a9823c5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs @@ -19,6 +19,8 @@ public class ConversationRelayAgentSetting public int? MaxIdlePrompts { get; set; } public List? IdleTimeouts { get; set; } public bool? IdleAsksAgain { get; set; } + public double? QuietWaitSeconds { get; set; } + public int? MaxQuietWaitPrompts { get; set; } public double? ResumeAfterCutInSeconds { get; set; } public bool? DropBackchannelTails { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index f4b34420b..5ce5fb7c5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -125,6 +125,18 @@ public class ConversationRelaySetting /// public bool IdleAsksAgain { get; set; } + /// + /// Seconds of caller silence before each soft check-in ("I'm still here whenever you're ready.", prompt + /// quiet-wait-check-in) while a flow waits for the caller to do something on their phone + /// (); the last one is before hanging up + /// + public double QuietWaitSeconds { get; set; } = 60; + + /// + /// How many soft check-ins a caller who stays silent while the flow waits quietly hears before the call is hung up + /// + public int MaxQuietWaitPrompts { get; set; } = 4; + /// /// Drop a caller's words that came in before they could hear the line just said (TooSoonToAnswer) when they only /// acknowledge ("I agree.", "Okay.") and that line asks something: the tail of what they said before, which the @@ -156,6 +168,8 @@ public ConversationRelaySetting For(string? agentId) settings.MaxIdlePrompts = agent.MaxIdlePrompts ?? MaxIdlePrompts; settings.IdleTimeouts = agent.IdleTimeouts ?? IdleTimeouts; settings.IdleAsksAgain = agent.IdleAsksAgain ?? IdleAsksAgain; + settings.QuietWaitSeconds = agent.QuietWaitSeconds ?? QuietWaitSeconds; + settings.MaxQuietWaitPrompts = agent.MaxQuietWaitPrompts ?? MaxQuietWaitPrompts; settings.ResumeAfterCutInSeconds = agent.ResumeAfterCutInSeconds ?? ResumeAfterCutInSeconds; settings.DropBackchannelTails = agent.DropBackchannelTails ?? DropBackchannelTails; return settings;