unfunny/baibot-withmcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for making Text-To-Speech send regular text messages instead of notices

Browse Source 
When speech-to-text/flow-type = `only_transcribe`, the bot will now send
text messages by default, not notices.

While notice messages may be less desirable with other bots in the room,
it's probably a better default for most people who enable "transcribe-only" mode.

This is an improvement related to https://github.com/etkecc/baibot/issues/14
This commit is contained in:
Slavi Pantaleev
2025-02-26 13:39:43 +02:00
parent e4e1fe0e7b
commit a1bd292752
23 changed files with 355 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/controller/chat_completion/mod.rs
View File

@@ -13,7 +13,8 @@ use crate::agent::ControllerTrait;
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
use crate::conversation::matrix::MatrixMessageProcessingParams;
use crate::entity::roomconfig::{
SpeechToTextFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
};
use crate::entity::MessagePayload;
use crate::strings;
@@ -71,21 +72,35 @@ pub async fn handle(
if let MessagePayload::Audio(audio_content) = &message_context.payload() {
original_message_is_audio = true;
let response_type = match speech_to_text_flow_type {
let (response_type, msg_type) = match speech_to_text_flow_type {
SpeechToTextFlowType::Ignore => {
tracing::debug!("Intentionally ignoring audio message");
return Ok(());
}
SpeechToTextFlowType::TranscribeAndGenerateText => {
tracing::debug!("Will be transcribing and possibly generating text..");
MessageResponseType::InThread(message_context.thread_info().clone())
(
MessageResponseType::InThread(message_context.thread_info().clone()),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
)
}
SpeechToTextFlowType::OnlyTranscribe => {
tracing::debug!("Will only be transcribing audio to text..");
if message_context.thread_info().is_thread_root_only() {
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone())
let msg_type = message_context
.room_config_context()
.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
(
MessageResponseType::Reply(
message_context.thread_info().root_event_id.clone(),
),
msg_type,
)
} else {
MessageResponseType::InThread(message_context.thread_info().clone())
(
MessageResponseType::InThread(message_context.thread_info().clone()),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
)
}
}
};
@@ -94,8 +109,14 @@ pub async fn handle(
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
}
let Some(speech_to_text_created_event_id_result) =
handle_stage_speech_to_text(bot, message_context, audio_content, response_type).await
let Some(speech_to_text_created_event_id_result) = handle_stage_speech_to_text(
bot,
message_context,
audio_content,
response_type,
msg_type,
)
.await
else {
return Ok(());
};
@@ -282,6 +303,7 @@ async fn handle_stage_speech_to_text(
message_context: &MessageContext,
audio_content: &AudioMessageEventContent,
response_type: MessageResponseType,
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
) -> Option<OwnedEventId> {
let agent = get_effective_agent_for_purpose_or_complain(
bot,
@@ -312,6 +334,7 @@ async fn handle_stage_speech_to_text(
&agent,
audio_content,
response_type.clone(),
msg_type,
)
.await;
@@ -572,6 +595,7 @@ async fn handle_stage_speech_to_text_actual_transcribing(
agent: &AgentInstance,
audio_content: &AudioMessageEventContent,
response_type: MessageResponseType,
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
) -> anyhow::Result<OwnedEventId> {
let src = &audio_content.source;
@@ -626,9 +650,6 @@ async fn handle_stage_speech_to_text_actual_transcribing(
//
// When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead,
// to make it clear to users that it's a transcription.
//
// Regardless of how we post this message, it will be posted as a notice,
// which can indicate to the bot (for potential future text-generation purposes) that this message is not a bot message.
let (transcribed_text, annotate_message_with_reaction) =
if let MessageResponseType::InThread(_) = response_type {
(
@@ -639,10 +660,22 @@ async fn handle_stage_speech_to_text_actual_transcribing(
(speech_to_text_result.text, true)
};
let result = bot
.messaging()
.send_notice_markdown_no_fail(message_context.room(), transcribed_text, response_type)
.await;
let result = match msg_type {
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => {
bot.messaging()
.send_text_markdown_no_fail(message_context.room(), transcribed_text, response_type)
.await
}
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
bot.messaging()
.send_notice_markdown_no_fail(
message_context.room(),
transcribed_text,
response_type,
)
.await
}
};
let event_id = result
.map(|result| result.event_id)