Add support for making Text-To-Speech send regular text messages instead of notices

When speech-to-text/flow-type = `only_transcribe`, the bot will now send
text messages by default, not notices.

While notice messages may be less desirable with other bots in the room,
it's probably a better default for most people who enable "transcribe-only" mode.

This is an improvement related to https://github.com/etkecc/baibot/issues/14
This commit is contained in:
Slavi Pantaleev
2025-02-26 13:39:43 +02:00
parent e4e1fe0e7b
commit a1bd292752
23 changed files with 355 additions and 32 deletions

View File

@@ -23,6 +23,19 @@ The following configuration values are recognized:
Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
### 🪄 Message Type for non-threaded only-transcribed messages
Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`.
The following configuration values are recognized:
- (default) `text`: the transcribed text is sent as a regular message. This is more convenient if you'd like to forward the transcribed message to other rooms.
- `notice`: the transcribed text is sent as a notice message. This provides better compatibility with other bots in the room, as they are less likely to interact with messages of type notice.
Example: `!bai config room speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages notice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
### 🔤 Language ### 🔤 Language
Lets you specify the language of the input voice messages, to avoid using auto-detection. Lets you specify the language of the input voice messages, to avoid using auto-detection.

View File

@@ -136,6 +136,8 @@ To operate in this mode, you can:
- adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe` - adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe`
- optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
### 🖌️ Image Generation ### 🖌️ Image Generation

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 22 KiB

View File

@@ -1,7 +1,8 @@
use crate::{ use crate::{
agent::{AgentPurpose, PublicIdentifier}, agent::{AgentPurpose, PublicIdentifier},
entity::roomconfig::{ entity::roomconfig::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
}, },
}; };
@@ -54,6 +55,11 @@ pub enum ConfigSpeechToTextSettingRelatedControllerType {
GetFlowType, GetFlowType,
SetFlowType(Option<SpeechToTextFlowType>), SetFlowType(Option<SpeechToTextFlowType>),
GetMsgTypeForNonThreadedOnlyTranscribedMessages,
SetMsgTypeForNonThreadedOnlyTranscribedMessages(
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
),
GetLanguage, GetLanguage,
SetLanguage(Option<String>), SetLanguage(Option<String>),
} }

View File

@@ -1,7 +1,13 @@
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
use crate::{controller::ControllerType, entity::roomconfig::SpeechToTextFlowType, strings}; use crate::{
controller::ControllerType,
entity::roomconfig::{
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
},
strings,
};
use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType; use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType;
@@ -48,6 +54,53 @@ pub(super) fn determine(
)); ));
} }
// msg_type_for_non_threaded_only_transcribed_messages
if let Some(remaining_text) =
text.strip_prefix("msg-type-for-non-threaded-only-transcribed-messages")
{
let remaining_text = remaining_text.trim();
if !remaining_text.is_empty() {
return Err(ControllerType::Error(
strings::cfg::configuration_getter_used_with_extra_text(
"msg-type-for-non-threaded-only-transcribed-messages",
remaining_text,
)
.to_owned(),
));
}
return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages);
}
if let Some(value_string) =
text.strip_prefix("set-msg-type-for-non-threaded-only-transcribed-messages")
{
let value_string = value_string.trim().to_owned();
let value_choice = if value_string.is_empty() {
None
} else {
let value_choice =
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::from_str(
&value_string.to_lowercase(),
);
if value_choice.is_none() {
return Err(ControllerType::Error(
strings::cfg::configuration_value_unrecognized(&value_string).to_owned(),
));
}
value_choice
};
return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(
value_choice,
));
}
// Language // Language
if let Some(remaining_text) = text.strip_prefix("language") { if let Some(remaining_text) = text.strip_prefix("language") {

View File

@@ -1,4 +1,7 @@
use crate::entity::roomconfig::{RoomSettings, SpeechToTextFlowType}; use crate::entity::roomconfig::{
RoomSettings, SpeechToTextFlowType,
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
};
use crate::{entity::MessageContext, Bot}; use crate::{entity::MessageContext, Bot};
use super::super::controller_type::{ use super::super::controller_type::{
@@ -52,6 +55,39 @@ pub(super) async fn dispatch(
} }
} }
ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages => {
let value = &room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages;
setting_get::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(bot, message_context, value).await
}
ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(value) => {
let value = value.to_owned();
let setter_callback = Box::new(move |room_settings: &mut RoomSettings| {
room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages = value;
});
match config_type {
SettingsStorageSource::Room => {
room_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
bot,
message_context,
&value,
setter_callback,
)
.await
}
SettingsStorageSource::Global => {
global_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
bot,
message_context,
&value,
setter_callback,
)
.await
}
}
}
ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => { ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => {
let value = &room_settings.speech_to_text.language; let value = &room_settings.speech_to_text.language;
setting_get::<String>(bot, message_context, value).await setting_get::<String>(bot, message_context, value).await

View File

@@ -3,7 +3,8 @@ use mxlink::MessageResponseType;
use crate::{ use crate::{
entity::{ entity::{
roomconfig::{ roomconfig::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
}, },
MessageContext, MessageContext,
@@ -346,6 +347,46 @@ fn build_section_speech_to_text(command_prefix: &str) -> String {
)); ));
message.push_str("\n\n"); message.push_str("\n\n");
// Msg Type For Non Threaded Only Transcribed Messages
message.push_str(&format!(
"#### {}",
strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading()
));
message.push_str("\n\n");
message.push_str(strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro());
message.push('\n');
message.push_str(
&strings::help::cfg::the_following_configuration_values_are_recognized(
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::choices(),
),
);
message.push_str("\n\n");
message.push_str(&format!(
"- {}",
&strings::help::cfg::current_setting_show(
command_prefix,
"speech-to-text msg-type-for-non-threaded-only-transcribed-messages"
)
));
message.push('\n');
message.push_str(&format!(
"- {}",
&strings::help::cfg::current_setting_set(
command_prefix,
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages VALUE"
)
));
message.push('\n');
message.push_str(&format!(
"- {}",
&strings::help::cfg::current_setting_unset(
command_prefix,
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages"
)
));
message.push_str("\n\n");
// Language // Language
message.push_str(&format!( message.push_str(&format!(

View File

@@ -507,6 +507,35 @@ async fn generate_speech_to_text_section(
flow_type_set_where, flow_type_set_where,
)); ));
// Msg Type For Non Threaded Only Transcribed Messages
let effective_msg_type_for_non_threaded_only_transcribed_messages =
room_config_context.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
let room_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
.room_config
.settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages;
let global_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
.global_config
.fallback_room_settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages;
let msg_type_for_non_threaded_only_transcribed_messages_set_where =
if room_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
strings::cfg::status_badge_set_in_room_config()
} else if global_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
strings::cfg::status_badge_set_in_global_config()
} else {
strings::cfg::status_badge_using_hardcoded_default()
};
message.push_str(&strings::cfg::status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
effective_msg_type_for_non_threaded_only_transcribed_messages,
msg_type_for_non_threaded_only_transcribed_messages_set_where,
));
// Language // Language
let effective_language = room_config_context.speech_to_text_language(); let effective_language = room_config_context.speech_to_text_language();

View File

@@ -13,7 +13,8 @@ use crate::agent::ControllerTrait;
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain; use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
use crate::conversation::matrix::MatrixMessageProcessingParams; use crate::conversation::matrix::MatrixMessageProcessingParams;
use crate::entity::roomconfig::{ use crate::entity::roomconfig::{
SpeechToTextFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
}; };
use crate::entity::MessagePayload; use crate::entity::MessagePayload;
use crate::strings; use crate::strings;
@@ -71,21 +72,35 @@ pub async fn handle(
if let MessagePayload::Audio(audio_content) = &message_context.payload() { if let MessagePayload::Audio(audio_content) = &message_context.payload() {
original_message_is_audio = true; original_message_is_audio = true;
let response_type = match speech_to_text_flow_type { let (response_type, msg_type) = match speech_to_text_flow_type {
SpeechToTextFlowType::Ignore => { SpeechToTextFlowType::Ignore => {
tracing::debug!("Intentionally ignoring audio message"); tracing::debug!("Intentionally ignoring audio message");
return Ok(()); return Ok(());
} }
SpeechToTextFlowType::TranscribeAndGenerateText => { SpeechToTextFlowType::TranscribeAndGenerateText => {
tracing::debug!("Will be transcribing and possibly generating text.."); tracing::debug!("Will be transcribing and possibly generating text..");
MessageResponseType::InThread(message_context.thread_info().clone()) (
MessageResponseType::InThread(message_context.thread_info().clone()),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
)
} }
SpeechToTextFlowType::OnlyTranscribe => { SpeechToTextFlowType::OnlyTranscribe => {
tracing::debug!("Will only be transcribing audio to text.."); tracing::debug!("Will only be transcribing audio to text..");
if message_context.thread_info().is_thread_root_only() { if message_context.thread_info().is_thread_root_only() {
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()) let msg_type = message_context
.room_config_context()
.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
(
MessageResponseType::Reply(
message_context.thread_info().root_event_id.clone(),
),
msg_type,
)
} else { } else {
MessageResponseType::InThread(message_context.thread_info().clone()) (
MessageResponseType::InThread(message_context.thread_info().clone()),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
)
} }
} }
}; };
@@ -94,8 +109,14 @@ pub async fn handle(
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await); _typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
} }
let Some(speech_to_text_created_event_id_result) = let Some(speech_to_text_created_event_id_result) = handle_stage_speech_to_text(
handle_stage_speech_to_text(bot, message_context, audio_content, response_type).await bot,
message_context,
audio_content,
response_type,
msg_type,
)
.await
else { else {
return Ok(()); return Ok(());
}; };
@@ -282,6 +303,7 @@ async fn handle_stage_speech_to_text(
message_context: &MessageContext, message_context: &MessageContext,
audio_content: &AudioMessageEventContent, audio_content: &AudioMessageEventContent,
response_type: MessageResponseType, response_type: MessageResponseType,
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
) -> Option<OwnedEventId> { ) -> Option<OwnedEventId> {
let agent = get_effective_agent_for_purpose_or_complain( let agent = get_effective_agent_for_purpose_or_complain(
bot, bot,
@@ -312,6 +334,7 @@ async fn handle_stage_speech_to_text(
&agent, &agent,
audio_content, audio_content,
response_type.clone(), response_type.clone(),
msg_type,
) )
.await; .await;
@@ -572,6 +595,7 @@ async fn handle_stage_speech_to_text_actual_transcribing(
agent: &AgentInstance, agent: &AgentInstance,
audio_content: &AudioMessageEventContent, audio_content: &AudioMessageEventContent,
response_type: MessageResponseType, response_type: MessageResponseType,
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
) -> anyhow::Result<OwnedEventId> { ) -> anyhow::Result<OwnedEventId> {
let src = &audio_content.source; let src = &audio_content.source;
@@ -626,9 +650,6 @@ async fn handle_stage_speech_to_text_actual_transcribing(
// //
// When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead, // When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead,
// to make it clear to users that it's a transcription. // to make it clear to users that it's a transcription.
//
// Regardless of how we post this message, it will be posted as a notice,
// which can indicate to the bot (for potential future text-generation purposes) that this message is not a bot message.
let (transcribed_text, annotate_message_with_reaction) = let (transcribed_text, annotate_message_with_reaction) =
if let MessageResponseType::InThread(_) = response_type { if let MessageResponseType::InThread(_) = response_type {
( (
@@ -639,10 +660,22 @@ async fn handle_stage_speech_to_text_actual_transcribing(
(speech_to_text_result.text, true) (speech_to_text_result.text, true)
}; };
let result = bot let result = match msg_type {
.messaging() SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => {
.send_notice_markdown_no_fail(message_context.room(), transcribed_text, response_type) bot.messaging()
.await; .send_text_markdown_no_fail(message_context.room(), transcribed_text, response_type)
.await
}
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
bot.messaging()
.send_notice_markdown_no_fail(
message_context.room(),
transcribed_text,
response_type,
)
.await
}
};
let event_id = result let event_id = result
.map(|result| result.event_id) .map(|result| result.event_id)

View File

@@ -6,11 +6,11 @@ use crate::agent::provider::ImageGenerationParams;
use crate::agent::AgentPurpose; use crate::agent::AgentPurpose;
use crate::agent::ControllerTrait; use crate::agent::ControllerTrait;
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain; use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
use crate::controller::utils::mime::get_file_extension;
use crate::conversation::create_llm_conversation_for_matrix_thread; use crate::conversation::create_llm_conversation_for_matrix_thread;
use crate::conversation::matrix::MatrixMessageProcessingParams; use crate::conversation::matrix::MatrixMessageProcessingParams;
use crate::strings; use crate::strings;
use crate::{entity::MessageContext, Bot}; use crate::{entity::MessageContext, Bot};
use crate::controller::utils::mime::get_file_extension;
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded. // We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
const STICKER_SIZE: &str = "256x256"; const STICKER_SIZE: &str = "256x256";
@@ -82,7 +82,10 @@ pub async fn handle_image(
.await; .await;
} }
let attachment_body_text = format!("generated-image.{}", get_file_extension(&response.mime_type)); let attachment_body_text = format!(
"generated-image.{}",
get_file_extension(&response.mime_type)
);
let mut event_content = matrix_link let mut event_content = matrix_link
.media() .media()
@@ -158,7 +161,10 @@ pub async fn handle_sticker(
.instrument(span) .instrument(span)
.await?; .await?;
let attachment_body_text = format!("generated-sticker.{}", get_file_extension(&response.mime_type)); let attachment_body_text = format!(
"generated-sticker.{}",
get_file_extension(&response.mime_type)
);
let mut event_content = matrix_link let mut event_content = matrix_link
.media() .media()

View File

@@ -12,5 +12,6 @@ pub fn get_file_extension(mime_type: &mime::Mime) -> String {
(mime::IMAGE, mime::PNG) => "png", (mime::IMAGE, mime::PNG) => "png",
(mime::IMAGE, mime::SVG) => "svg", (mime::IMAGE, mime::SVG) => "svg",
_ => "bin", _ => "bin",
}.to_string() }
.to_string()
} }

View File

@@ -6,8 +6,8 @@ use crate::{
}; };
pub mod agent; pub mod agent;
pub mod text_to_speech;
pub(super) mod mime; pub(super) mod mime;
pub mod text_to_speech;
pub async fn get_text_body_or_complain<'a>( pub async fn get_text_body_or_complain<'a>(
bot: &Bot, bot: &Bot,

View File

@@ -118,7 +118,10 @@ async fn do_generate_and_send_tts_for_message(
} }
}; };
let attachment_body_text = format!("generated-speech.{}", get_file_extension(&text_to_speech_result.mime_type)); let attachment_body_text = format!(
"generated-speech.{}",
get_file_extension(&text_to_speech_result.mime_type)
);
let event_content = matrix_link let event_content = matrix_link
.media() .media()

View File

@@ -45,7 +45,7 @@ fn convert_bot_notice_message(
// Such transcriptions are prefixed with an emoji and blockquoted. // Such transcriptions are prefixed with an emoji and blockquoted.
// If we find a notice that doesn't match this pattern, we skip it. // If we find a notice that doesn't match this pattern, we skip it.
// //
// It should be noted that transcriptions are sometimes posted as regular notice messages which do not include // It should be noted that transcriptions are sometimes posted as regular notice (or even text) messages which do not include
// the `> 🦻` formatting. This function will not handle these properly. // the `> 🦻` formatting. This function will not handle these properly.
if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) { if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) {

View File

@@ -4,7 +4,8 @@ use super::globalconfig::GlobalConfig;
use super::roomconfig::RoomConfig; use super::roomconfig::RoomConfig;
use crate::entity::roomconfig::{ use crate::entity::roomconfig::{
defaults as roomconfig_defaults, SpeechToTextFlowType, TextGenerationAutoUsage, defaults as roomconfig_defaults, SpeechToTextFlowType,
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage,
TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType, TextToSpeechUserMessagesFlowType,
}; };
@@ -37,6 +38,24 @@ impl RoomConfigContext {
.unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE) .unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE)
} }
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(
&self,
) -> SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
self.room_config
.settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages
.or({
self.global_config
.fallback_room_settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages
})
.unwrap_or(
roomconfig_defaults::SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE,
)
}
pub fn speech_to_text_language(&self) -> Option<String> { pub fn speech_to_text_language(&self) -> Option<String> {
self.room_config self.room_config
.settings .settings

View File

@@ -1,4 +1,4 @@
use super::SpeechToTextFlowType; use super::{SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages};
use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType}; use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType}; use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
@@ -15,3 +15,9 @@ pub const TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE: TextToSpeechUserMessagesFlowTy
pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType = pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType =
SpeechToTextFlowType::TranscribeAndGenerateText; SpeechToTextFlowType::TranscribeAndGenerateText;
// While notice messages may be less desirable with other bots in the room,
// it's probably a better default for most people who enable "transcribe-only" mode.
pub const SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE:
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages =
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text;

View File

@@ -13,7 +13,9 @@ mod text_generation;
mod text_to_speech; mod text_to_speech;
pub use handler::RoomSettingsHandler; pub use handler::RoomSettingsHandler;
pub use speech_to_text::SpeechToTextFlowType; pub use speech_to_text::{
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
};
pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType}; pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType}; pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};

View File

@@ -4,6 +4,20 @@ use serde::{Deserialize, Serialize};
pub struct RoomSettingsSpeechToText { pub struct RoomSettingsSpeechToText {
pub flow_type: Option<SpeechToTextFlowType>, pub flow_type: Option<SpeechToTextFlowType>,
/// Controls how the transcribed message is posted when dealing with:
/// - messages that only get transcribed (and do not trigger text-generation).
/// See `flow_type` and `SpeechToTextFlowType::OnlyTranscribe` for more details.
/// - incoming voice messages that are not part of a thread.
/// For messages that are part of a thread, we need to reply within the thread in a way (with a notice message)
/// that won't confuse the bot later, so we have no choice but to use a notice message.
///
/// Text-generation may happen either as a direct result of the incoming voice message or as part of a threaded conversation.
/// Transcribed messages should not be attributed to the bot for the purposes of text-generation,
/// so any time there's a chance of text-generation happening, we should use `SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice`
/// and optionally prefix the message with `> 🦻`.
pub msg_type_for_non_threaded_only_transcribed_messages:
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
/// The language of the input audio. /// The language of the input audio.
/// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency. /// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency.
pub language: Option<String>, pub language: Option<String>,
@@ -57,3 +71,39 @@ impl std::fmt::Display for SpeechToTextFlowType {
} }
} }
} }
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)]
pub enum SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
/// Send the transcribed message text as a regular message
#[serde(rename = "text")]
Text,
/// Send the transcribed message text as a notice message
#[serde(rename = "notice")]
Notice,
}
impl SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
pub fn choices() -> Vec<Self> {
vec![Self::Text, Self::Notice]
}
pub fn from_str(s: &str) -> Option<Self> {
match s {
"text" => Some(Self::Text),
"notice" => Some(Self::Notice),
_ => None,
}
}
}
impl std::fmt::Display for SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => write!(f, "text"),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
write!(f, "notice")
}
}
}
}

View File

@@ -70,6 +70,8 @@ pub enum TextGenerationAutoUsage {
/// Text Generation is to be performed when the original message was sent as audio (voice). /// Text Generation is to be performed when the original message was sent as audio (voice).
/// The voice message would be transcribed to text (subject to other configuration) /// The voice message would be transcribed to text (subject to other configuration)
/// and text generation would be triggered. /// and text generation would be triggered.
///
/// Also see `SpeechToTextFlowType`.
#[serde(rename = "only_for_voice")] #[serde(rename = "only_for_voice")]
OnlyForVoice, OnlyForVoice,

View File

@@ -5,8 +5,9 @@ use mxlink::helpers::account_data_config::RoomConfigManager as AccountDataRoomCo
pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler}; pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler};
pub use entity::{ pub use entity::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType,
}; };
pub type RoomConfigurationManager = pub type RoomConfigurationManager =

View File

@@ -4,7 +4,8 @@ use crate::{
PublicIdentifier, PublicIdentifier,
}, },
entity::roomconfig::{ entity::roomconfig::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
}, },
utils::text::block_quote, utils::text::block_quote,
@@ -269,6 +270,16 @@ pub fn status_speech_to_text_entry_flow_type(
format!("- 🪄 Flow type: `{}` ({})\n", value, set_where) format!("- 🪄 Flow type: `{}` ({})\n", value, set_where)
} }
pub fn status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
value: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
set_where: &str,
) -> String {
format!(
"- 🪄 Message type for non-threaded only-transcribed messages: `{}` ({})\n",
value, set_where
)
}
pub fn status_speech_to_text_entry_language(value: Option<String>, set_where: &str) -> String { pub fn status_speech_to_text_entry_language(value: Option<String>, set_where: &str) -> String {
let formatted = match value { let formatted = match value {
Some(value) => format!("`{}` ({})", value, set_where), Some(value) => format!("`{}` ({})", value, set_where),

View File

@@ -200,6 +200,15 @@ pub fn speech_to_text_flow_type_intro() -> &'static str {
"Controls how voice messages are handled." "Controls how voice messages are handled."
} }
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading() -> &'static str
{
"🪄 Message Type for non-threaded only-transcribed messages"
}
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro() -> &'static str {
"Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`."
}
pub fn speech_to_text_language_heading() -> &'static str { pub fn speech_to_text_language_heading() -> &'static str {
"🔤 Language" "🔤 Language"
} }

View File

@@ -16,7 +16,7 @@ pub fn create_transcribed_message_text(text: &str) -> String {
/// If the provided text string does not match the expected format, None is returned. /// If the provided text string does not match the expected format, None is returned.
/// ///
/// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include /// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include
/// the `> 🦻` prefixing. This function will not handle these properly. /// the `> 🦻` prefix. This function will not handle these properly.
pub fn parse_transcribed_message_text(text: &str) -> Option<String> { pub fn parse_transcribed_message_text(text: &str) -> Option<String> {
if !text.starts_with("> ") { if !text.starts_with("> ") {
return None; return None;