Add support for making Text-To-Speech send regular text messages instead of notices

When speech-to-text/flow-type = `only_transcribe`, the bot will now send
text messages by default, not notices.

While notice messages may be less desirable with other bots in the room,
it's probably a better default for most people who enable "transcribe-only" mode.

This is an improvement related to https://github.com/etkecc/baibot/issues/14
This commit is contained in:
Slavi Pantaleev
2025-02-26 13:39:43 +02:00
parent e4e1fe0e7b
commit a1bd292752
23 changed files with 355 additions and 32 deletions

View File

@@ -4,7 +4,8 @@ use super::globalconfig::GlobalConfig;
use super::roomconfig::RoomConfig;
use crate::entity::roomconfig::{
defaults as roomconfig_defaults, SpeechToTextFlowType, TextGenerationAutoUsage,
defaults as roomconfig_defaults, SpeechToTextFlowType,
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage,
TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType,
};
@@ -37,6 +38,24 @@ impl RoomConfigContext {
.unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE)
}
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(
&self,
) -> SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
self.room_config
.settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages
.or({
self.global_config
.fallback_room_settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages
})
.unwrap_or(
roomconfig_defaults::SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE,
)
}
pub fn speech_to_text_language(&self) -> Option<String> {
self.room_config
.settings

View File

@@ -1,4 +1,4 @@
use super::SpeechToTextFlowType;
use super::{SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages};
use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
@@ -15,3 +15,9 @@ pub const TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE: TextToSpeechUserMessagesFlowTy
pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType =
SpeechToTextFlowType::TranscribeAndGenerateText;
// While notice messages may be less desirable with other bots in the room,
// it's probably a better default for most people who enable "transcribe-only" mode.
pub const SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE:
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages =
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text;

View File

@@ -13,7 +13,9 @@ mod text_generation;
mod text_to_speech;
pub use handler::RoomSettingsHandler;
pub use speech_to_text::SpeechToTextFlowType;
pub use speech_to_text::{
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
};
pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};

View File

@@ -4,6 +4,20 @@ use serde::{Deserialize, Serialize};
pub struct RoomSettingsSpeechToText {
pub flow_type: Option<SpeechToTextFlowType>,
/// Controls how the transcribed message is posted when dealing with:
/// - messages that only get transcribed (and do not trigger text-generation).
/// See `flow_type` and `SpeechToTextFlowType::OnlyTranscribe` for more details.
/// - incoming voice messages that are not part of a thread.
/// For messages that are part of a thread, we need to reply within the thread in a way (with a notice message)
/// that won't confuse the bot later, so we have no choice but to use a notice message.
///
/// Text-generation may happen either as a direct result of the incoming voice message or as part of a threaded conversation.
/// Transcribed messages should not be attributed to the bot for the purposes of text-generation,
/// so any time there's a chance of text-generation happening, we should use `SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice`
/// and optionally prefix the message with `> 🦻`.
pub msg_type_for_non_threaded_only_transcribed_messages:
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
/// The language of the input audio.
/// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency.
pub language: Option<String>,
@@ -57,3 +71,39 @@ impl std::fmt::Display for SpeechToTextFlowType {
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)]
pub enum SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
/// Send the transcribed message text as a regular message
#[serde(rename = "text")]
Text,
/// Send the transcribed message text as a notice message
#[serde(rename = "notice")]
Notice,
}
impl SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
pub fn choices() -> Vec<Self> {
vec![Self::Text, Self::Notice]
}
pub fn from_str(s: &str) -> Option<Self> {
match s {
"text" => Some(Self::Text),
"notice" => Some(Self::Notice),
_ => None,
}
}
}
impl std::fmt::Display for SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => write!(f, "text"),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
write!(f, "notice")
}
}
}
}

View File

@@ -70,6 +70,8 @@ pub enum TextGenerationAutoUsage {
/// Text Generation is to be performed when the original message was sent as audio (voice).
/// The voice message would be transcribed to text (subject to other configuration)
/// and text generation would be triggered.
///
/// Also see `SpeechToTextFlowType`.
#[serde(rename = "only_for_voice")]
OnlyForVoice,

View File

@@ -5,8 +5,9 @@ use mxlink::helpers::account_data_config::RoomConfigManager as AccountDataRoomCo
pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler};
pub use entity::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType,
};
pub type RoomConfigurationManager =