Add support for making Text-To-Speech send regular text messages instead of notices
When speech-to-text/flow-type = `only_transcribe`, the bot will now send text messages by default, not notices. While notice messages may be less desirable with other bots in the room, it's probably a better default for most people who enable "transcribe-only" mode. This is an improvement related to https://github.com/etkecc/baibot/issues/14
This commit is contained in:
@@ -23,6 +23,19 @@ The following configuration values are recognized:
|
||||
Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
|
||||
|
||||
|
||||
### 🪄 Message Type for non-threaded only-transcribed messages
|
||||
|
||||
Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`.
|
||||
|
||||
The following configuration values are recognized:
|
||||
|
||||
- (default) `text`: the transcribed text is sent as a regular message. This is more convenient if you'd like to forward the transcribed message to other rooms.
|
||||
|
||||
- `notice`: the transcribed text is sent as a notice message. This provides better compatibility with other bots in the room, as they are less likely to interact with messages of type notice.
|
||||
|
||||
Example: `!bai config room speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages notice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
|
||||
|
||||
|
||||
### 🔤 Language
|
||||
|
||||
Lets you specify the language of the input voice messages, to avoid using auto-detection.
|
||||
|
||||
@@ -136,6 +136,8 @@ To operate in this mode, you can:
|
||||
|
||||
- adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe`
|
||||
|
||||
- optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
|
||||
|
||||
|
||||
### 🖌️ Image Generation
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 22 KiB |
@@ -1,7 +1,8 @@
|
||||
use crate::{
|
||||
agent::{AgentPurpose, PublicIdentifier},
|
||||
entity::roomconfig::{
|
||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
},
|
||||
};
|
||||
@@ -54,6 +55,11 @@ pub enum ConfigSpeechToTextSettingRelatedControllerType {
|
||||
GetFlowType,
|
||||
SetFlowType(Option<SpeechToTextFlowType>),
|
||||
|
||||
GetMsgTypeForNonThreadedOnlyTranscribedMessages,
|
||||
SetMsgTypeForNonThreadedOnlyTranscribedMessages(
|
||||
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
|
||||
),
|
||||
|
||||
GetLanguage,
|
||||
SetLanguage(Option<String>),
|
||||
}
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use crate::{controller::ControllerType, entity::roomconfig::SpeechToTextFlowType, strings};
|
||||
use crate::{
|
||||
controller::ControllerType,
|
||||
entity::roomconfig::{
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
},
|
||||
strings,
|
||||
};
|
||||
|
||||
use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType;
|
||||
|
||||
@@ -48,6 +54,53 @@ pub(super) fn determine(
|
||||
));
|
||||
}
|
||||
|
||||
// msg_type_for_non_threaded_only_transcribed_messages
|
||||
|
||||
if let Some(remaining_text) =
|
||||
text.strip_prefix("msg-type-for-non-threaded-only-transcribed-messages")
|
||||
{
|
||||
let remaining_text = remaining_text.trim();
|
||||
|
||||
if !remaining_text.is_empty() {
|
||||
return Err(ControllerType::Error(
|
||||
strings::cfg::configuration_getter_used_with_extra_text(
|
||||
"msg-type-for-non-threaded-only-transcribed-messages",
|
||||
remaining_text,
|
||||
)
|
||||
.to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages);
|
||||
}
|
||||
|
||||
if let Some(value_string) =
|
||||
text.strip_prefix("set-msg-type-for-non-threaded-only-transcribed-messages")
|
||||
{
|
||||
let value_string = value_string.trim().to_owned();
|
||||
|
||||
let value_choice = if value_string.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let value_choice =
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::from_str(
|
||||
&value_string.to_lowercase(),
|
||||
);
|
||||
|
||||
if value_choice.is_none() {
|
||||
return Err(ControllerType::Error(
|
||||
strings::cfg::configuration_value_unrecognized(&value_string).to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
value_choice
|
||||
};
|
||||
|
||||
return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(
|
||||
value_choice,
|
||||
));
|
||||
}
|
||||
|
||||
// Language
|
||||
|
||||
if let Some(remaining_text) = text.strip_prefix("language") {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use crate::entity::roomconfig::{RoomSettings, SpeechToTextFlowType};
|
||||
use crate::entity::roomconfig::{
|
||||
RoomSettings, SpeechToTextFlowType,
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
};
|
||||
use crate::{entity::MessageContext, Bot};
|
||||
|
||||
use super::super::controller_type::{
|
||||
@@ -52,6 +55,39 @@ pub(super) async fn dispatch(
|
||||
}
|
||||
}
|
||||
|
||||
ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages => {
|
||||
let value = &room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages;
|
||||
setting_get::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(bot, message_context, value).await
|
||||
}
|
||||
ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(value) => {
|
||||
let value = value.to_owned();
|
||||
|
||||
let setter_callback = Box::new(move |room_settings: &mut RoomSettings| {
|
||||
room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages = value;
|
||||
});
|
||||
|
||||
match config_type {
|
||||
SettingsStorageSource::Room => {
|
||||
room_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
|
||||
bot,
|
||||
message_context,
|
||||
&value,
|
||||
setter_callback,
|
||||
)
|
||||
.await
|
||||
}
|
||||
SettingsStorageSource::Global => {
|
||||
global_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
|
||||
bot,
|
||||
message_context,
|
||||
&value,
|
||||
setter_callback,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => {
|
||||
let value = &room_settings.speech_to_text.language;
|
||||
setting_get::<String>(bot, message_context, value).await
|
||||
|
||||
@@ -3,7 +3,8 @@ use mxlink::MessageResponseType;
|
||||
use crate::{
|
||||
entity::{
|
||||
roomconfig::{
|
||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
},
|
||||
MessageContext,
|
||||
@@ -346,6 +347,46 @@ fn build_section_speech_to_text(command_prefix: &str) -> String {
|
||||
));
|
||||
message.push_str("\n\n");
|
||||
|
||||
// Msg Type For Non Threaded Only Transcribed Messages
|
||||
|
||||
message.push_str(&format!(
|
||||
"#### {}",
|
||||
strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading()
|
||||
));
|
||||
message.push_str("\n\n");
|
||||
message.push_str(strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro());
|
||||
message.push('\n');
|
||||
message.push_str(
|
||||
&strings::help::cfg::the_following_configuration_values_are_recognized(
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::choices(),
|
||||
),
|
||||
);
|
||||
message.push_str("\n\n");
|
||||
message.push_str(&format!(
|
||||
"- {}",
|
||||
&strings::help::cfg::current_setting_show(
|
||||
command_prefix,
|
||||
"speech-to-text msg-type-for-non-threaded-only-transcribed-messages"
|
||||
)
|
||||
));
|
||||
message.push('\n');
|
||||
message.push_str(&format!(
|
||||
"- {}",
|
||||
&strings::help::cfg::current_setting_set(
|
||||
command_prefix,
|
||||
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages VALUE"
|
||||
)
|
||||
));
|
||||
message.push('\n');
|
||||
message.push_str(&format!(
|
||||
"- {}",
|
||||
&strings::help::cfg::current_setting_unset(
|
||||
command_prefix,
|
||||
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages"
|
||||
)
|
||||
));
|
||||
message.push_str("\n\n");
|
||||
|
||||
// Language
|
||||
|
||||
message.push_str(&format!(
|
||||
|
||||
@@ -507,6 +507,35 @@ async fn generate_speech_to_text_section(
|
||||
flow_type_set_where,
|
||||
));
|
||||
|
||||
// Msg Type For Non Threaded Only Transcribed Messages
|
||||
|
||||
let effective_msg_type_for_non_threaded_only_transcribed_messages =
|
||||
room_config_context.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
|
||||
let room_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
|
||||
.room_config
|
||||
.settings
|
||||
.speech_to_text
|
||||
.msg_type_for_non_threaded_only_transcribed_messages;
|
||||
let global_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
|
||||
.global_config
|
||||
.fallback_room_settings
|
||||
.speech_to_text
|
||||
.msg_type_for_non_threaded_only_transcribed_messages;
|
||||
|
||||
let msg_type_for_non_threaded_only_transcribed_messages_set_where =
|
||||
if room_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
|
||||
strings::cfg::status_badge_set_in_room_config()
|
||||
} else if global_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
|
||||
strings::cfg::status_badge_set_in_global_config()
|
||||
} else {
|
||||
strings::cfg::status_badge_using_hardcoded_default()
|
||||
};
|
||||
|
||||
message.push_str(&strings::cfg::status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
|
||||
effective_msg_type_for_non_threaded_only_transcribed_messages,
|
||||
msg_type_for_non_threaded_only_transcribed_messages_set_where,
|
||||
));
|
||||
|
||||
// Language
|
||||
|
||||
let effective_language = room_config_context.speech_to_text_language();
|
||||
|
||||
@@ -13,7 +13,8 @@ use crate::agent::ControllerTrait;
|
||||
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
||||
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||
use crate::entity::roomconfig::{
|
||||
SpeechToTextFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
};
|
||||
use crate::entity::MessagePayload;
|
||||
use crate::strings;
|
||||
@@ -71,21 +72,35 @@ pub async fn handle(
|
||||
if let MessagePayload::Audio(audio_content) = &message_context.payload() {
|
||||
original_message_is_audio = true;
|
||||
|
||||
let response_type = match speech_to_text_flow_type {
|
||||
let (response_type, msg_type) = match speech_to_text_flow_type {
|
||||
SpeechToTextFlowType::Ignore => {
|
||||
tracing::debug!("Intentionally ignoring audio message");
|
||||
return Ok(());
|
||||
}
|
||||
SpeechToTextFlowType::TranscribeAndGenerateText => {
|
||||
tracing::debug!("Will be transcribing and possibly generating text..");
|
||||
MessageResponseType::InThread(message_context.thread_info().clone())
|
||||
(
|
||||
MessageResponseType::InThread(message_context.thread_info().clone()),
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
|
||||
)
|
||||
}
|
||||
SpeechToTextFlowType::OnlyTranscribe => {
|
||||
tracing::debug!("Will only be transcribing audio to text..");
|
||||
if message_context.thread_info().is_thread_root_only() {
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone())
|
||||
let msg_type = message_context
|
||||
.room_config_context()
|
||||
.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
|
||||
(
|
||||
MessageResponseType::Reply(
|
||||
message_context.thread_info().root_event_id.clone(),
|
||||
),
|
||||
msg_type,
|
||||
)
|
||||
} else {
|
||||
MessageResponseType::InThread(message_context.thread_info().clone())
|
||||
(
|
||||
MessageResponseType::InThread(message_context.thread_info().clone()),
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -94,8 +109,14 @@ pub async fn handle(
|
||||
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
|
||||
}
|
||||
|
||||
let Some(speech_to_text_created_event_id_result) =
|
||||
handle_stage_speech_to_text(bot, message_context, audio_content, response_type).await
|
||||
let Some(speech_to_text_created_event_id_result) = handle_stage_speech_to_text(
|
||||
bot,
|
||||
message_context,
|
||||
audio_content,
|
||||
response_type,
|
||||
msg_type,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -282,6 +303,7 @@ async fn handle_stage_speech_to_text(
|
||||
message_context: &MessageContext,
|
||||
audio_content: &AudioMessageEventContent,
|
||||
response_type: MessageResponseType,
|
||||
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
) -> Option<OwnedEventId> {
|
||||
let agent = get_effective_agent_for_purpose_or_complain(
|
||||
bot,
|
||||
@@ -312,6 +334,7 @@ async fn handle_stage_speech_to_text(
|
||||
&agent,
|
||||
audio_content,
|
||||
response_type.clone(),
|
||||
msg_type,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -572,6 +595,7 @@ async fn handle_stage_speech_to_text_actual_transcribing(
|
||||
agent: &AgentInstance,
|
||||
audio_content: &AudioMessageEventContent,
|
||||
response_type: MessageResponseType,
|
||||
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
) -> anyhow::Result<OwnedEventId> {
|
||||
let src = &audio_content.source;
|
||||
|
||||
@@ -626,9 +650,6 @@ async fn handle_stage_speech_to_text_actual_transcribing(
|
||||
//
|
||||
// When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead,
|
||||
// to make it clear to users that it's a transcription.
|
||||
//
|
||||
// Regardless of how we post this message, it will be posted as a notice,
|
||||
// which can indicate to the bot (for potential future text-generation purposes) that this message is not a bot message.
|
||||
let (transcribed_text, annotate_message_with_reaction) =
|
||||
if let MessageResponseType::InThread(_) = response_type {
|
||||
(
|
||||
@@ -639,10 +660,22 @@ async fn handle_stage_speech_to_text_actual_transcribing(
|
||||
(speech_to_text_result.text, true)
|
||||
};
|
||||
|
||||
let result = bot
|
||||
.messaging()
|
||||
.send_notice_markdown_no_fail(message_context.room(), transcribed_text, response_type)
|
||||
.await;
|
||||
let result = match msg_type {
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => {
|
||||
bot.messaging()
|
||||
.send_text_markdown_no_fail(message_context.room(), transcribed_text, response_type)
|
||||
.await
|
||||
}
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
|
||||
bot.messaging()
|
||||
.send_notice_markdown_no_fail(
|
||||
message_context.room(),
|
||||
transcribed_text,
|
||||
response_type,
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
|
||||
let event_id = result
|
||||
.map(|result| result.event_id)
|
||||
|
||||
@@ -6,11 +6,11 @@ use crate::agent::provider::ImageGenerationParams;
|
||||
use crate::agent::AgentPurpose;
|
||||
use crate::agent::ControllerTrait;
|
||||
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
||||
use crate::controller::utils::mime::get_file_extension;
|
||||
use crate::conversation::create_llm_conversation_for_matrix_thread;
|
||||
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||
use crate::strings;
|
||||
use crate::{entity::MessageContext, Bot};
|
||||
use crate::controller::utils::mime::get_file_extension;
|
||||
|
||||
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
|
||||
const STICKER_SIZE: &str = "256x256";
|
||||
@@ -82,7 +82,10 @@ pub async fn handle_image(
|
||||
.await;
|
||||
}
|
||||
|
||||
let attachment_body_text = format!("generated-image.{}", get_file_extension(&response.mime_type));
|
||||
let attachment_body_text = format!(
|
||||
"generated-image.{}",
|
||||
get_file_extension(&response.mime_type)
|
||||
);
|
||||
|
||||
let mut event_content = matrix_link
|
||||
.media()
|
||||
@@ -158,7 +161,10 @@ pub async fn handle_sticker(
|
||||
.instrument(span)
|
||||
.await?;
|
||||
|
||||
let attachment_body_text = format!("generated-sticker.{}", get_file_extension(&response.mime_type));
|
||||
let attachment_body_text = format!(
|
||||
"generated-sticker.{}",
|
||||
get_file_extension(&response.mime_type)
|
||||
);
|
||||
|
||||
let mut event_content = matrix_link
|
||||
.media()
|
||||
|
||||
@@ -12,5 +12,6 @@ pub fn get_file_extension(mime_type: &mime::Mime) -> String {
|
||||
(mime::IMAGE, mime::PNG) => "png",
|
||||
(mime::IMAGE, mime::SVG) => "svg",
|
||||
_ => "bin",
|
||||
}.to_string()
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
@@ -6,8 +6,8 @@ use crate::{
|
||||
};
|
||||
|
||||
pub mod agent;
|
||||
pub mod text_to_speech;
|
||||
pub(super) mod mime;
|
||||
pub mod text_to_speech;
|
||||
|
||||
pub async fn get_text_body_or_complain<'a>(
|
||||
bot: &Bot,
|
||||
|
||||
@@ -118,7 +118,10 @@ async fn do_generate_and_send_tts_for_message(
|
||||
}
|
||||
};
|
||||
|
||||
let attachment_body_text = format!("generated-speech.{}", get_file_extension(&text_to_speech_result.mime_type));
|
||||
let attachment_body_text = format!(
|
||||
"generated-speech.{}",
|
||||
get_file_extension(&text_to_speech_result.mime_type)
|
||||
);
|
||||
|
||||
let event_content = matrix_link
|
||||
.media()
|
||||
|
||||
@@ -45,7 +45,7 @@ fn convert_bot_notice_message(
|
||||
// Such transcriptions are prefixed with an emoji and blockquoted.
|
||||
// If we find a notice that doesn't match this pattern, we skip it.
|
||||
//
|
||||
// It should be noted that transcriptions are sometimes posted as regular notice messages which do not include
|
||||
// It should be noted that transcriptions are sometimes posted as regular notice (or even text) messages which do not include
|
||||
// the `> 🦻` formatting. This function will not handle these properly.
|
||||
|
||||
if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) {
|
||||
|
||||
@@ -4,7 +4,8 @@ use super::globalconfig::GlobalConfig;
|
||||
use super::roomconfig::RoomConfig;
|
||||
|
||||
use crate::entity::roomconfig::{
|
||||
defaults as roomconfig_defaults, SpeechToTextFlowType, TextGenerationAutoUsage,
|
||||
defaults as roomconfig_defaults, SpeechToTextFlowType,
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage,
|
||||
TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
|
||||
TextToSpeechUserMessagesFlowType,
|
||||
};
|
||||
@@ -37,6 +38,24 @@ impl RoomConfigContext {
|
||||
.unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE)
|
||||
}
|
||||
|
||||
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(
|
||||
&self,
|
||||
) -> SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||
self.room_config
|
||||
.settings
|
||||
.speech_to_text
|
||||
.msg_type_for_non_threaded_only_transcribed_messages
|
||||
.or({
|
||||
self.global_config
|
||||
.fallback_room_settings
|
||||
.speech_to_text
|
||||
.msg_type_for_non_threaded_only_transcribed_messages
|
||||
})
|
||||
.unwrap_or(
|
||||
roomconfig_defaults::SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn speech_to_text_language(&self) -> Option<String> {
|
||||
self.room_config
|
||||
.settings
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use super::SpeechToTextFlowType;
|
||||
use super::{SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages};
|
||||
use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
|
||||
use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
|
||||
|
||||
@@ -15,3 +15,9 @@ pub const TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE: TextToSpeechUserMessagesFlowTy
|
||||
|
||||
pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType =
|
||||
SpeechToTextFlowType::TranscribeAndGenerateText;
|
||||
|
||||
// While notice messages may be less desirable with other bots in the room,
|
||||
// it's probably a better default for most people who enable "transcribe-only" mode.
|
||||
pub const SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE:
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages =
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text;
|
||||
|
||||
@@ -13,7 +13,9 @@ mod text_generation;
|
||||
mod text_to_speech;
|
||||
|
||||
pub use handler::RoomSettingsHandler;
|
||||
pub use speech_to_text::SpeechToTextFlowType;
|
||||
pub use speech_to_text::{
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
};
|
||||
pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
|
||||
pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
|
||||
|
||||
|
||||
@@ -4,6 +4,20 @@ use serde::{Deserialize, Serialize};
|
||||
pub struct RoomSettingsSpeechToText {
|
||||
pub flow_type: Option<SpeechToTextFlowType>,
|
||||
|
||||
/// Controls how the transcribed message is posted when dealing with:
|
||||
/// - messages that only get transcribed (and do not trigger text-generation).
|
||||
/// See `flow_type` and `SpeechToTextFlowType::OnlyTranscribe` for more details.
|
||||
/// - incoming voice messages that are not part of a thread.
|
||||
/// For messages that are part of a thread, we need to reply within the thread in a way (with a notice message)
|
||||
/// that won't confuse the bot later, so we have no choice but to use a notice message.
|
||||
///
|
||||
/// Text-generation may happen either as a direct result of the incoming voice message or as part of a threaded conversation.
|
||||
/// Transcribed messages should not be attributed to the bot for the purposes of text-generation,
|
||||
/// so any time there's a chance of text-generation happening, we should use `SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice`
|
||||
/// and optionally prefix the message with `> 🦻`.
|
||||
pub msg_type_for_non_threaded_only_transcribed_messages:
|
||||
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
|
||||
|
||||
/// The language of the input audio.
|
||||
/// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency.
|
||||
pub language: Option<String>,
|
||||
@@ -57,3 +71,39 @@ impl std::fmt::Display for SpeechToTextFlowType {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)]
|
||||
pub enum SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||
/// Send the transcribed message text as a regular message
|
||||
#[serde(rename = "text")]
|
||||
Text,
|
||||
|
||||
/// Send the transcribed message text as a notice message
|
||||
#[serde(rename = "notice")]
|
||||
Notice,
|
||||
}
|
||||
|
||||
impl SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||
pub fn choices() -> Vec<Self> {
|
||||
vec![Self::Text, Self::Notice]
|
||||
}
|
||||
|
||||
pub fn from_str(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"text" => Some(Self::Text),
|
||||
"notice" => Some(Self::Notice),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => write!(f, "text"),
|
||||
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
|
||||
write!(f, "notice")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +70,8 @@ pub enum TextGenerationAutoUsage {
|
||||
/// Text Generation is to be performed when the original message was sent as audio (voice).
|
||||
/// The voice message would be transcribed to text (subject to other configuration)
|
||||
/// and text generation would be triggered.
|
||||
///
|
||||
/// Also see `SpeechToTextFlowType`.
|
||||
#[serde(rename = "only_for_voice")]
|
||||
OnlyForVoice,
|
||||
|
||||
|
||||
@@ -5,8 +5,9 @@ use mxlink::helpers::account_data_config::RoomConfigManager as AccountDataRoomCo
|
||||
|
||||
pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler};
|
||||
pub use entity::{
|
||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
|
||||
TextToSpeechUserMessagesFlowType,
|
||||
};
|
||||
|
||||
pub type RoomConfigurationManager =
|
||||
|
||||
@@ -4,7 +4,8 @@ use crate::{
|
||||
PublicIdentifier,
|
||||
},
|
||||
entity::roomconfig::{
|
||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
},
|
||||
utils::text::block_quote,
|
||||
@@ -269,6 +270,16 @@ pub fn status_speech_to_text_entry_flow_type(
|
||||
format!("- 🪄 Flow type: `{}` ({})\n", value, set_where)
|
||||
}
|
||||
|
||||
pub fn status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
|
||||
value: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
set_where: &str,
|
||||
) -> String {
|
||||
format!(
|
||||
"- 🪄 Message type for non-threaded only-transcribed messages: `{}` ({})\n",
|
||||
value, set_where
|
||||
)
|
||||
}
|
||||
|
||||
pub fn status_speech_to_text_entry_language(value: Option<String>, set_where: &str) -> String {
|
||||
let formatted = match value {
|
||||
Some(value) => format!("`{}` ({})", value, set_where),
|
||||
|
||||
@@ -200,6 +200,15 @@ pub fn speech_to_text_flow_type_intro() -> &'static str {
|
||||
"Controls how voice messages are handled."
|
||||
}
|
||||
|
||||
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading() -> &'static str
|
||||
{
|
||||
"🪄 Message Type for non-threaded only-transcribed messages"
|
||||
}
|
||||
|
||||
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro() -> &'static str {
|
||||
"Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`."
|
||||
}
|
||||
|
||||
pub fn speech_to_text_language_heading() -> &'static str {
|
||||
"🔤 Language"
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ pub fn create_transcribed_message_text(text: &str) -> String {
|
||||
/// If the provided text string does not match the expected format, None is returned.
|
||||
///
|
||||
/// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include
|
||||
/// the `> 🦻` prefixing. This function will not handle these properly.
|
||||
/// the `> 🦻` prefix. This function will not handle these properly.
|
||||
pub fn parse_transcribed_message_text(text: &str) -> Option<String> {
|
||||
if !text.starts_with("> ") {
|
||||
return None;
|
||||
|
||||
Reference in New Issue
Block a user