Add support for making Text-To-Speech send regular text messages instead of notices
When speech-to-text/flow-type = `only_transcribe`, the bot will now send text messages by default, not notices. While notice messages may be less desirable with other bots in the room, it's probably a better default for most people who enable "transcribe-only" mode. This is an improvement related to https://github.com/etkecc/baibot/issues/14
This commit is contained in:
@@ -23,6 +23,19 @@ The following configuration values are recognized:
|
|||||||
Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
|
Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
|
||||||
|
|
||||||
|
|
||||||
|
### 🪄 Message Type for non-threaded only-transcribed messages
|
||||||
|
|
||||||
|
Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`.
|
||||||
|
|
||||||
|
The following configuration values are recognized:
|
||||||
|
|
||||||
|
- (default) `text`: the transcribed text is sent as a regular message. This is more convenient if you'd like to forward the transcribed message to other rooms.
|
||||||
|
|
||||||
|
- `notice`: the transcribed text is sent as a notice message. This provides better compatibility with other bots in the room, as they are less likely to interact with messages of type notice.
|
||||||
|
|
||||||
|
Example: `!bai config room speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages notice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
|
||||||
|
|
||||||
|
|
||||||
### 🔤 Language
|
### 🔤 Language
|
||||||
|
|
||||||
Lets you specify the language of the input voice messages, to avoid using auto-detection.
|
Lets you specify the language of the input voice messages, to avoid using auto-detection.
|
||||||
|
|||||||
@@ -136,6 +136,8 @@ To operate in this mode, you can:
|
|||||||
|
|
||||||
- adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe`
|
- adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe`
|
||||||
|
|
||||||
|
- optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
|
||||||
|
|
||||||
|
|
||||||
### 🖌️ Image Generation
|
### 🖌️ Image Generation
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 22 KiB |
@@ -1,7 +1,8 @@
|
|||||||
use crate::{
|
use crate::{
|
||||||
agent::{AgentPurpose, PublicIdentifier},
|
agent::{AgentPurpose, PublicIdentifier},
|
||||||
entity::roomconfig::{
|
entity::roomconfig::{
|
||||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -54,6 +55,11 @@ pub enum ConfigSpeechToTextSettingRelatedControllerType {
|
|||||||
GetFlowType,
|
GetFlowType,
|
||||||
SetFlowType(Option<SpeechToTextFlowType>),
|
SetFlowType(Option<SpeechToTextFlowType>),
|
||||||
|
|
||||||
|
GetMsgTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
SetMsgTypeForNonThreadedOnlyTranscribedMessages(
|
||||||
|
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
|
||||||
|
),
|
||||||
|
|
||||||
GetLanguage,
|
GetLanguage,
|
||||||
SetLanguage(Option<String>),
|
SetLanguage(Option<String>),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
use crate::{controller::ControllerType, entity::roomconfig::SpeechToTextFlowType, strings};
|
use crate::{
|
||||||
|
controller::ControllerType,
|
||||||
|
entity::roomconfig::{
|
||||||
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
},
|
||||||
|
strings,
|
||||||
|
};
|
||||||
|
|
||||||
use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType;
|
use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType;
|
||||||
|
|
||||||
@@ -48,6 +54,53 @@ pub(super) fn determine(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// msg_type_for_non_threaded_only_transcribed_messages
|
||||||
|
|
||||||
|
if let Some(remaining_text) =
|
||||||
|
text.strip_prefix("msg-type-for-non-threaded-only-transcribed-messages")
|
||||||
|
{
|
||||||
|
let remaining_text = remaining_text.trim();
|
||||||
|
|
||||||
|
if !remaining_text.is_empty() {
|
||||||
|
return Err(ControllerType::Error(
|
||||||
|
strings::cfg::configuration_getter_used_with_extra_text(
|
||||||
|
"msg-type-for-non-threaded-only-transcribed-messages",
|
||||||
|
remaining_text,
|
||||||
|
)
|
||||||
|
.to_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(value_string) =
|
||||||
|
text.strip_prefix("set-msg-type-for-non-threaded-only-transcribed-messages")
|
||||||
|
{
|
||||||
|
let value_string = value_string.trim().to_owned();
|
||||||
|
|
||||||
|
let value_choice = if value_string.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
let value_choice =
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::from_str(
|
||||||
|
&value_string.to_lowercase(),
|
||||||
|
);
|
||||||
|
|
||||||
|
if value_choice.is_none() {
|
||||||
|
return Err(ControllerType::Error(
|
||||||
|
strings::cfg::configuration_value_unrecognized(&value_string).to_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
value_choice
|
||||||
|
};
|
||||||
|
|
||||||
|
return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(
|
||||||
|
value_choice,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
// Language
|
// Language
|
||||||
|
|
||||||
if let Some(remaining_text) = text.strip_prefix("language") {
|
if let Some(remaining_text) = text.strip_prefix("language") {
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
use crate::entity::roomconfig::{RoomSettings, SpeechToTextFlowType};
|
use crate::entity::roomconfig::{
|
||||||
|
RoomSettings, SpeechToTextFlowType,
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
};
|
||||||
use crate::{entity::MessageContext, Bot};
|
use crate::{entity::MessageContext, Bot};
|
||||||
|
|
||||||
use super::super::controller_type::{
|
use super::super::controller_type::{
|
||||||
@@ -52,6 +55,39 @@ pub(super) async fn dispatch(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages => {
|
||||||
|
let value = &room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages;
|
||||||
|
setting_get::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(bot, message_context, value).await
|
||||||
|
}
|
||||||
|
ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(value) => {
|
||||||
|
let value = value.to_owned();
|
||||||
|
|
||||||
|
let setter_callback = Box::new(move |room_settings: &mut RoomSettings| {
|
||||||
|
room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages = value;
|
||||||
|
});
|
||||||
|
|
||||||
|
match config_type {
|
||||||
|
SettingsStorageSource::Room => {
|
||||||
|
room_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
|
||||||
|
bot,
|
||||||
|
message_context,
|
||||||
|
&value,
|
||||||
|
setter_callback,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
SettingsStorageSource::Global => {
|
||||||
|
global_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
|
||||||
|
bot,
|
||||||
|
message_context,
|
||||||
|
&value,
|
||||||
|
setter_callback,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => {
|
ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => {
|
||||||
let value = &room_settings.speech_to_text.language;
|
let value = &room_settings.speech_to_text.language;
|
||||||
setting_get::<String>(bot, message_context, value).await
|
setting_get::<String>(bot, message_context, value).await
|
||||||
|
|||||||
@@ -3,7 +3,8 @@ use mxlink::MessageResponseType;
|
|||||||
use crate::{
|
use crate::{
|
||||||
entity::{
|
entity::{
|
||||||
roomconfig::{
|
roomconfig::{
|
||||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||||
},
|
},
|
||||||
MessageContext,
|
MessageContext,
|
||||||
@@ -346,6 +347,46 @@ fn build_section_speech_to_text(command_prefix: &str) -> String {
|
|||||||
));
|
));
|
||||||
message.push_str("\n\n");
|
message.push_str("\n\n");
|
||||||
|
|
||||||
|
// Msg Type For Non Threaded Only Transcribed Messages
|
||||||
|
|
||||||
|
message.push_str(&format!(
|
||||||
|
"#### {}",
|
||||||
|
strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading()
|
||||||
|
));
|
||||||
|
message.push_str("\n\n");
|
||||||
|
message.push_str(strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro());
|
||||||
|
message.push('\n');
|
||||||
|
message.push_str(
|
||||||
|
&strings::help::cfg::the_following_configuration_values_are_recognized(
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::choices(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
message.push_str("\n\n");
|
||||||
|
message.push_str(&format!(
|
||||||
|
"- {}",
|
||||||
|
&strings::help::cfg::current_setting_show(
|
||||||
|
command_prefix,
|
||||||
|
"speech-to-text msg-type-for-non-threaded-only-transcribed-messages"
|
||||||
|
)
|
||||||
|
));
|
||||||
|
message.push('\n');
|
||||||
|
message.push_str(&format!(
|
||||||
|
"- {}",
|
||||||
|
&strings::help::cfg::current_setting_set(
|
||||||
|
command_prefix,
|
||||||
|
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages VALUE"
|
||||||
|
)
|
||||||
|
));
|
||||||
|
message.push('\n');
|
||||||
|
message.push_str(&format!(
|
||||||
|
"- {}",
|
||||||
|
&strings::help::cfg::current_setting_unset(
|
||||||
|
command_prefix,
|
||||||
|
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages"
|
||||||
|
)
|
||||||
|
));
|
||||||
|
message.push_str("\n\n");
|
||||||
|
|
||||||
// Language
|
// Language
|
||||||
|
|
||||||
message.push_str(&format!(
|
message.push_str(&format!(
|
||||||
|
|||||||
@@ -507,6 +507,35 @@ async fn generate_speech_to_text_section(
|
|||||||
flow_type_set_where,
|
flow_type_set_where,
|
||||||
));
|
));
|
||||||
|
|
||||||
|
// Msg Type For Non Threaded Only Transcribed Messages
|
||||||
|
|
||||||
|
let effective_msg_type_for_non_threaded_only_transcribed_messages =
|
||||||
|
room_config_context.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
|
||||||
|
let room_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
|
||||||
|
.room_config
|
||||||
|
.settings
|
||||||
|
.speech_to_text
|
||||||
|
.msg_type_for_non_threaded_only_transcribed_messages;
|
||||||
|
let global_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
|
||||||
|
.global_config
|
||||||
|
.fallback_room_settings
|
||||||
|
.speech_to_text
|
||||||
|
.msg_type_for_non_threaded_only_transcribed_messages;
|
||||||
|
|
||||||
|
let msg_type_for_non_threaded_only_transcribed_messages_set_where =
|
||||||
|
if room_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
|
||||||
|
strings::cfg::status_badge_set_in_room_config()
|
||||||
|
} else if global_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
|
||||||
|
strings::cfg::status_badge_set_in_global_config()
|
||||||
|
} else {
|
||||||
|
strings::cfg::status_badge_using_hardcoded_default()
|
||||||
|
};
|
||||||
|
|
||||||
|
message.push_str(&strings::cfg::status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
|
||||||
|
effective_msg_type_for_non_threaded_only_transcribed_messages,
|
||||||
|
msg_type_for_non_threaded_only_transcribed_messages_set_where,
|
||||||
|
));
|
||||||
|
|
||||||
// Language
|
// Language
|
||||||
|
|
||||||
let effective_language = room_config_context.speech_to_text_language();
|
let effective_language = room_config_context.speech_to_text_language();
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ use crate::agent::ControllerTrait;
|
|||||||
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
||||||
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||||
use crate::entity::roomconfig::{
|
use crate::entity::roomconfig::{
|
||||||
SpeechToTextFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||||
};
|
};
|
||||||
use crate::entity::MessagePayload;
|
use crate::entity::MessagePayload;
|
||||||
use crate::strings;
|
use crate::strings;
|
||||||
@@ -71,21 +72,35 @@ pub async fn handle(
|
|||||||
if let MessagePayload::Audio(audio_content) = &message_context.payload() {
|
if let MessagePayload::Audio(audio_content) = &message_context.payload() {
|
||||||
original_message_is_audio = true;
|
original_message_is_audio = true;
|
||||||
|
|
||||||
let response_type = match speech_to_text_flow_type {
|
let (response_type, msg_type) = match speech_to_text_flow_type {
|
||||||
SpeechToTextFlowType::Ignore => {
|
SpeechToTextFlowType::Ignore => {
|
||||||
tracing::debug!("Intentionally ignoring audio message");
|
tracing::debug!("Intentionally ignoring audio message");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
SpeechToTextFlowType::TranscribeAndGenerateText => {
|
SpeechToTextFlowType::TranscribeAndGenerateText => {
|
||||||
tracing::debug!("Will be transcribing and possibly generating text..");
|
tracing::debug!("Will be transcribing and possibly generating text..");
|
||||||
MessageResponseType::InThread(message_context.thread_info().clone())
|
(
|
||||||
|
MessageResponseType::InThread(message_context.thread_info().clone()),
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
SpeechToTextFlowType::OnlyTranscribe => {
|
SpeechToTextFlowType::OnlyTranscribe => {
|
||||||
tracing::debug!("Will only be transcribing audio to text..");
|
tracing::debug!("Will only be transcribing audio to text..");
|
||||||
if message_context.thread_info().is_thread_root_only() {
|
if message_context.thread_info().is_thread_root_only() {
|
||||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone())
|
let msg_type = message_context
|
||||||
|
.room_config_context()
|
||||||
|
.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
|
||||||
|
(
|
||||||
|
MessageResponseType::Reply(
|
||||||
|
message_context.thread_info().root_event_id.clone(),
|
||||||
|
),
|
||||||
|
msg_type,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
MessageResponseType::InThread(message_context.thread_info().clone())
|
(
|
||||||
|
MessageResponseType::InThread(message_context.thread_info().clone()),
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -94,8 +109,14 @@ pub async fn handle(
|
|||||||
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
|
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(speech_to_text_created_event_id_result) =
|
let Some(speech_to_text_created_event_id_result) = handle_stage_speech_to_text(
|
||||||
handle_stage_speech_to_text(bot, message_context, audio_content, response_type).await
|
bot,
|
||||||
|
message_context,
|
||||||
|
audio_content,
|
||||||
|
response_type,
|
||||||
|
msg_type,
|
||||||
|
)
|
||||||
|
.await
|
||||||
else {
|
else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
@@ -282,6 +303,7 @@ async fn handle_stage_speech_to_text(
|
|||||||
message_context: &MessageContext,
|
message_context: &MessageContext,
|
||||||
audio_content: &AudioMessageEventContent,
|
audio_content: &AudioMessageEventContent,
|
||||||
response_type: MessageResponseType,
|
response_type: MessageResponseType,
|
||||||
|
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
) -> Option<OwnedEventId> {
|
) -> Option<OwnedEventId> {
|
||||||
let agent = get_effective_agent_for_purpose_or_complain(
|
let agent = get_effective_agent_for_purpose_or_complain(
|
||||||
bot,
|
bot,
|
||||||
@@ -312,6 +334,7 @@ async fn handle_stage_speech_to_text(
|
|||||||
&agent,
|
&agent,
|
||||||
audio_content,
|
audio_content,
|
||||||
response_type.clone(),
|
response_type.clone(),
|
||||||
|
msg_type,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
@@ -572,6 +595,7 @@ async fn handle_stage_speech_to_text_actual_transcribing(
|
|||||||
agent: &AgentInstance,
|
agent: &AgentInstance,
|
||||||
audio_content: &AudioMessageEventContent,
|
audio_content: &AudioMessageEventContent,
|
||||||
response_type: MessageResponseType,
|
response_type: MessageResponseType,
|
||||||
|
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
) -> anyhow::Result<OwnedEventId> {
|
) -> anyhow::Result<OwnedEventId> {
|
||||||
let src = &audio_content.source;
|
let src = &audio_content.source;
|
||||||
|
|
||||||
@@ -626,9 +650,6 @@ async fn handle_stage_speech_to_text_actual_transcribing(
|
|||||||
//
|
//
|
||||||
// When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead,
|
// When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead,
|
||||||
// to make it clear to users that it's a transcription.
|
// to make it clear to users that it's a transcription.
|
||||||
//
|
|
||||||
// Regardless of how we post this message, it will be posted as a notice,
|
|
||||||
// which can indicate to the bot (for potential future text-generation purposes) that this message is not a bot message.
|
|
||||||
let (transcribed_text, annotate_message_with_reaction) =
|
let (transcribed_text, annotate_message_with_reaction) =
|
||||||
if let MessageResponseType::InThread(_) = response_type {
|
if let MessageResponseType::InThread(_) = response_type {
|
||||||
(
|
(
|
||||||
@@ -639,10 +660,22 @@ async fn handle_stage_speech_to_text_actual_transcribing(
|
|||||||
(speech_to_text_result.text, true)
|
(speech_to_text_result.text, true)
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = bot
|
let result = match msg_type {
|
||||||
.messaging()
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => {
|
||||||
.send_notice_markdown_no_fail(message_context.room(), transcribed_text, response_type)
|
bot.messaging()
|
||||||
.await;
|
.send_text_markdown_no_fail(message_context.room(), transcribed_text, response_type)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
|
||||||
|
bot.messaging()
|
||||||
|
.send_notice_markdown_no_fail(
|
||||||
|
message_context.room(),
|
||||||
|
transcribed_text,
|
||||||
|
response_type,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let event_id = result
|
let event_id = result
|
||||||
.map(|result| result.event_id)
|
.map(|result| result.event_id)
|
||||||
|
|||||||
@@ -6,11 +6,11 @@ use crate::agent::provider::ImageGenerationParams;
|
|||||||
use crate::agent::AgentPurpose;
|
use crate::agent::AgentPurpose;
|
||||||
use crate::agent::ControllerTrait;
|
use crate::agent::ControllerTrait;
|
||||||
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
||||||
|
use crate::controller::utils::mime::get_file_extension;
|
||||||
use crate::conversation::create_llm_conversation_for_matrix_thread;
|
use crate::conversation::create_llm_conversation_for_matrix_thread;
|
||||||
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||||
use crate::strings;
|
use crate::strings;
|
||||||
use crate::{entity::MessageContext, Bot};
|
use crate::{entity::MessageContext, Bot};
|
||||||
use crate::controller::utils::mime::get_file_extension;
|
|
||||||
|
|
||||||
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
|
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
|
||||||
const STICKER_SIZE: &str = "256x256";
|
const STICKER_SIZE: &str = "256x256";
|
||||||
@@ -82,7 +82,10 @@ pub async fn handle_image(
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let attachment_body_text = format!("generated-image.{}", get_file_extension(&response.mime_type));
|
let attachment_body_text = format!(
|
||||||
|
"generated-image.{}",
|
||||||
|
get_file_extension(&response.mime_type)
|
||||||
|
);
|
||||||
|
|
||||||
let mut event_content = matrix_link
|
let mut event_content = matrix_link
|
||||||
.media()
|
.media()
|
||||||
@@ -158,7 +161,10 @@ pub async fn handle_sticker(
|
|||||||
.instrument(span)
|
.instrument(span)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let attachment_body_text = format!("generated-sticker.{}", get_file_extension(&response.mime_type));
|
let attachment_body_text = format!(
|
||||||
|
"generated-sticker.{}",
|
||||||
|
get_file_extension(&response.mime_type)
|
||||||
|
);
|
||||||
|
|
||||||
let mut event_content = matrix_link
|
let mut event_content = matrix_link
|
||||||
.media()
|
.media()
|
||||||
|
|||||||
@@ -12,5 +12,6 @@ pub fn get_file_extension(mime_type: &mime::Mime) -> String {
|
|||||||
(mime::IMAGE, mime::PNG) => "png",
|
(mime::IMAGE, mime::PNG) => "png",
|
||||||
(mime::IMAGE, mime::SVG) => "svg",
|
(mime::IMAGE, mime::SVG) => "svg",
|
||||||
_ => "bin",
|
_ => "bin",
|
||||||
}.to_string()
|
}
|
||||||
|
.to_string()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ use crate::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
pub mod agent;
|
pub mod agent;
|
||||||
pub mod text_to_speech;
|
|
||||||
pub(super) mod mime;
|
pub(super) mod mime;
|
||||||
|
pub mod text_to_speech;
|
||||||
|
|
||||||
pub async fn get_text_body_or_complain<'a>(
|
pub async fn get_text_body_or_complain<'a>(
|
||||||
bot: &Bot,
|
bot: &Bot,
|
||||||
|
|||||||
@@ -118,7 +118,10 @@ async fn do_generate_and_send_tts_for_message(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let attachment_body_text = format!("generated-speech.{}", get_file_extension(&text_to_speech_result.mime_type));
|
let attachment_body_text = format!(
|
||||||
|
"generated-speech.{}",
|
||||||
|
get_file_extension(&text_to_speech_result.mime_type)
|
||||||
|
);
|
||||||
|
|
||||||
let event_content = matrix_link
|
let event_content = matrix_link
|
||||||
.media()
|
.media()
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ fn convert_bot_notice_message(
|
|||||||
// Such transcriptions are prefixed with an emoji and blockquoted.
|
// Such transcriptions are prefixed with an emoji and blockquoted.
|
||||||
// If we find a notice that doesn't match this pattern, we skip it.
|
// If we find a notice that doesn't match this pattern, we skip it.
|
||||||
//
|
//
|
||||||
// It should be noted that transcriptions are sometimes posted as regular notice messages which do not include
|
// It should be noted that transcriptions are sometimes posted as regular notice (or even text) messages which do not include
|
||||||
// the `> 🦻` formatting. This function will not handle these properly.
|
// the `> 🦻` formatting. This function will not handle these properly.
|
||||||
|
|
||||||
if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) {
|
if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) {
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ use super::globalconfig::GlobalConfig;
|
|||||||
use super::roomconfig::RoomConfig;
|
use super::roomconfig::RoomConfig;
|
||||||
|
|
||||||
use crate::entity::roomconfig::{
|
use crate::entity::roomconfig::{
|
||||||
defaults as roomconfig_defaults, SpeechToTextFlowType, TextGenerationAutoUsage,
|
defaults as roomconfig_defaults, SpeechToTextFlowType,
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage,
|
||||||
TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
|
TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
|
||||||
TextToSpeechUserMessagesFlowType,
|
TextToSpeechUserMessagesFlowType,
|
||||||
};
|
};
|
||||||
@@ -37,6 +38,24 @@ impl RoomConfigContext {
|
|||||||
.unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE)
|
.unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(
|
||||||
|
&self,
|
||||||
|
) -> SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||||
|
self.room_config
|
||||||
|
.settings
|
||||||
|
.speech_to_text
|
||||||
|
.msg_type_for_non_threaded_only_transcribed_messages
|
||||||
|
.or({
|
||||||
|
self.global_config
|
||||||
|
.fallback_room_settings
|
||||||
|
.speech_to_text
|
||||||
|
.msg_type_for_non_threaded_only_transcribed_messages
|
||||||
|
})
|
||||||
|
.unwrap_or(
|
||||||
|
roomconfig_defaults::SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn speech_to_text_language(&self) -> Option<String> {
|
pub fn speech_to_text_language(&self) -> Option<String> {
|
||||||
self.room_config
|
self.room_config
|
||||||
.settings
|
.settings
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use super::SpeechToTextFlowType;
|
use super::{SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages};
|
||||||
use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
|
use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
|
||||||
use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
|
use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
|
||||||
|
|
||||||
@@ -15,3 +15,9 @@ pub const TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE: TextToSpeechUserMessagesFlowTy
|
|||||||
|
|
||||||
pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType =
|
pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType =
|
||||||
SpeechToTextFlowType::TranscribeAndGenerateText;
|
SpeechToTextFlowType::TranscribeAndGenerateText;
|
||||||
|
|
||||||
|
// While notice messages may be less desirable with other bots in the room,
|
||||||
|
// it's probably a better default for most people who enable "transcribe-only" mode.
|
||||||
|
pub const SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE:
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages =
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text;
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ mod text_generation;
|
|||||||
mod text_to_speech;
|
mod text_to_speech;
|
||||||
|
|
||||||
pub use handler::RoomSettingsHandler;
|
pub use handler::RoomSettingsHandler;
|
||||||
pub use speech_to_text::SpeechToTextFlowType;
|
pub use speech_to_text::{
|
||||||
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
};
|
||||||
pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
|
pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
|
||||||
pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
|
pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,20 @@ use serde::{Deserialize, Serialize};
|
|||||||
pub struct RoomSettingsSpeechToText {
|
pub struct RoomSettingsSpeechToText {
|
||||||
pub flow_type: Option<SpeechToTextFlowType>,
|
pub flow_type: Option<SpeechToTextFlowType>,
|
||||||
|
|
||||||
|
/// Controls how the transcribed message is posted when dealing with:
|
||||||
|
/// - messages that only get transcribed (and do not trigger text-generation).
|
||||||
|
/// See `flow_type` and `SpeechToTextFlowType::OnlyTranscribe` for more details.
|
||||||
|
/// - incoming voice messages that are not part of a thread.
|
||||||
|
/// For messages that are part of a thread, we need to reply within the thread in a way (with a notice message)
|
||||||
|
/// that won't confuse the bot later, so we have no choice but to use a notice message.
|
||||||
|
///
|
||||||
|
/// Text-generation may happen either as a direct result of the incoming voice message or as part of a threaded conversation.
|
||||||
|
/// Transcribed messages should not be attributed to the bot for the purposes of text-generation,
|
||||||
|
/// so any time there's a chance of text-generation happening, we should use `SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice`
|
||||||
|
/// and optionally prefix the message with `> 🦻`.
|
||||||
|
pub msg_type_for_non_threaded_only_transcribed_messages:
|
||||||
|
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
|
||||||
|
|
||||||
/// The language of the input audio.
|
/// The language of the input audio.
|
||||||
/// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency.
|
/// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency.
|
||||||
pub language: Option<String>,
|
pub language: Option<String>,
|
||||||
@@ -57,3 +71,39 @@ impl std::fmt::Display for SpeechToTextFlowType {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)]
|
||||||
|
pub enum SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||||
|
/// Send the transcribed message text as a regular message
|
||||||
|
#[serde(rename = "text")]
|
||||||
|
Text,
|
||||||
|
|
||||||
|
/// Send the transcribed message text as a notice message
|
||||||
|
#[serde(rename = "notice")]
|
||||||
|
Notice,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||||
|
pub fn choices() -> Vec<Self> {
|
||||||
|
vec![Self::Text, Self::Notice]
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_str(s: &str) -> Option<Self> {
|
||||||
|
match s {
|
||||||
|
"text" => Some(Self::Text),
|
||||||
|
"notice" => Some(Self::Notice),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => write!(f, "text"),
|
||||||
|
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
|
||||||
|
write!(f, "notice")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ pub enum TextGenerationAutoUsage {
|
|||||||
/// Text Generation is to be performed when the original message was sent as audio (voice).
|
/// Text Generation is to be performed when the original message was sent as audio (voice).
|
||||||
/// The voice message would be transcribed to text (subject to other configuration)
|
/// The voice message would be transcribed to text (subject to other configuration)
|
||||||
/// and text generation would be triggered.
|
/// and text generation would be triggered.
|
||||||
|
///
|
||||||
|
/// Also see `SpeechToTextFlowType`.
|
||||||
#[serde(rename = "only_for_voice")]
|
#[serde(rename = "only_for_voice")]
|
||||||
OnlyForVoice,
|
OnlyForVoice,
|
||||||
|
|
||||||
|
|||||||
@@ -5,8 +5,9 @@ use mxlink::helpers::account_data_config::RoomConfigManager as AccountDataRoomCo
|
|||||||
|
|
||||||
pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler};
|
pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler};
|
||||||
pub use entity::{
|
pub use entity::{
|
||||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
|
||||||
|
TextToSpeechUserMessagesFlowType,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub type RoomConfigurationManager =
|
pub type RoomConfigurationManager =
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ use crate::{
|
|||||||
PublicIdentifier,
|
PublicIdentifier,
|
||||||
},
|
},
|
||||||
entity::roomconfig::{
|
entity::roomconfig::{
|
||||||
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||||
},
|
},
|
||||||
utils::text::block_quote,
|
utils::text::block_quote,
|
||||||
@@ -269,6 +270,16 @@ pub fn status_speech_to_text_entry_flow_type(
|
|||||||
format!("- 🪄 Flow type: `{}` ({})\n", value, set_where)
|
format!("- 🪄 Flow type: `{}` ({})\n", value, set_where)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
|
||||||
|
value: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||||
|
set_where: &str,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"- 🪄 Message type for non-threaded only-transcribed messages: `{}` ({})\n",
|
||||||
|
value, set_where
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn status_speech_to_text_entry_language(value: Option<String>, set_where: &str) -> String {
|
pub fn status_speech_to_text_entry_language(value: Option<String>, set_where: &str) -> String {
|
||||||
let formatted = match value {
|
let formatted = match value {
|
||||||
Some(value) => format!("`{}` ({})", value, set_where),
|
Some(value) => format!("`{}` ({})", value, set_where),
|
||||||
|
|||||||
@@ -200,6 +200,15 @@ pub fn speech_to_text_flow_type_intro() -> &'static str {
|
|||||||
"Controls how voice messages are handled."
|
"Controls how voice messages are handled."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading() -> &'static str
|
||||||
|
{
|
||||||
|
"🪄 Message Type for non-threaded only-transcribed messages"
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro() -> &'static str {
|
||||||
|
"Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`."
|
||||||
|
}
|
||||||
|
|
||||||
pub fn speech_to_text_language_heading() -> &'static str {
|
pub fn speech_to_text_language_heading() -> &'static str {
|
||||||
"🔤 Language"
|
"🔤 Language"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ pub fn create_transcribed_message_text(text: &str) -> String {
|
|||||||
/// If the provided text string does not match the expected format, None is returned.
|
/// If the provided text string does not match the expected format, None is returned.
|
||||||
///
|
///
|
||||||
/// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include
|
/// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include
|
||||||
/// the `> 🦻` prefixing. This function will not handle these properly.
|
/// the `> 🦻` prefix. This function will not handle these properly.
|
||||||
pub fn parse_transcribed_message_text(text: &str) -> Option<String> {
|
pub fn parse_transcribed_message_text(text: &str) -> Option<String> {
|
||||||
if !text.starts_with("> ") {
|
if !text.starts_with("> ") {
|
||||||
return None;
|
return None;
|
||||||
|
|||||||
Reference in New Issue
Block a user