Compare commits

..

28 Commits

Author SHA1 Message Date
Slavi Pantaleev
8f956d2329 Release 1.5.1 2025-03-31 14:37:23 +03:00
Slavi Pantaleev
ba4aa35987 Upgrade Rust compiler in container image (1.85.0 -> 1.85.1) 2025-03-31 14:33:49 +03:00
Slavi Pantaleev
06d699a17d Update dependencies 2025-03-31 14:33:49 +03:00
Slavi Pantaleev
77d41fb7eb Update services 2025-03-31 14:33:49 +03:00
Slavi Pantaleev
aaf283dde3 Upgrade Element Web (v1.11.93 -> v1.11.96) and adapt to its own way of running as a non-privileged user 2025-03-31 14:33:49 +03:00
Slavi Pantaleev
17eafa86af Release 1.5.0 2025-02-27 11:29:26 +02:00
Slavi Pantaleev
4704934b06 Run amd64 builds on a self-hosted runner 2025-02-27 11:06:18 +02:00
Slavi Pantaleev
6719538530 Switch to using native ARM64 builders
The workflow code is inspired by https://github.com/matrix-org/matrix-hookshot/pull/1007

Ref: https://github.blog/news-insights/product-news/arm64-on-github-actions-powering-faster-more-efficient-build-systems/

This decreases build times from ~40 minutes (on powerful self-hosted runners) to ~8 minutes (on public runners).
2025-02-27 10:31:23 +02:00
Slavi Pantaleev
59e2746578 Remove explicit lifetime to fix clippy-reported warning 2025-02-27 09:58:58 +02:00
Slavi Pantaleev
47d8edea70 Add support for configuring max_completion_tokens for OpenAI
Related to db9422740c
2025-02-27 09:58:58 +02:00
Slavi Pantaleev
692d61b239 Replace Anthropic library (anthropic-rs -> anthropic) and switch default recommended model (claude-3-5-sonnet-20240620 -> claude-3-7-sonnet-20250219)
Fixes https://github.com/etkecc/baibot/issues/22

Ultimate related to `anthropic-rs` hardcoding models as enumeration variants in the code
and not updating them. See:
- https://github.com/roushou/mesh/issues/1
- https://github.com/roushou/mesh/pull/2

https://github.com/cortesi/misanthropy was also considered as an
alternative, but it did not allow configuring the base API URL like our
old Anthropic library (`anthropic-rs`) and like our new choice (`anthropic`).
We'd rather not lose support for this, so we're going with the `anthropic` library.
2025-02-27 09:44:49 +02:00
Slavi Pantaleev
05902f4c17 Add fmt justfile recipe 2025-02-27 09:31:26 +02:00
Slavi Pantaleev
7e66068b16 Update dependencies 2025-02-27 07:48:10 +02:00
Slavi Pantaleev
406141cd7d fmt 2025-02-27 07:46:16 +02:00
Slavi Pantaleev
c051da2f4a Add config setting controlling if a self-introduction message is posted after joining a room
Fixes https://github.com/etkecc/baibot/issues/32
2025-02-26 20:51:24 +02:00
Slavi Pantaleev
1ff7e8cf79 Switch Rust edition (2021 -> 2024) 2025-02-26 20:51:24 +02:00
Slavi Pantaleev
b3bca98e84 Upgrade Rust compiler in container image (1.82.0 -> 1.85.0) 2025-02-26 20:51:24 +02:00
Slavi Pantaleev
c07b712318 Upgrade mxlink (1.5.0 -> 1.6.0) and matrix-sdk (0.9.0 -> 0.10.0) 2025-02-26 20:51:24 +02:00
Slavi Pantaleev
6741483056 Update dependencies 2025-02-26 20:51:24 +02:00
Slavi Pantaleev
06b2b6d776 Use progress indicator emoji (⏳), not 🦻 to indicate that speech-to-text is happening
🦻 is used for another purpose - to denote that a message is one coming
from speech-to-text, by:

- having the bot react to its own speech-to-text transcription message
  with the 🦻 emoji when it's posted in a non-thread

- having the bot prefix its speech-to-text transcription messages with
  `> 🦻` when it's posted in a thread

⏳ is already used as a progress indicator for other features, so it
makes sense to use it for indicating that speech-to-text is happening for a given audio message as well.
While 🦻 was an even more descriptive illustration of what's actually happening to the audio message
("it's being heard by the bot"), us using the 🦻 emoji for different things didn't seem good.
2025-02-26 20:51:24 +02:00
Slavi Pantaleev
a1bd292752 Add support for making Text-To-Speech send regular text messages instead of notices
When speech-to-text/flow-type = `only_transcribe`, the bot will now send
text messages by default, not notices.

While notice messages may be less desirable with other bots in the room,
it's probably a better default for most people who enable "transcribe-only" mode.

This is an improvement related to https://github.com/etkecc/baibot/issues/14
2025-02-26 20:51:08 +02:00
Slavi Pantaleev
e4e1fe0e7b Upgrade some components 2025-02-26 09:39:48 +02:00
Slavi Pantaleev
45a2d96029 Enable necessary feature for matrix-sdk which affects us if used as a library
Fixes https://github.com/etkecc/rust-mxlink/issues/1
2025-02-02 07:51:25 +02:00
Slavi Pantaleev
ec1879d212 Populate image/audio attachment body with a filename, not with text
Various clients (including newer versions of Element Web), do not like
it when the `body` field of the attachment is not a file name.

For images, a preview may not be shown and downloading the attachment
may suggest that the whole long text is used as a filename (which is odd).

There is value (improved accessibility, etc.)
in adding better descriptions (especially to generated images),
but given that it's currently problematic, I'm getting rid of it.
It's better and safer if we stick to using filenames.
2025-01-24 11:35:08 +02:00
Slavi Pantaleev
5e6a600895 Update dependencies 2025-01-24 10:55:12 +02:00
Slavi Pantaleev
3db924b124 Upgrade services 2025-01-24 10:01:50 +02:00
Slavi Pantaleev
ff7a5ef7af Install SQLite 3 for test-and-clippy CI job 2024-12-12 12:25:56 +02:00
Slavi Pantaleev
cd7d9137e8 Fix broken commit link in changelog entry 2024-12-12 12:03:25 +02:00
105 changed files with 1834 additions and 921 deletions

View File

@@ -3,13 +3,14 @@ on:
push:
branches: [ "main" ]
tags: [ "v*" ]
schedule:
- cron: '0 0 * * 1'
permissions:
checks: write
contents: write
packages: write
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
test-and-clippy:
name: Unit testing and linting
@@ -17,20 +18,46 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- name: Install SQLite3
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
- run: cargo test --all-features
- run: cargo clippy
build-publish:
name: Build and Publish
runs-on: self-hosted
docker-clean-metadata:
runs-on: ubuntu-latest
outputs:
json: ${{ steps.meta.outputs.json }}
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
with:
platforms: arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to ghcr.io
images: |
ghcr.io/${{ github.repository }}
tags: |
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
type=semver,pattern={{raw}}
docker-build:
permissions:
contents: read
packages: write
attestations: write
id-token: write
strategy:
matrix:
include:
- os: self-hosted
arch: amd64
- os: ubuntu-24.04-arm
arch: arm64
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Log in to the GitHub Container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
@@ -40,17 +67,41 @@ jobs:
id: meta
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/${{ github.repository }}
registry.etke.cc/${{ github.repository }}
tags: |
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
type=semver,pattern={{raw}}
- name: Build and push
flavor: |
latest=auto
suffix=-${{ matrix.arch }},onlatest=true
images: |
ghcr.io/${{ github.repository }}
- name: Build and push Docker images
uses: docker/build-push-action@v6
with:
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
file: Dockerfile.ci
docker-manifest:
needs:
- docker-build
- docker-clean-metadata
runs-on: ubuntu-latest
strategy:
matrix:
image: ${{ fromJson(needs.docker-clean-metadata.outputs.json).tags }}
steps:
- name: Log in to the GitHub Container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create and push manifest
run: |
docker manifest create ${{ matrix.image }} ${{ matrix.image }}-amd64 ${{ matrix.image }}-arm64
docker manifest push ${{ matrix.image }}

View File

@@ -1,6 +1,29 @@
# (2025-03-31) Version 1.5.1
- (**Internal Improvement**) Dependency updates
# (2025-02-27) Version 1.5.0
- (**Feature**) Add support for sending Speech-to-Text replies for [Transcribe-only mode](./docs/features.md#transcribe-only-mode) as regular text messages instead of notices and doing it so by default ([a1bd292752](https://github.com/etkecc/baibot/commit/a1bd292752bdd37a196788c73d00b5619e843a78)) - improvement for [issue #14](https://github.com/etkecc/baibot/issues/14). See [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./docs/configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages) for details.
- (**Feature**) Add config setting controlling if a self-introduction message is posted after joining a room ([c051da2f4a](https://github.com/etkecc/baibot/commit/c051da2f4a161de0974ebb917f7a52d01f5a001f)) - fixes [issue #32](https://github.com/etkecc/baibot/issues/32). You may wish to add a `room.post_join_self_introduction_enabled` property to your configuration. See the [sample config](./etc/app/config.yml.dist) for details. If unspecified, it defaults to `true` anyway which preserves the old behavior.
- (**Feature**) Add support for configuring `max_completion_tokens` for OpenAI ([47d8edea70](https://github.com/etkecc/baibot/commit/47d8edea705a44aa25a9bfaec4888c0f9ea8700e))
- (**Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.6.1 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.10.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.10.0))
- (**Improvement**) Populate image/audio attachment `body` with a filename, not with text to avoid incorrect rendering in Element Web, etc. ([ec1879d212](https://github.com/etkecc/baibot/commit/ec1879d212fa8d6e5f8590486e94c72abfcb75a5))
- (**Improvement**) Replace Anthropic library ([anthropic-rs](https://crates.io/crates/anthropic-rs) -> [anthropic](https://crates.io/crates/anthropic)) and switch default recommended model (`claude-3-5-sonnet-20240620` -> `claude-3-7-sonnet-20250219`) ([692d61b239](https://github.com/etkecc/baibot/commit/692d61b2398f073b81d32d4cbe8145ab3929e48c)) - fixes [issue #22](https://github.com/etkecc/baibot/issues/22)
- (**Internal Improvement**) Switch to native building of `arm64` container images to decrease total build times from ~40 minutes to ~8 minutes ([6719538530b](https://github.com/etkecc/baibot/commit/6719538530bf76b3ff2d24077b2a7fa868276b79))
- (**Internal Improvement**) Various other internal changes, including upgrading [Rust from 1.82 to 1.85 and switching to Rust edition 2024](https://blog.rust-lang.org/2025/02/20/Rust-1.85.0.html)
# (2024-12-12) Version 1.4.1
- (**Bugfix**) Fix detection for whether the bot is the last member in a room, to avoid incorrectly leaving multi-user rooms that have had at least one person `leave` ([d2d846816e](https://github.com/etkecc/baibot/commit/d2d846816e58c132e30cad28440552453afe1af4))
- (**Bugfix**) Fix detection for whether the bot is the last member in a room, to avoid incorrectly leaving multi-user rooms that have had at least one person `leave` ([3c47d40781](https://github.com/etkecc/baibot/commit/3c47d407819aa9c0121117a411858238724f06da))
# (2024-11-19) Version 1.4.0

1506
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,32 +7,33 @@ license = "AGPL-3.0-or-later"
readme = "README.md"
keywords = ["matrix", "chat", "bot", "AI", "LLM"]
include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
version = "1.4.1"
edition = "2021"
version = "1.5.1"
edition = "2024"
[lib]
name = "baibot"
path = "src/lib.rs"
[dependencies]
anthropic-rs = "0.1.*"
anthropic = "=0.0.8"
anyhow = "1.0.*"
async-openai = "0.26.*"
async-openai = "0.28.*"
base64 = "0.22.*"
chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
# We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
matrix-sdk = { version = "0.8.0", default-features = false }
# We add the `native-tls` feature, because of https://github.com/etkecc/rust-mxlink/issues/1
matrix-sdk = { version = "0.10.0", default-features = false, features = ["native-tls"] }
mxidwc = "1.0.*"
mxlink = ">=1.4.1"
mxlink = ">=1.6.0"
etke_openai_api_rust = "0.1.*"
quick_cache = "0.6.*"
regex = "1.11.*"
serde = { version = "1.0.*", features = ["derive"], default-features = false }
serde_json = "1.0.*"
serde_yaml = "0.9.*"
tempfile = "3.14.*"
tempfile = "3.19.*"
tiktoken-rs = { version = "0.6.*", features = ["async-openai"] }
tokio = { version = "1.41.*", features = ["rt", "rt-multi-thread", "macros"] }
tokio = { version = "1.44.*", features = ["rt", "rt-multi-thread", "macros"] }
tracing = "0.1.*"
tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }
url = "2.5.*"

View File

@@ -4,7 +4,7 @@
# #
#######################################
FROM docker.io/rust:1.82.0-slim-bookworm AS build
FROM docker.io/rust:1.85.1-slim-bookworm AS build
RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev

View File

@@ -4,7 +4,7 @@
# #
#######################################
FROM docker.io/rust:1.82.0-slim-bookworm AS build
FROM docker.io/rust:1.85.1-slim-bookworm AS build
RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev

View File

@@ -23,6 +23,19 @@ The following configuration values are recognized:
Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
### 🪄 Message Type for non-threaded only-transcribed messages
Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`.
The following configuration values are recognized:
- (default) `text`: the transcribed text is sent as a regular message. This is more convenient if you'd like to forward the transcribed message to other rooms.
- `notice`: the transcribed text is sent as a notice message. This provides better compatibility with other bots in the room, as they are less likely to interact with messages of type notice.
Example: `!bai config room speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages notice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
### 🔤 Language
Lets you specify the language of the input voice messages, to avoid using auto-detection.

View File

@@ -136,6 +136,8 @@ To operate in this mode, you can:
- adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe`
- optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
### 🖌️ Image Generation

View File

@@ -1,7 +1,7 @@
base_url: https://api.anthropic.com/v1
api_key: YOUR_API_KEY_HERE
text_generation:
model_id: claude-3-5-sonnet-20240620
model_id: claude-3-7-sonnet-20250219
prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
temperature: 1.0
max_response_tokens: 8192

View File

@@ -6,9 +6,9 @@ text_generation:
prompt: null
temperature: 1.0
# o1 models do not support max_response_tokens.
# They use `max_completion_tokens` as an alternative,
# but we don't support it yet (see https://github.com/64bit/async-openai/issues/272).
# They use `max_completion_tokens` as an alternative
max_response_tokens: null
max_completion_tokens: 16384
max_context_tokens: 128000
speech_to_text:
model_id: whisper-1

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 22 KiB

View File

@@ -32,6 +32,10 @@ user:
# Command prefix. Leave empty to use the default (!bai).
command_prefix: "!bai"
room:
# Whether the bot should send an introduction message after joining a room.
post_join_self_introduction_enabled: true
access:
# Space-separated list of MXID patterns which specify who is an admin.
admin_patterns:
@@ -76,6 +80,8 @@ agents:
# prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
# temperature: 1.0
# max_response_tokens: 16384
# # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
# max_completion_tokens: ~
# max_context_tokens: 128000
# speech_to_text:
# model_id: whisper-1

View File

@@ -1,6 +1,6 @@
services:
postgres:
image: docker.io/postgres:16.4-alpine
image: docker.io/postgres:16.8-alpine
user: ${UID}:${GID}
restart: unless-stopped
environment:
@@ -13,7 +13,7 @@ services:
- /etc/passwd:/etc/passwd:ro
synapse:
image: ghcr.io/element-hq/synapse:v1.118.0
image: ghcr.io/element-hq/synapse:v1.127.1
user: "${UID}:${GID}"
restart: unless-stopped
entrypoint: python
@@ -26,14 +26,20 @@ services:
- ./synapse/media-store:/media-store
element-web:
image: docker.io/vectorim/element-web:v1.11.84
image: ghcr.io/element-hq/element-web:v1.11.96
user: "${UID}:${GID}"
restart: unless-stopped
environment:
ELEMENT_WEB_PORT: 8080
ports:
- "${SERVICE_ELEMENT_WEB_BIND_PORT_HTTP}:8080"
volumes:
- ../../etc/services/core/element-web/nginx.conf:/etc/nginx/nginx.conf:ro
- ../../etc/services/core/element-web/config.json:/app/config.json:ro
tmpfs:
- /var/cache/nginx:rw,mode=777
- /var/run:rw,mode=777
- /tmp/element-web-config:rw,mode=777
- /etc/nginx/conf.d:rw,mode=777
networks:
default:

View File

@@ -3,7 +3,7 @@
"default_is_url": "https://vector.im",
"integrations_ui_url": "https://scalar.vector.im/",
"integrations_rest_url": "https://scalar.vector.im/api",
"bug_report_endpoint_url": "https://riot.im/bugreports/submit",
"bug_report_endpoint_url": "https://element.io/bugreports/submit",
"enableLabs": true,
"roomDirectory": {
"servers": [

View File

@@ -1,60 +0,0 @@
# This is a custom nginx configuration file that we use in the container (instead of the default one),
# because it allows us to run nginx with a non-root user.
#
# For this to work, the default vhost file (`/etc/nginx/conf.d/default.conf`) also needs to be removed.
# (mounting `/dev/null` over `/etc/nginx/conf.d/default.conf` works well)
#
# The following changes have been done compared to a default nginx configuration file:
# - default server port is changed (80 -> 8080), so that a non-root user can bind it
# - various temp paths are changed to `/tmp`, so that a non-root user can write to them
# - the `user` directive was removed, as we don't want nginx to switch users
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
client_body_temp_path /tmp/client_body_temp;
proxy_temp_path /tmp/proxy_temp;
fastcgi_temp_path /tmp/fastcgi_temp;
uwsgi_temp_path /tmp/uwsgi_temp;
scgi_temp_path /tmp/scgi_temp;
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
server {
listen 8080;
server_name localhost;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
}

View File

@@ -1,6 +1,6 @@
services:
ollama:
image: docker.io/ollama/ollama:0.4.1
image: docker.io/ollama/ollama:0.6.3
restart: unless-stopped
ports:
- "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"

View File

@@ -32,6 +32,10 @@ run-in-container *extra_args: app-container-prepare build-container-image-debug
test *extra_args:
RUST_BACKTRACE=1 cargo test {{ extra_args }}
# Formats the code
fmt:
RUST_BACKTRACE=1 cargo fmt --all
# Builds a debug binary (target/debug/*)
build-debug *extra_args:
RUST_BACKTRACE=1 cargo build {{ extra_args }}

View File

@@ -1,6 +1,6 @@
use super::{
provider::{self, ControllerType},
AgentDefinition, AgentProvider, PublicIdentifier,
provider::{self, ControllerType},
};
// Dead-code is allowed. We do not use these enum struct payloads directly,

View File

@@ -1,7 +1,7 @@
use super::instantiation;
use super::instantiation::AgentInstance;
use super::AgentDefinition;
use super::PublicIdentifier;
use super::instantiation;
use super::instantiation::AgentInstance;
use crate::entity::RoomConfigContext;
#[derive(Debug)]

View File

@@ -11,11 +11,11 @@ pub use manager::Manager;
pub use definition::AgentDefinition;
pub use instantiation::create_from_provider_and_yaml_value_config;
pub use instantiation::default_config_for_provider;
pub use instantiation::AgentInstance;
pub use instantiation::Error as AgentInstantiationError;
pub use instantiation::Result as AgentInstantiationResult;
pub use instantiation::create_from_provider_and_yaml_value_config;
pub use instantiation::default_config_for_provider;
pub use provider::{AgentProvider, AgentProviderInfo, ControllerTrait};
pub use purpose::AgentPurpose;

View File

@@ -1,7 +1,5 @@
use serde::{Deserialize, Serialize};
use anthropic_rs::models::claude::ClaudeModel;
use crate::agent::{default_prompt, provider::ConfigTrait};
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -28,6 +26,9 @@ impl ConfigTrait for Config {
if self.base_url.is_empty() {
return Err("The base URL must not be empty.".to_owned());
}
if !self.base_url.ends_with("/v1") {
return Err("The base URL must end with '/v1'.".to_owned());
}
if self.api_key.is_empty() {
return Err("The API key must not be empty.".to_owned());
}
@@ -67,5 +68,5 @@ impl Default for TextGenerationConfig {
}
fn default_text_model_id() -> String {
ClaudeModel::Claude35Sonnet.as_str().to_owned()
"claude-3-7-sonnet-20250219".to_owned()
}

View File

@@ -1,30 +1,26 @@
use std::fmt::Debug;
use std::str::FromStr;
use std::sync::Arc;
use anthropic_rs::completion::message::{ContentType, System};
use anthropic_rs::{
client::Client as AnthropicClient, config::Config as AnthropicConfig,
models::claude::ClaudeModel,
};
use anthropic::client::{Client, ClientBuilder};
use anthropic::types::ContentBlock;
use super::super::ControllerTrait;
use crate::agent::AgentPurpose;
use crate::agent::provider::entity::{
ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
TextToSpeechParams, TextToSpeechResult,
};
use crate::agent::provider::{ImageGenerationParams, SpeechToTextParams, SpeechToTextResult};
use crate::agent::AgentPurpose;
use crate::conversation::llm::{
shorten_messages_list_to_context_size, Author as LLMAuthor, Conversation as LLMConversation,
Message as LLMMessage,
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
shorten_messages_list_to_context_size,
};
use crate::strings;
use super::config::Config;
struct ControllerInner {
client: AnthropicClient,
client: Client,
}
#[derive(Clone)]
@@ -43,18 +39,20 @@ impl Debug for Controller {
impl Controller {
pub fn new(config: Config) -> anyhow::Result<Self> {
let anthropic_config =
AnthropicConfig::new(config.api_key.clone()).with_base_url(config.base_url.clone());
// The previous library that we used expected a base URL that ends with "/v1"
// (e.g. "https://api.anthropic.com/v1"), while the new one doesn't.
//
// To keep backward compatibility, we don't ask people to change their configuration
// and rather adapt by removing the "/v1" from the base URL.
if !config.base_url.ends_with("/v1") {
return Err(anyhow::anyhow!("base_url must end with '/v1'"));
}
let client = match AnthropicClient::new(anthropic_config) {
Ok(client) => client,
Err(err) => {
return Err(anyhow::anyhow!(
"Failed to create Anthropic client: {}",
err.to_string()
));
}
};
let base_url = &config.base_url[..config.base_url.len() - 3];
let client = ClientBuilder::default()
.api_base(base_url.to_string())
.api_key(config.api_key.clone())
.build()?;
Ok(Self {
config,
@@ -142,29 +140,17 @@ impl ControllerTrait for Controller {
let mut request = super::utils::create_anthropic_message_request(conversation_messages);
let model = match ClaudeModel::from_str(&text_generation_config.model_id) {
Ok(model) => model,
Err(err) => {
tracing::debug!(?err, "Failed to parse model ID");
return Err(anyhow::anyhow!(
"Failed to parse model ID: {}",
&text_generation_config.model_id
));
}
};
let temperature = params
.temperature_override
.unwrap_or(text_generation_config.temperature);
if let Some(prompt_message) = prompt_message {
request.system = Some(System::Text(prompt_message.message_text));
request.system = prompt_message.message_text;
}
request.model = model;
request.temperature = Some(temperature);
request.max_tokens = text_generation_config.max_response_tokens;
request.model = text_generation_config.model_id.clone();
request.temperature = Some(temperature as f64);
request.max_tokens = text_generation_config.max_response_tokens as usize;
if let Ok(request_as_json) = serde_json::to_string(&request) {
tracing::trace!(
@@ -175,19 +161,27 @@ impl ControllerTrait for Controller {
);
}
let response = self.inner.client.create_message(request).await?;
let response = self.inner.client.messages(request).await?;
tracing::trace!(?response, "Got response from Anthropic create message API");
// response.content usually contains a single element, but we support handling multiple to account for all possibilities
let mut text_parts = vec![];
for content in response.content {
let content_type = content.content_type;
match content_type {
ContentType::Text => {
text_parts.push(content.text);
} // There are no other content types to handle yet, but there may be in the future
match content {
ContentBlock::Text { text } => {
text_parts.push(text);
}
ContentBlock::Image {
source,
media_type,
data: _,
} => {
text_parts.push(format!(
"The model responded with an image of type {}: {}",
media_type, source
));
}
}
}

View File

@@ -7,8 +7,8 @@ pub use controller::Controller;
use super::super::AgentInstantiationError;
use super::super::AgentInstantiationResult;
use super::controller::ControllerType;
use super::ConfigTrait;
use super::controller::ControllerType;
pub fn create_controller_from_yaml_value_config(
agent_id: &str,

View File

@@ -1,8 +1,8 @@
use anthropic_rs::completion::message::{Content, ContentType, Message, MessageRequest, Role};
use anthropic::types::{ContentBlock, Message, MessagesRequest, MessagesRequestBuilder, Role};
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessageRequest {
pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessagesRequest {
let mut messages = vec![];
for message in llm_messages {
@@ -14,8 +14,7 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
}
};
let content = vec![Content {
content_type: ContentType::Text,
let content = vec![ContentBlock::Text {
text: message.message_text,
}];
@@ -24,9 +23,9 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
messages.push(message);
}
MessageRequest {
stream: false,
messages,
..Default::default()
}
MessagesRequestBuilder::default()
.messages(messages)
.stream(false)
.build()
.expect("Failed to build messages request")
}

View File

@@ -1,11 +1,11 @@
use crate::{agent::AgentPurpose, conversation::llm::Conversation};
use super::{
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
entity::{
ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
TextToSpeechParams, TextToSpeechResult,
},
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
};
pub trait ControllerTrait {

View File

@@ -67,9 +67,7 @@ impl AgentProvider {
wiki_url: Some("https://en.wikipedia.org/wiki/Anthropic"),
sign_up_url: Some("https://console.anthropic.com/"),
models_list_url: Some("https://docs.anthropic.com/en/docs/about-claude/models"),
supported_purposes: vec![
AgentPurpose::TextGeneration,
],
supported_purposes: vec![AgentPurpose::TextGeneration],
},
Self::Groq => AgentProviderInfo {
id: Self::Groq.to_static_str(),
@@ -79,10 +77,7 @@ impl AgentProvider {
wiki_url: Some("https://en.wikipedia.org/wiki/Groq"),
sign_up_url: Some("https://console.groq.com/login"),
models_list_url: Some("https://console.groq.com/docs/models"),
supported_purposes: vec![
AgentPurpose::TextGeneration,
AgentPurpose::SpeechToText,
],
supported_purposes: vec![AgentPurpose::TextGeneration, AgentPurpose::SpeechToText],
},
Self::LocalAI => AgentProviderInfo {
id: Self::LocalAI.to_static_str(),
@@ -106,9 +101,7 @@ impl AgentProvider {
wiki_url: Some("https://en.wikipedia.org/wiki/Mistral_AI"),
sign_up_url: Some("https://auth.mistral.ai/ui/registration"),
models_list_url: Some("https://docs.mistral.ai/getting-started/models/"),
supported_purposes: vec![
AgentPurpose::TextGeneration,
],
supported_purposes: vec![AgentPurpose::TextGeneration],
},
Self::Ollama => AgentProviderInfo {
id: Self::Ollama.to_static_str(),
@@ -118,9 +111,7 @@ impl AgentProvider {
wiki_url: None,
sign_up_url: None,
models_list_url: Some("https://ollama.com/library"),
supported_purposes: vec![
AgentPurpose::TextGeneration,
],
supported_purposes: vec![AgentPurpose::TextGeneration],
},
Self::OpenAI => AgentProviderInfo {
id: Self::OpenAI.to_static_str(),
@@ -160,9 +151,7 @@ impl AgentProvider {
wiki_url: None,
sign_up_url: Some("https://openrouter.ai/"),
models_list_url: Some("https://openrouter.ai/models"),
supported_purposes: vec![
AgentPurpose::TextGeneration,
],
supported_purposes: vec![AgentPurpose::TextGeneration],
},
Self::TogetherAI => AgentProviderInfo {
id: Self::TogetherAI.to_static_str(),
@@ -172,9 +161,7 @@ impl AgentProvider {
wiki_url: None,
sign_up_url: Some("https://api.together.ai/signup"),
models_list_url: Some("https://api.together.xyz/models"),
supported_purposes: vec![
AgentPurpose::TextGeneration,
],
supported_purposes: vec![AgentPurpose::TextGeneration],
},
}
}

View File

@@ -58,6 +58,9 @@ pub struct TextGenerationConfig {
#[serde(default)]
pub max_response_tokens: Option<u32>,
#[serde(default)]
pub max_completion_tokens: Option<u32>,
#[serde(default)]
pub max_context_tokens: u32,
}
@@ -69,6 +72,7 @@ impl Default for TextGenerationConfig {
prompt: Some(default_prompt().to_owned()),
temperature: super::super::default_temperature(),
max_response_tokens: Some(16_384),
max_completion_tokens: None,
max_context_tokens: 128_000,
}
}

View File

@@ -1,36 +1,36 @@
use std::ops::Deref;
use async_openai::{
Client as OpenAIClient,
config::OpenAIConfig,
types::{
ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageRequestArgs,
CreateSpeechRequestArgs, CreateTranscriptionRequestArgs,
},
Client as OpenAIClient,
};
use super::super::ControllerTrait;
use crate::{
agent::{
AgentPurpose,
provider::{
entity::{ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult},
openai::utils::convert_string_to_enum,
},
AgentPurpose,
},
strings,
};
use crate::{
agent::{
provider::{
entity::{TextGenerationParams, TextGenerationResult},
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
entity::{TextGenerationParams, TextGenerationResult},
},
utils::base64_decode,
},
conversation::llm::{
shorten_messages_list_to_context_size, Author as LLMAuthor,
Conversation as LLMConversation, Message as LLMMessage,
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
shorten_messages_list_to_context_size,
},
};
@@ -144,6 +144,10 @@ impl ControllerTrait for Controller {
request_builder.max_tokens(max_response_tokens);
}
if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens {
request_builder.max_completion_tokens(max_completion_tokens);
}
let request = request_builder.build()?;
if let Ok(request_as_json) = serde_json::to_string(&request) {

View File

@@ -13,8 +13,8 @@ pub(super) use config::TextToSpeechConfig;
use super::super::AgentInstantiationError;
use super::super::AgentInstantiationResult;
use super::controller::ControllerType;
use super::ConfigTrait;
use super::controller::ControllerType;
pub fn create_controller_from_yaml_value_config(
agent_id: &str,

View File

@@ -93,6 +93,7 @@ impl TryInto<OpenAITextGenerationConfig> for TextGenerationConfig {
prompt: self.prompt,
temperature: self.temperature,
max_response_tokens: self.max_response_tokens,
max_completion_tokens: None,
max_context_tokens: self.max_context_tokens,
})
}

View File

@@ -7,20 +7,20 @@ use super::super::ControllerTrait;
use crate::agent::utils::base64_decode;
use crate::{
agent::provider::{
entity::{TextGenerationParams, TextGenerationResult},
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
entity::{TextGenerationParams, TextGenerationResult},
},
conversation::llm::{
shorten_messages_list_to_context_size, Author as LLMAuthor,
Conversation as LLMConversation, Message as LLMMessage,
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
shorten_messages_list_to_context_size,
},
};
use crate::{
agent::{
AgentPurpose,
provider::entity::{
ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult,
},
AgentPurpose,
},
strings,
};

View File

@@ -21,8 +21,8 @@ pub use controller::Controller;
use super::super::AgentInstantiationError;
use super::super::AgentInstantiationResult;
use super::controller::ControllerType;
use super::ConfigTrait;
use super::controller::ControllerType;
pub fn create_controller_from_yaml_value_config(
agent_id: &str,

View File

@@ -1,4 +1,4 @@
use base64::{engine::general_purpose::STANDARD, Engine as _};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use crate::{
agent::{

View File

@@ -1,11 +1,11 @@
use std::sync::Arc;
use std::{future::Future, pin::Pin};
use mxlink::matrix_sdk::Room;
use mxlink::matrix_sdk::media::{MediaFormat, MediaRequestParameters};
use mxlink::matrix_sdk::ruma::{
events::room::MediaSource, MilliSecondsSinceUnixEpoch, OwnedUserId,
MilliSecondsSinceUnixEpoch, OwnedUserId, events::room::MediaSource,
};
use mxlink::matrix_sdk::Room;
use mxlink::{
InitConfig, LoginConfig, LoginCredentials, LoginEncryption, MatrixLink, PersistenceConfig,
@@ -140,6 +140,10 @@ impl Bot {
&self.inner.config.command_prefix
}
pub(crate) fn post_join_self_introduction_enabled(&self) -> bool {
self.inner.config.room.post_join_self_introduction_enabled
}
pub(crate) fn homeserver_name(&self) -> &str {
&self.inner.config.homeserver.server_name
}

View File

@@ -5,7 +5,7 @@ use anyhow::anyhow;
use crate::agent::AgentPurpose;
pub use crate::entity::cfg::{defaults as cfg_defaults, env as cfg_env, Config};
pub use crate::entity::cfg::{Config, defaults as cfg_defaults, env as cfg_env};
pub fn load() -> anyhow::Result<Config> {
let config_file_path = env::var(cfg_env::BAIBOT_CONFIG_FILE_PATH)
@@ -35,6 +35,9 @@ pub fn load() -> anyhow::Result<Config> {
}
cfg_env::BAIBOT_USER_NAME => config.user.name = value,
cfg_env::BAIBOT_COMMAND_PREFIX => config.command_prefix = value,
cfg_env::BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED => {
config.room.post_join_self_introduction_enabled = value.parse::<bool>()?;
}
cfg_env::BAIBOT_LOGGING => {
config.logging = value;
}

View File

@@ -1,9 +1,9 @@
use mxlink::matrix_sdk::{
ruma::{
api::client::receipt::create_receipt::v3::ReceiptType,
events::room::message::OriginalSyncRoomMessageEvent, OwnedEventId,
},
Room,
ruma::{
OwnedEventId, api::client::receipt::create_receipt::v3::ReceiptType,
events::room::message::OriginalSyncRoomMessageEvent,
},
};
use mxlink::{CallbackError, MessageResponseType};

View File

@@ -1,12 +1,12 @@
use mxlink::matrix_sdk::{
ruma::{
events::{
room::message::Relation, AnySyncMessageLikeEvent, AnySyncTimelineEvent,
SyncMessageLikeEvent,
},
OwnedEventId, OwnedUserId,
},
Room,
ruma::{
OwnedEventId, OwnedUserId,
events::{
AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent,
room::message::Relation,
},
},
};
use mxlink::CallbackError;

View File

@@ -1,9 +1,9 @@
use mxlink::{
matrix_sdk::{
ruma::events::{room::member::StrippedRoomMemberEvent, AnySyncTimelineEvent},
Room,
},
InvitationDecision,
matrix_sdk::{
Room,
ruma::events::{AnySyncTimelineEvent, room::member::StrippedRoomMemberEvent},
},
};
use mxlink::CallbackError;

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
use super::AccessControllerType;

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
let mut message = String::new();

View File

@@ -4,5 +4,5 @@ pub mod help;
mod room_local_agent_managers;
mod users;
pub use determination::{determine_controller, AccessControllerType};
pub use determination::{AccessControllerType, determine_controller};
pub use dispatching::dispatch_controller;

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
pub async fn handle_get(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
let message = match &message_context

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
pub async fn handle_get(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
let message = match &message_context.global_config().access.user_patterns {

View File

@@ -3,15 +3,15 @@ mod tests;
use mxlink::MessageResponseType;
use crate::agent::provider::{ControllerTrait, PingResult};
use crate::agent::PublicIdentifier;
use crate::agent::{create_from_provider_and_yaml_value_config, AgentDefinition};
use crate::agent::provider::{ControllerTrait, PingResult};
use crate::agent::{AgentDefinition, create_from_provider_and_yaml_value_config};
use crate::agent::{AgentInstance, AgentProvider};
use crate::controller::utils::get_text_body_or_complain;
use crate::entity::globalconfig::GlobalConfigurationManager;
use crate::entity::roomconfig::RoomConfigurationManager;
use crate::strings;
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
struct ParsedAgentConfig {
agent: AgentInstance,

View File

@@ -1,9 +1,9 @@
use mxlink::MessageResponseType;
use crate::entity::{
globalconfig::GlobalConfigurationManager, roomconfig::RoomConfigurationManager, MessageContext,
MessageContext, globalconfig::GlobalConfigurationManager, roomconfig::RoomConfigurationManager,
};
use crate::{agent::PublicIdentifier, strings, Bot};
use crate::{Bot, agent::PublicIdentifier, strings};
pub async fn handle(
bot: &Bot,

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{agent::PublicIdentifier, entity::MessageContext, strings, Bot};
use crate::{Bot, agent::PublicIdentifier, entity::MessageContext, strings};
pub async fn handle(
bot: &Bot,

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
// Anyone can access this help command, because certain subcommands ("list")

View File

@@ -2,7 +2,7 @@ use mxlink::MessageResponseType;
use crate::agent::AgentPurpose;
use crate::strings;
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
let agents = bot

View File

@@ -1,4 +1,4 @@
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
pub mod create;
pub mod delete;
@@ -7,7 +7,7 @@ pub mod determination;
pub mod help;
pub mod list;
pub use determination::{determine_controller, AgentControllerType};
pub use determination::{AgentControllerType, determine_controller};
pub async fn dispatch_controller(
handler: &AgentControllerType,

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
pub async fn handle_get<T>(
bot: &Bot,

View File

@@ -1,7 +1,8 @@
use crate::{
agent::{AgentPurpose, PublicIdentifier},
entity::roomconfig::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
},
};
@@ -54,6 +55,11 @@ pub enum ConfigSpeechToTextSettingRelatedControllerType {
GetFlowType,
SetFlowType(Option<SpeechToTextFlowType>),
GetMsgTypeForNonThreadedOnlyTranscribedMessages,
SetMsgTypeForNonThreadedOnlyTranscribedMessages(
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
),
GetLanguage,
SetLanguage(Option<String>),
}

View File

@@ -1,7 +1,13 @@
#[cfg(test)]
mod tests;
use crate::{controller::ControllerType, entity::roomconfig::SpeechToTextFlowType, strings};
use crate::{
controller::ControllerType,
entity::roomconfig::{
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
},
strings,
};
use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType;
@@ -48,6 +54,53 @@ pub(super) fn determine(
));
}
// msg_type_for_non_threaded_only_transcribed_messages
if let Some(remaining_text) =
text.strip_prefix("msg-type-for-non-threaded-only-transcribed-messages")
{
let remaining_text = remaining_text.trim();
if !remaining_text.is_empty() {
return Err(ControllerType::Error(
strings::cfg::configuration_getter_used_with_extra_text(
"msg-type-for-non-threaded-only-transcribed-messages",
remaining_text,
)
.to_owned(),
));
}
return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages);
}
if let Some(value_string) =
text.strip_prefix("set-msg-type-for-non-threaded-only-transcribed-messages")
{
let value_string = value_string.trim().to_owned();
let value_choice = if value_string.is_empty() {
None
} else {
let value_choice =
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::from_str(
&value_string.to_lowercase(),
);
if value_choice.is_none() {
return Err(ControllerType::Error(
strings::cfg::configuration_value_unrecognized(&value_string).to_owned(),
));
}
value_choice
};
return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(
value_choice,
));
}
// Language
if let Some(remaining_text) = text.strip_prefix("language") {

View File

@@ -1,5 +1,5 @@
use crate::strings;
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
use mxlink::MessageResponseType;
use super::controller_type::{

View File

@@ -1,5 +1,8 @@
use crate::entity::roomconfig::{RoomSettings, SpeechToTextFlowType};
use crate::{entity::MessageContext, Bot};
use crate::entity::roomconfig::{
RoomSettings, SpeechToTextFlowType,
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
};
use crate::{Bot, entity::MessageContext};
use super::super::controller_type::{
ConfigSpeechToTextSettingRelatedControllerType, SettingsStorageSource,
@@ -52,6 +55,39 @@ pub(super) async fn dispatch(
}
}
ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages => {
let value = &room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages;
setting_get::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(bot, message_context, value).await
}
ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(value) => {
let value = value.to_owned();
let setter_callback = Box::new(move |room_settings: &mut RoomSettings| {
room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages = value;
});
match config_type {
SettingsStorageSource::Room => {
room_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
bot,
message_context,
&value,
setter_callback,
)
.await
}
SettingsStorageSource::Global => {
global_setting_set::<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>(
bot,
message_context,
&value,
setter_callback,
)
.await
}
}
}
ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => {
let value = &room_settings.speech_to_text.language;
setting_get::<String>(bot, message_context, value).await

View File

@@ -1,7 +1,7 @@
use crate::entity::roomconfig::{
RoomSettings, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
};
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
use super::super::controller_type::{
ConfigTextGenerationSettingRelatedControllerType, SettingsStorageSource,

View File

@@ -1,7 +1,7 @@
use crate::entity::roomconfig::{
RoomSettings, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
};
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
use super::super::controller_type::{
ConfigTextToSpeechSettingRelatedControllerType, SettingsStorageSource,

View File

@@ -1,7 +1,7 @@
use mxlink::MessageResponseType;
use crate::entity::{roomconfig::RoomSettings, MessageContext};
use crate::{strings, Bot};
use crate::entity::{MessageContext, roomconfig::RoomSettings};
use crate::{Bot, strings};
pub async fn handle_set<T>(
bot: &Bot,

View File

@@ -1,9 +1,10 @@
use mxlink::MessageResponseType;
use crate::{
Bot,
agent::{AgentPurpose, PublicIdentifier},
entity::{globalconfig::GlobalConfigurationManager, MessageContext},
strings, Bot,
entity::{MessageContext, globalconfig::GlobalConfigurationManager},
strings,
};
pub async fn handle_get(

View File

@@ -1,14 +1,16 @@
use mxlink::MessageResponseType;
use crate::{
Bot,
entity::{
MessageContext,
roomconfig::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
},
MessageContext,
},
strings, Bot,
strings,
};
pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
@@ -346,6 +348,46 @@ fn build_section_speech_to_text(command_prefix: &str) -> String {
));
message.push_str("\n\n");
// Msg Type For Non Threaded Only Transcribed Messages
message.push_str(&format!(
"#### {}",
strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading()
));
message.push_str("\n\n");
message.push_str(strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro());
message.push('\n');
message.push_str(
&strings::help::cfg::the_following_configuration_values_are_recognized(
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::choices(),
),
);
message.push_str("\n\n");
message.push_str(&format!(
"- {}",
&strings::help::cfg::current_setting_show(
command_prefix,
"speech-to-text msg-type-for-non-threaded-only-transcribed-messages"
)
));
message.push('\n');
message.push_str(&format!(
"- {}",
&strings::help::cfg::current_setting_set(
command_prefix,
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages VALUE"
)
));
message.push('\n');
message.push_str(&format!(
"- {}",
&strings::help::cfg::current_setting_unset(
command_prefix,
"speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages"
)
));
message.push_str("\n\n");
// Language
message.push_str(&format!(

View File

@@ -1,7 +1,7 @@
use mxlink::MessageResponseType;
use crate::entity::{roomconfig::RoomSettings, MessageContext};
use crate::{strings, Bot};
use crate::entity::{MessageContext, roomconfig::RoomSettings};
use crate::{Bot, strings};
pub async fn handle_set<T>(
bot: &Bot,

View File

@@ -1,9 +1,10 @@
use mxlink::MessageResponseType;
use crate::{
Bot,
agent::{AgentPurpose, PublicIdentifier},
entity::MessageContext,
strings, Bot,
strings,
};
use crate::entity::roomconfig::RoomConfigurationManager;

View File

@@ -1,15 +1,16 @@
use mxlink::MessageResponseType;
use crate::{
Bot,
agent::{
utils::get_effective_agent_for_purpose, AgentInstance, AgentPurpose, ControllerTrait,
Manager as AgentManager, PublicIdentifier,
AgentInstance, AgentPurpose, ControllerTrait, Manager as AgentManager, PublicIdentifier,
utils::get_effective_agent_for_purpose,
},
entity::{
roomconfig::{RoomConfig, RoomSettingsHandler},
MessageContext, RoomConfigContext,
roomconfig::{RoomConfig, RoomSettingsHandler},
},
strings, Bot,
strings,
};
pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
@@ -507,6 +508,35 @@ async fn generate_speech_to_text_section(
flow_type_set_where,
));
// Msg Type For Non Threaded Only Transcribed Messages
let effective_msg_type_for_non_threaded_only_transcribed_messages =
room_config_context.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
let room_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
.room_config
.settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages;
let global_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context
.global_config
.fallback_room_settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages;
let msg_type_for_non_threaded_only_transcribed_messages_set_where =
if room_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
strings::cfg::status_badge_set_in_room_config()
} else if global_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() {
strings::cfg::status_badge_set_in_global_config()
} else {
strings::cfg::status_badge_using_hardcoded_default()
};
message.push_str(&strings::cfg::status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
effective_msg_type_for_non_threaded_only_transcribed_messages,
msg_type_for_non_threaded_only_transcribed_messages_set_where,
));
// Language
let effective_language = room_config_context.speech_to_text_language();

View File

@@ -1,30 +1,31 @@
use mxlink::matrix_sdk::ruma::events::room::message::AudioMessageEventContent;
use mxlink::matrix_sdk::ruma::OwnedEventId;
use mxlink::matrix_sdk::ruma::events::room::message::AudioMessageEventContent;
use mxlink::{MatrixLink, MessageResponseType};
use tracing::Instrument;
use crate::agent::provider::{
SpeechToTextParams, TextGenerationParams, TextGenerationPromptVariables,
};
use crate::agent::AgentInstance;
use crate::agent::AgentPurpose;
use crate::agent::ControllerTrait;
use crate::agent::provider::{
SpeechToTextParams, TextGenerationParams, TextGenerationPromptVariables,
};
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
use crate::conversation::matrix::MatrixMessageProcessingParams;
use crate::entity::roomconfig::{
SpeechToTextFlowType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
};
use crate::entity::MessagePayload;
use crate::entity::roomconfig::{
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
};
use crate::strings;
use crate::utils::text_to_speech::create_transcribed_message_text;
use crate::{
Bot,
conversation::{
create_llm_conversation_for_matrix_reply_chain, create_llm_conversation_for_matrix_thread,
matrix::create_list_of_bot_user_prefixes_to_strip,
},
entity::MessageContext,
Bot,
};
#[derive(Debug, PartialEq)]
@@ -71,21 +72,35 @@ pub async fn handle(
if let MessagePayload::Audio(audio_content) = &message_context.payload() {
original_message_is_audio = true;
let response_type = match speech_to_text_flow_type {
let (response_type, msg_type) = match speech_to_text_flow_type {
SpeechToTextFlowType::Ignore => {
tracing::debug!("Intentionally ignoring audio message");
return Ok(());
}
SpeechToTextFlowType::TranscribeAndGenerateText => {
tracing::debug!("Will be transcribing and possibly generating text..");
MessageResponseType::InThread(message_context.thread_info().clone())
(
MessageResponseType::InThread(message_context.thread_info().clone()),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
)
}
SpeechToTextFlowType::OnlyTranscribe => {
tracing::debug!("Will only be transcribing audio to text..");
if message_context.thread_info().is_thread_root_only() {
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone())
let msg_type = message_context
.room_config_context()
.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages();
(
MessageResponseType::Reply(
message_context.thread_info().root_event_id.clone(),
),
msg_type,
)
} else {
MessageResponseType::InThread(message_context.thread_info().clone())
(
MessageResponseType::InThread(message_context.thread_info().clone()),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice,
)
}
}
};
@@ -94,8 +109,14 @@ pub async fn handle(
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
}
let Some(speech_to_text_created_event_id_result) =
handle_stage_speech_to_text(bot, message_context, audio_content, response_type).await
let Some(speech_to_text_created_event_id_result) = handle_stage_speech_to_text(
bot,
message_context,
audio_content,
response_type,
msg_type,
)
.await
else {
return Ok(());
};
@@ -282,6 +303,7 @@ async fn handle_stage_speech_to_text(
message_context: &MessageContext,
audio_content: &AudioMessageEventContent,
response_type: MessageResponseType,
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
) -> Option<OwnedEventId> {
let agent = get_effective_agent_for_purpose_or_complain(
bot,
@@ -302,7 +324,7 @@ async fn handle_stage_speech_to_text(
.react_no_fail(
message_context.room(),
message_context.event_id().clone(),
AgentPurpose::SpeechToText.emoji().to_owned(),
strings::PROGRESS_INDICATOR_EMOJI.to_owned(),
)
.await;
@@ -312,6 +334,7 @@ async fn handle_stage_speech_to_text(
&agent,
audio_content,
response_type.clone(),
msg_type,
)
.await;
@@ -572,6 +595,7 @@ async fn handle_stage_speech_to_text_actual_transcribing(
agent: &AgentInstance,
audio_content: &AudioMessageEventContent,
response_type: MessageResponseType,
msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
) -> anyhow::Result<OwnedEventId> {
let src = &audio_content.source;
@@ -626,9 +650,6 @@ async fn handle_stage_speech_to_text_actual_transcribing(
//
// When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead,
// to make it clear to users that it's a transcription.
//
// Regardless of how we post this message, it will be posted as a notice,
// which can indicate to the bot (for potential future text-generation purposes) that this message is not a bot message.
let (transcribed_text, annotate_message_with_reaction) =
if let MessageResponseType::InThread(_) = response_type {
(
@@ -639,10 +660,22 @@ async fn handle_stage_speech_to_text_actual_transcribing(
(speech_to_text_result.text, true)
};
let result = bot
.messaging()
.send_notice_markdown_no_fail(message_context.room(), transcribed_text, response_type)
.await;
let result = match msg_type {
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => {
bot.messaging()
.send_text_markdown_no_fail(message_context.room(), transcribed_text, response_type)
.await
}
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
bot.messaging()
.send_notice_markdown_no_fail(
message_context.room(),
transcribed_text,
response_type,
)
.await
}
};
let event_id = result
.map(|result| result.event_id)

View File

@@ -4,8 +4,8 @@ mod tests;
use super::chat_completion::ChatCompletionControllerType;
use crate::{
entity::{
roomconfig::TextGenerationPrefixRequirementType, InteractionTrigger, MessageContext,
MessagePayload,
InteractionTrigger, MessageContext, MessagePayload,
roomconfig::TextGenerationPrefixRequirementType,
},
strings,
};

View File

@@ -142,8 +142,7 @@ fn determine_text_controller() {
// This test case is the same as the one above, just with a different prefix requirement setting.
// We expect the same result.
TestCase {
name:
"Regular message with bot mention triggers completion (command prefix requirement)",
name: "Regular message with bot mention triggers completion (command prefix requirement)",
input: "Regular text goes here",
is_mentioning_bot: true,
room_text_generation_prefix_requirement_type:

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
use super::ControllerType;

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
let sender_can_manage_global_config = message_context.sender_can_manage_global_config();

View File

@@ -2,14 +2,15 @@ use mxlink::{MatrixLink, MessageResponseType};
use tracing::Instrument;
use crate::agent::provider::ImageGenerationParams;
use crate::agent::AgentPurpose;
use crate::agent::ControllerTrait;
use crate::agent::provider::ImageGenerationParams;
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
use crate::controller::utils::mime::get_file_extension;
use crate::conversation::create_llm_conversation_for_matrix_thread;
use crate::conversation::matrix::MatrixMessageProcessingParams;
use crate::strings;
use crate::{entity::MessageContext, Bot};
use crate::{Bot, entity::MessageContext};
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
const STICKER_SIZE: &str = "256x256";
@@ -81,7 +82,10 @@ pub async fn handle_image(
.await;
}
let attachment_body_text = format!("Generated image based on: {}", actual_prompt);
let attachment_body_text = format!(
"generated-image.{}",
get_file_extension(&response.mime_type)
);
let mut event_content = matrix_link
.media()
@@ -157,7 +161,10 @@ pub async fn handle_sticker(
.instrument(span)
.await?;
let attachment_body_text = format!("Generated sticker image based on: {}", original_prompt);
let attachment_body_text = format!(
"generated-sticker.{}",
get_file_extension(&response.mime_type)
);
let mut event_content = matrix_link
.media()

View File

@@ -68,46 +68,50 @@ mod tests {
// Multiple complex user messages dispersed with assistant messages
TestCase {
original_prompt: "Generate a picture of an elephant",
messages: vec![Message {
author: Author::User,
message_text: "Must be blue".to_owned(),
timestamp,
},
Message {
author: Author::Assistant,
message_text: "Whatever".to_owned(),
timestamp,
},
Message {
author: Author::User,
message_text: "Must be 3-legged.\nMust be flying.".to_owned(),
timestamp,
}],
messages: vec![
Message {
author: Author::User,
message_text: "Must be blue".to_owned(),
timestamp,
},
Message {
author: Author::Assistant,
message_text: "Whatever".to_owned(),
timestamp,
},
Message {
author: Author::User,
message_text: "Must be 3-legged.\nMust be flying.".to_owned(),
timestamp,
},
],
expected_prompt: "Generate a picture of an elephant\nOther criteria:\n- Must be blue\n- Must be 3-legged.. Must be flying.",
},
// "Again" is ignored.
TestCase {
original_prompt: "Generate a picture of a grizzly bear",
messages: vec![Message {
author: Author::User,
message_text: "Must be blue".to_owned(),
timestamp,
},
Message {
author: Author::Assistant,
message_text: "Whatever".to_owned(),
timestamp,
},
Message {
author: Author::User,
message_text: "Again".to_owned(),
timestamp,
},
Message {
author: Author::User,
message_text: "again".to_owned(),
timestamp,
}],
messages: vec![
Message {
author: Author::User,
message_text: "Must be blue".to_owned(),
timestamp,
},
Message {
author: Author::Assistant,
message_text: "Whatever".to_owned(),
timestamp,
},
Message {
author: Author::User,
message_text: "Again".to_owned(),
timestamp,
},
Message {
author: Author::User,
message_text: "again".to_owned(),
timestamp,
},
],
expected_prompt: "Generate a picture of a grizzly bear\nOther criteria:\n- Must be blue",
},
];

View File

@@ -1,13 +1,21 @@
use mxlink::MessageResponseType;
use crate::entity::RoomConfigContext;
use crate::{strings, Bot};
use crate::{Bot, strings};
pub async fn handle(
bot: &Bot,
room: &mxlink::matrix_sdk::Room,
room_config_context: &RoomConfigContext,
) -> anyhow::Result<()> {
if !bot.post_join_self_introduction_enabled() {
tracing::debug!(
"Post-join self-introduction is disabled - not sending introduction message"
);
return Ok(());
}
let agent_manager = bot.agent_manager();
bot.messaging()

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{agent::AgentProvider, entity::MessageContext, strings, Bot};
use crate::{Bot, agent::AgentProvider, entity::MessageContext, strings};
use super::ControllerType;

View File

@@ -3,9 +3,9 @@ use std::ops::Deref;
use mxlink::MatrixLink;
use crate::{
Bot,
agent::AgentPurpose,
entity::{MessageContext, MessagePayload},
Bot,
};
mod text_to_speech;

View File

@@ -1,7 +1,7 @@
use mxlink::{MatrixLink, MessageResponseType};
use mxlink::matrix_sdk::ruma::{
events::room::message::TextMessageEventContent, OwnedEventId, OwnedUserId,
OwnedEventId, OwnedUserId, events::room::message::TextMessageEventContent,
};
use crate::entity::roomconfig::{
@@ -9,8 +9,8 @@ use crate::entity::roomconfig::{
};
use crate::{
agent::AgentPurpose, controller::utils::agent::get_effective_agent_for_purpose_or_complain,
entity::MessageContext, Bot,
Bot, agent::AgentPurpose,
controller::utils::agent::get_effective_agent_for_purpose_or_complain, entity::MessageContext,
};
pub(super) async fn handle(
@@ -34,7 +34,9 @@ pub(super) async fn handle(
reacted_to_event_sender_id,
matrix_link.user_id(),
) {
tracing::debug!("Ignoring request for on-demand text-to-speech (via reaction) due to room configuration");
tracing::debug!(
"Ignoring request for on-demand text-to-speech (via reaction) due to room configuration"
);
return Ok(());
}

View File

@@ -1,6 +1,6 @@
use mxlink::MessageResponseType;
use crate::{entity::MessageContext, strings, Bot};
use crate::{Bot, entity::MessageContext, strings};
use super::ControllerType;

View File

@@ -1,16 +1,17 @@
use mxlink::MessageResponseType;
use crate::{
Bot,
agent::{
utils::{get_effective_agent_for_purpose, AgentForPurposeDeterminationError},
AgentInstance, AgentPurpose,
utils::{AgentForPurposeDeterminationError, get_effective_agent_for_purpose},
},
entity::MessageContext,
strings, Bot,
strings,
};
pub async fn get_effective_agent_for_purpose_or_complain<'a>(
bot: &'a Bot,
pub async fn get_effective_agent_for_purpose_or_complain(
bot: &Bot,
message_context: &MessageContext,
agent_purpose: AgentPurpose,
response_type: MessageResponseType,

View File

@@ -0,0 +1,17 @@
use mxlink::mime;
pub fn get_file_extension(mime_type: &mime::Mime) -> String {
match (mime_type.type_(), mime_type.subtype()) {
(mime::AUDIO, mime::BASIC) => "au",
(mime::AUDIO, mime::MPEG) => "mp3",
(mime::AUDIO, mime::MP4) => "m4a",
(mime::AUDIO, mime::OGG) => "ogg",
(mime::IMAGE, mime::BMP) => "bmp",
(mime::IMAGE, mime::GIF) => "gif",
(mime::IMAGE, mime::JPEG) => "jpg",
(mime::IMAGE, mime::PNG) => "png",
(mime::IMAGE, mime::SVG) => "svg",
_ => "bin",
}
.to_string()
}

View File

@@ -1,11 +1,12 @@
use mxlink::MessageResponseType;
use crate::{
entity::{MessageContext, MessagePayload},
Bot,
entity::{MessageContext, MessagePayload},
};
pub mod agent;
pub(super) mod mime;
pub mod text_to_speech;
pub async fn get_text_body_or_complain<'a>(

View File

@@ -3,10 +3,12 @@ use mxlink::{MatrixLink, MessageResponseType};
use tracing::Instrument;
use crate::controller::utils::mime::get_file_extension;
use crate::{
agent::{provider::TextToSpeechParams, AgentInstance, AgentPurpose, ControllerTrait},
Bot,
agent::{AgentInstance, AgentPurpose, ControllerTrait, provider::TextToSpeechParams},
entity::MessageContext,
strings, Bot,
strings,
};
pub async fn generate_and_send_tts_for_message(
@@ -117,7 +119,10 @@ async fn do_generate_and_send_tts_for_message(
}
};
let attachment_body_text = strings::text_to_speech::alternate_body_text();
let attachment_body_text = format!(
"generated-speech.{}",
get_file_extension(&text_to_speech_result.mime_type)
);
let event_content = matrix_link
.media()

View File

@@ -25,8 +25,8 @@ fn test_messages_by_the_bot_are_identified_correctly() {
}
#[test]
fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_considered_sent_by_user(
) {
fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_considered_sent_by_user()
{
let bot_user_id =
OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID");

View File

@@ -1,6 +1,6 @@
use tiktoken_rs::CoreBPE;
use tiktoken_rs::get_bpe_from_tokenizer;
use tiktoken_rs::tokenizer;
use tiktoken_rs::CoreBPE;
use super::{Author, Message};

View File

@@ -45,7 +45,7 @@ fn convert_bot_notice_message(
// Such transcriptions are prefixed with an emoji and blockquoted.
// If we find a notice that doesn't match this pattern, we skip it.
//
// It should be noted that transcriptions are sometimes posted as regular notice messages which do not include
// It should be noted that transcriptions are sometimes posted as regular notice (or even text) messages which do not include
// the `> 🦻` formatting. This function will not handle these properly.
if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) {

View File

@@ -1,5 +1,5 @@
use mxlink::matrix_sdk::ruma::OwnedRoomId;
use mxlink::matrix_sdk::Room;
use mxlink::matrix_sdk::ruma::OwnedRoomId;
use mxlink::MatrixLink;
use quick_cache::sync::Cache;

View File

@@ -1,6 +1,6 @@
use mxlink::matrix_sdk::Room;
use mxlink::matrix_sdk::deserialized_responses::TimelineEvent;
use mxlink::matrix_sdk::ruma::OwnedEventId;
use mxlink::matrix_sdk::Room;
use quick_cache::sync::Cache;

View File

@@ -5,17 +5,17 @@ use std::sync::Arc;
use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId};
use mxlink::matrix_sdk::{
Room,
deserialized_responses::TimelineEvent,
ruma::events::{
relation::Thread,
room::message::{
sanitize::remove_plain_reply_fallback, MessageType, OriginalSyncRoomMessageEvent,
Relation, RoomMessageEventContent,
},
AnyMessageLikeEventContent, AnySyncMessageLikeEvent, AnySyncTimelineEvent,
SyncMessageLikeEvent,
relation::Thread,
room::message::{
MessageType, OriginalSyncRoomMessageEvent, Relation, RoomMessageEventContent,
sanitize::remove_plain_reply_fallback,
},
},
Room,
};
use mxlink::{MatrixLink, ThreadGetMessagesParams, ThreadInfo};

View File

@@ -1,14 +1,14 @@
use std::sync::Arc;
use mxlink::matrix_sdk::ruma::OwnedEventId;
use mxlink::MatrixLink;
use mxlink::matrix_sdk::ruma::OwnedEventId;
use crate::conversation::matrix::MatrixMessage;
use super::llm::{convert_matrix_message_to_llm_message, Conversation, Message};
use super::llm::{Conversation, Message, convert_matrix_message_to_llm_message};
use super::matrix::{
get_matrix_messages_in_reply_chain, get_matrix_messages_in_thread, process_matrix_messages,
MatrixMessageProcessingParams, RoomEventFetcher,
MatrixMessageProcessingParams, RoomEventFetcher, get_matrix_messages_in_reply_chain,
get_matrix_messages_in_thread, process_matrix_messages,
};
pub async fn create_llm_conversation_for_matrix_thread(

View File

@@ -19,6 +19,9 @@ pub struct Config {
#[serde(default = "super::defaults::command_prefix")]
pub command_prefix: String,
#[serde(default)]
pub room: ConfigRoom,
pub access: ConfigAccess,
pub agents: ConfigAgents,
@@ -37,6 +40,7 @@ impl Config {
self.homeserver.validate()?;
self.user.validate()?;
self.persistence.validate()?;
self.room.validate()?;
self.access.validate()?;
if self.command_prefix.is_empty() {
@@ -243,6 +247,27 @@ impl PersistenceConfig {
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ConfigRoom {
#[serde(default = "super::defaults::room_post_join_self_introduction_enabled")]
pub post_join_self_introduction_enabled: bool,
}
impl ConfigRoom {
pub fn validate(&self) -> anyhow::Result<()> {
Ok(())
}
}
impl Default for ConfigRoom {
fn default() -> Self {
Self {
post_join_self_introduction_enabled:
super::defaults::room_post_join_self_introduction_enabled(),
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ConfigAccess {
// Contains the admin whitelist patterns before parsing into regex.

View File

@@ -18,6 +18,10 @@ pub(super) fn command_prefix() -> String {
COMMAND_PREFIX.to_owned()
}
pub(super) fn room_post_join_self_introduction_enabled() -> bool {
true
}
pub(super) fn persistence_data_dir_path() -> Option<String> {
None
}

View File

@@ -11,6 +11,9 @@ pub const BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE: &str =
pub const BAIBOT_COMMAND_PREFIX: &str = "BAIBOT_COMMAND_PREFIX";
pub const BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED: &str =
"BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED";
pub const BAIBOT_LOGGING: &str = "BAIBOT_LOGGING";
pub const BAIBOT_ACCESS_ADMIN_PATTERNS: &str = "BAIBOT_ACCESS_ADMIN_PATTERNS";

View File

@@ -1,11 +1,11 @@
use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId, RoomId};
use mxlink::matrix_sdk::Room;
use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId, RoomId};
use mxlink::ThreadInfo;
use super::{
globalconfig::GlobalConfig, roomconfig::RoomConfig, MessagePayload, RoomConfigContext,
TriggerEventInfo,
MessagePayload, RoomConfigContext, TriggerEventInfo, globalconfig::GlobalConfig,
roomconfig::RoomConfig,
};
#[derive(Debug)]

View File

@@ -4,9 +4,9 @@ use super::globalconfig::GlobalConfig;
use super::roomconfig::RoomConfig;
use crate::entity::roomconfig::{
defaults as roomconfig_defaults, SpeechToTextFlowType, TextGenerationAutoUsage,
TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType, defaults as roomconfig_defaults,
};
#[derive(Debug)]
@@ -37,6 +37,24 @@ impl RoomConfigContext {
.unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE)
}
pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(
&self,
) -> SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
self.room_config
.settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages
.or({
self.global_config
.fallback_room_settings
.speech_to_text
.msg_type_for_non_threaded_only_transcribed_messages
})
.unwrap_or(
roomconfig_defaults::SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE,
)
}
pub fn speech_to_text_language(&self) -> Option<String> {
self.room_config
.settings

View File

@@ -1,4 +1,4 @@
use super::SpeechToTextFlowType;
use super::{SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages};
use super::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
@@ -15,3 +15,9 @@ pub const TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE: TextToSpeechUserMessagesFlowTy
pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType =
SpeechToTextFlowType::TranscribeAndGenerateText;
// While notice messages may be less desirable with other bots in the room,
// it's probably a better default for most people who enable "transcribe-only" mode.
pub const SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE:
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages =
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text;

View File

@@ -13,7 +13,9 @@ mod text_generation;
mod text_to_speech;
pub use handler::RoomSettingsHandler;
pub use speech_to_text::SpeechToTextFlowType;
pub use speech_to_text::{
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
};
pub use text_generation::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType};
pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType};
@@ -44,7 +46,9 @@ impl RoomConfigTrait for RoomConfig {}
impl RoomConfig {
pub async fn with_room(mut self, room: Room) -> Self {
tracing::trace!("Determining room members count to decide on a suitable text-generation/prefix-requirement-type default");
tracing::trace!(
"Determining room members count to decide on a suitable text-generation/prefix-requirement-type default"
);
let members = room.members(RoomMemberships::ACTIVE).await;
@@ -58,12 +62,19 @@ impl RoomConfig {
text_generation::TextGenerationPrefixRequirementType::No
};
tracing::info!(?members_count, ?prefix_requirement_type, "Determined text-generation/prefix-requirement-type based on room members count");
tracing::info!(
?members_count,
?prefix_requirement_type,
"Determined text-generation/prefix-requirement-type based on room members count"
);
prefix_requirement_type
}
Err(err) => {
tracing::error!(?err, "Failed to get members of room - will default text-generation/prefix-requirement-type to No");
tracing::error!(
?err,
"Failed to get members of room - will default text-generation/prefix-requirement-type to No"
);
text_generation::TextGenerationPrefixRequirementType::No
}
};

View File

@@ -4,6 +4,20 @@ use serde::{Deserialize, Serialize};
pub struct RoomSettingsSpeechToText {
pub flow_type: Option<SpeechToTextFlowType>,
/// Controls how the transcribed message is posted when dealing with:
/// - messages that only get transcribed (and do not trigger text-generation).
/// See `flow_type` and `SpeechToTextFlowType::OnlyTranscribe` for more details.
/// - incoming voice messages that are not part of a thread.
/// For messages that are part of a thread, we need to reply within the thread in a way (with a notice message)
/// that won't confuse the bot later, so we have no choice but to use a notice message.
///
/// Text-generation may happen either as a direct result of the incoming voice message or as part of a threaded conversation.
/// Transcribed messages should not be attributed to the bot for the purposes of text-generation,
/// so any time there's a chance of text-generation happening, we should use `SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice`
/// and optionally prefix the message with `> 🦻`.
pub msg_type_for_non_threaded_only_transcribed_messages:
Option<SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages>,
/// The language of the input audio.
/// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency.
pub language: Option<String>,
@@ -57,3 +71,39 @@ impl std::fmt::Display for SpeechToTextFlowType {
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)]
pub enum SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
/// Send the transcribed message text as a regular message
#[serde(rename = "text")]
Text,
/// Send the transcribed message text as a notice message
#[serde(rename = "notice")]
Notice,
}
impl SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
pub fn choices() -> Vec<Self> {
vec![Self::Text, Self::Notice]
}
pub fn from_str(s: &str) -> Option<Self> {
match s {
"text" => Some(Self::Text),
"notice" => Some(Self::Notice),
_ => None,
}
}
}
impl std::fmt::Display for SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => write!(f, "text"),
SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => {
write!(f, "notice")
}
}
}
}

View File

@@ -70,6 +70,8 @@ pub enum TextGenerationAutoUsage {
/// Text Generation is to be performed when the original message was sent as audio (voice).
/// The voice message would be transcribed to text (subject to other configuration)
/// and text generation would be triggered.
///
/// Also see `SpeechToTextFlowType`.
#[serde(rename = "only_for_voice")]
OnlyForVoice,

View File

@@ -5,8 +5,9 @@ use mxlink::helpers::account_data_config::RoomConfigManager as AccountDataRoomCo
pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler};
pub use entity::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType,
TextToSpeechUserMessagesFlowType,
};
pub type RoomConfigurationManager =

View File

@@ -6,5 +6,5 @@ mod entity;
mod strings;
mod utils;
pub use bot::{load_config, Bot};
pub use bot::{Bot, load_config};
pub use entity::cfg::Config;

View File

@@ -1,7 +1,7 @@
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::EnvFilter;
use tracing_subscriber::fmt::format::FmtSpan;
use baibot::{load_config, Bot, Config};
use baibot::{Bot, Config, load_config};
#[tokio::main]
async fn main() -> anyhow::Result<()> {

View File

@@ -1,10 +1,11 @@
use crate::{
agent::{
utils::AgentForPurposeDeterminationInfoConfigurationSource, AgentInstance, AgentPurpose,
PublicIdentifier,
AgentInstance, AgentPurpose, PublicIdentifier,
utils::AgentForPurposeDeterminationInfoConfigurationSource,
},
entity::roomconfig::{
SpeechToTextFlowType, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
},
utils::text::block_quote,
@@ -44,7 +45,9 @@ pub fn configuration_getter_used_with_extra_text(
getter_name: &str,
remaining_text: &str,
) -> String {
format!("You're invoking a getter command (`{getter_name}`), but passing additional text (`{remaining_text}`) as if you're invoking a setter.\n\nPerhaps you meant to invoke `set-{getter_name}`?")
format!(
"You're invoking a getter command (`{getter_name}`), but passing additional text (`{remaining_text}`) as if you're invoking a setter.\n\nPerhaps you meant to invoke `set-{getter_name}`?"
)
}
pub fn configuration_value_unrecognized(value: &str) -> String {
@@ -52,7 +55,10 @@ pub fn configuration_value_unrecognized(value: &str) -> String {
}
pub fn configuration_value_not_f32(value: &str) -> String {
format!("The value `{}` could not be converted to a [floating point number](https://en.wikipedia.org/wiki/Floating-point_arithmetic).", value)
format!(
"The value `{}` could not be converted to a [floating point number](https://en.wikipedia.org/wiki/Floating-point_arithmetic).",
value
)
}
pub fn status_room_config_handlers_heading() -> &'static str {
@@ -269,6 +275,16 @@ pub fn status_speech_to_text_entry_flow_type(
format!("- 🪄 Flow type: `{}` ({})\n", value, set_where)
}
pub fn status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages(
value: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
set_where: &str,
) -> String {
format!(
"- 🪄 Message type for non-threaded only-transcribed messages: `{}` ({})\n",
value, set_where
)
}
pub fn status_speech_to_text_entry_language(value: Option<String>, set_where: &str) -> String {
let formatted = match value {
Some(value) => format!("`{}` ({})", value, set_where),

View File

@@ -32,7 +32,9 @@ pub fn users_command_get(command_prefix: &str) -> String {
}
pub fn users_command_set(command_prefix: &str) -> String {
format!("- **Set** the list of allowed users: `{command_prefix} access set-users SPACE_SEPARATED_PATTERNS`")
format!(
"- **Set** the list of allowed users: `{command_prefix} access set-users SPACE_SEPARATED_PATTERNS`"
)
}
pub fn example_user_patterns(own_server_name: &str) -> String {
@@ -64,7 +66,9 @@ pub fn room_local_agent_managers_heading() -> String {
}
pub fn room_local_agent_managers_intro(command_prefix: &str) -> String {
format!("Room-local agent managers are users privileged to **create their own agents** (see `{command_prefix} agent`) in rooms.")
format!(
"Room-local agent managers are users privileged to **create their own agents** (see `{command_prefix} agent`) in rooms."
)
}
pub fn room_local_agent_managers_security_warning() -> String {
@@ -72,9 +76,13 @@ pub fn room_local_agent_managers_security_warning() -> String {
}
pub fn room_local_agent_managers_command_get(command_prefix: &str) -> String {
format!("- **Show** the currently allowed users: `{command_prefix} access room-local-agent-managers`")
format!(
"- **Show** the currently allowed users: `{command_prefix} access room-local-agent-managers`"
)
}
pub fn room_local_agent_managers_command_set(command_prefix: &str) -> String {
format!("- **Set** the list of allowed users: `{command_prefix} access set-room-local-agent-managers SPACE_SEPARATED_PATTERNS`")
format!(
"- **Set** the list of allowed users: `{command_prefix} access set-room-local-agent-managers SPACE_SEPARATED_PATTERNS`"
)
}

View File

@@ -3,7 +3,9 @@ pub fn heading() -> String {
}
pub fn intro(command_prefix: &str) -> String {
format!("An agent is an instantiation and configuration of some **☁️ provider** (see `{command_prefix} provider`).")
format!(
"An agent is an instantiation and configuration of some **☁️ provider** (see `{command_prefix} provider`)."
)
}
pub fn intro_handler_relation(command_prefix: &str) -> String {
@@ -25,7 +27,9 @@ pub fn list_agents(command_prefix: &str) -> String {
}
pub fn show_agent_details(command_prefix: &str) -> String {
format!("- **Show** full details for a given agent: `{command_prefix} agent details FULL_AGENT_IDENTIFIER`")
format!(
"- **Show** full details for a given agent: `{command_prefix} agent details FULL_AGENT_IDENTIFIER`"
)
}
pub fn create_agent_intro() -> &'static str {
@@ -33,11 +37,15 @@ pub fn create_agent_intro() -> &'static str {
}
pub fn create_agent_room_local(command_prefix: &str) -> String {
format!("\t- (Accessible in **this room only**) `{command_prefix} agent create-room-local PROVIDER_ID AGENT_ID`")
format!(
"\t- (Accessible in **this room only**) `{command_prefix} agent create-room-local PROVIDER_ID AGENT_ID`"
)
}
pub fn create_agent_global(command_prefix: &str) -> String {
format!("\t- (Accessible in **all rooms**) `{command_prefix} agent create-global PROVIDER_ID AGENT_ID`")
format!(
"\t- (Accessible in **all rooms**) `{command_prefix} agent create-global PROVIDER_ID AGENT_ID`"
)
}
pub fn create_agent_example(command_prefix: &str) -> String {

Some files were not shown because too many files have changed in this diff Show More