Compare commits
46 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2455117e41 | ||
|
|
3e9c110afc | ||
|
|
d9b5524c97 | ||
|
|
9b169a7d28 | ||
|
|
cb29419d75 | ||
|
|
51ca8c9948 | ||
|
|
12b938d2d1 | ||
|
|
f3d1b32ad7 | ||
|
|
3d3bd3c9f9 | ||
|
|
a25845e89e | ||
|
|
479f54b93d | ||
|
|
90ab6807ac | ||
|
|
5022f79bf5 | ||
|
|
2ba3b5a437 | ||
|
|
2819011b9d | ||
|
|
ede9065f77 | ||
|
|
5e61f5b3a3 | ||
|
|
8633e82f62 | ||
|
|
6aa3d70d57 | ||
|
|
75e29ee3fb | ||
|
|
eab7978b9f | ||
|
|
765e7c17b2 | ||
|
|
748d2b7fd4 | ||
|
|
527759dd02 | ||
|
|
3290255bad | ||
|
|
9b987395b3 | ||
|
|
4852d1fe92 | ||
|
|
8bd313f0d4 | ||
|
|
711e1099d6 | ||
|
|
91c8dd8f7d | ||
|
|
afc5572d6a | ||
|
|
73e13dcf2f | ||
|
|
2bebd109b1 | ||
|
|
7f7c58be1f | ||
|
|
47e5a464a0 | ||
|
|
85f751e514 | ||
|
|
95acad3558 | ||
|
|
304056c59a | ||
|
|
bedc0335f1 | ||
|
|
a8be8c3c1e | ||
|
|
826fa728a9 | ||
|
|
5aef8e8b2f | ||
|
|
f70f20181e | ||
|
|
fcdd4f39ee | ||
|
|
891adfec49 | ||
|
|
35ab79844b |
12
.github/workflows/workflow.yml
vendored
12
.github/workflows/workflow.yml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@1.93.0
|
||||
- name: Install SQLite3
|
||||
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
|
||||
- run: cargo test --all-features
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
steps:
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ github.repository }}
|
||||
@@ -58,14 +58,14 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Log in to the GitHub Container registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
ghcr.io/${{ github.repository }}
|
||||
|
||||
- name: Build and push Docker images
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Log in to the GitHub Container registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
26
CHANGELOG.md
26
CHANGELOG.md
@@ -1,3 +1,29 @@
|
||||
# (2026-03-24) Version 1.16.1
|
||||
|
||||
- (**Bugfix**) Fix compatibility with [async-openai](https://crates.io/crates/async-openai) 0.34.0 by populating the new `phase` field required for OpenAI Responses API message inputs. baibot does not currently distinguish between assistant `commentary` and `final_answer` turns, so using `None` preserves the previous behavior while remaining compatible with the updated crate.
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
|
||||
# (2026-03-20) Version 1.16.0
|
||||
|
||||
- (**Feature**) Add support for file attachments (`m.file` Matrix messages) in conversations. Files like PDFs, text documents, spreadsheets, code files, etc. are now downloaded and forwarded to the LLM alongside the conversation context, similar to how images (`m.image`) are already handled. See the [💬 Text Generation](./docs/features.md#-text-generation) documentation for details and known limitations.
|
||||
|
||||
- (**Improvement**) Use the [mime_guess](https://crates.io/crates/mime_guess) crate for MIME type detection from file extensions, replacing a hand-maintained mapping. This covers hundreds of file extensions out of the box.
|
||||
|
||||
|
||||
|
||||
# (2026-03-07) Version 1.15.0
|
||||
|
||||
- (**Feature**) Add support for authentication via access tokens (for [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service)/OIDC-enabled homeservers) as an alternative to password authentication. See [🔐 Authentication](./docs/configuration/authentication.md) for setup details. Thanks to [Taylor Southwick](https://github.com/twsouthwick) for the contribution in [#83](https://github.com/etkecc/baibot/pull/83)!
|
||||
|
||||
- (**Internal Improvement**) Pin the Rust toolchain to `1.93.0` in both CI and local development to avoid `matrix-sdk` build failures on newer stable toolchains.
|
||||
|
||||
- (**Internal Improvement**) Documentation updates.
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
|
||||
# (2026-02-18) Version 1.14.3
|
||||
|
||||
- (**Internal Improvement**) Add [Renovate](https://docs.renovatebot.com/) configuration for automated dependency updates
|
||||
|
||||
373
Cargo.lock
generated
373
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
10
Cargo.toml
10
Cargo.toml
@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
|
||||
readme = "README.md"
|
||||
keywords = ["matrix", "chat", "bot", "AI", "LLM"]
|
||||
include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
|
||||
version = "1.14.3"
|
||||
version = "1.16.1"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
@@ -17,7 +17,7 @@ path = "src/lib.rs"
|
||||
[dependencies]
|
||||
anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" }
|
||||
anyhow = "1.0.*"
|
||||
async-openai = { version = "0.33.0", features = ["audio", "chat-completion", "image", "responses"] }
|
||||
async-openai = { version = "0.34.0", features = ["audio", "chat-completion", "image", "responses"] }
|
||||
base64 = "0.22.*"
|
||||
chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
|
||||
# We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
|
||||
@@ -25,16 +25,16 @@ chrono = { version = "0.4.*", default-features = false, features = ["std", "now"
|
||||
matrix-sdk = { version = "0.16.0", default-features = false, features = ["native-tls"] }
|
||||
mime_guess = "2.0.*"
|
||||
mxidwc = "1.0.*"
|
||||
mxlink = ">=1.12.0"
|
||||
mxlink = ">=1.13.0"
|
||||
etke_openai_api_rust = "0.1.*"
|
||||
quick_cache = "0.6.*"
|
||||
regex = "1.12.*"
|
||||
serde = { version = "1.0.*", features = ["derive"], default-features = false }
|
||||
serde_json = "1.0.*"
|
||||
serde_yaml_ng = "0.10.*"
|
||||
tempfile = "3.25.*"
|
||||
tempfile = "3.27.*"
|
||||
tiktoken-rs = { version = "0.9.*", default-features = false }
|
||||
tokio = { version = "1.49.*", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
tokio = { version = "1.50.*", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
tracing = "0.1.*"
|
||||
tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }
|
||||
url = "2.5.*"
|
||||
|
||||
@@ -22,6 +22,8 @@ You can see the list of supported environment variables in the [🦀 src/entity/
|
||||
> [!WARNING]
|
||||
> The static configuration contains an `initial_global_config` key, which is used to populate the bot's global configuration (stored as [dynamic configuration](#dynamic-configuration)) the first time the bot starts. Modifying this subsequently will not have any effect. After initial global configuration creation, it's expected to be managed dynamically via chat commands.
|
||||
|
||||
For Matrix-account authentication setup, see [🔐 Authentication](./authentication.md).
|
||||
|
||||
|
||||
### Dynamic configuration
|
||||
|
||||
|
||||
23
docs/configuration/authentication.md
Normal file
23
docs/configuration/authentication.md
Normal file
@@ -0,0 +1,23 @@
|
||||
## 🔐 Authentication
|
||||
|
||||
baibot supports 2 authentication modes for the Matrix account (`user.*` keys in config).
|
||||
|
||||
Set **exactly one** mode. If both are set (or neither is set), startup validation fails.
|
||||
|
||||
### Password authentication
|
||||
|
||||
- Config key: `user.password`
|
||||
- Environment variable: `BAIBOT_USER_PASSWORD`
|
||||
|
||||
### Access token authentication
|
||||
|
||||
- Config keys: `user.access_token` + `user.device_id`
|
||||
- Environment variables: `BAIBOT_USER_ACCESS_TOKEN` + `BAIBOT_USER_DEVICE_ID`
|
||||
|
||||
Access-token authentication is useful for OIDC-enabled homeservers (e.g. those using [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service)).
|
||||
|
||||
Example token-generation command:
|
||||
|
||||
```sh
|
||||
mas-cli manage issue-compatibility-token <username> [device_id]
|
||||
```
|
||||
@@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
|
||||
|
||||
The bot supports the following use-purposes:
|
||||
|
||||
- [💬 text-generation](../features.md#-text-generation): communicating with you via text (though certain models may "see" images as well)
|
||||
- [💬 text-generation](../features.md#-text-generation): communicating with you via text (though certain models may also process images and files)
|
||||
- [🦻 speech-to-text](../features.md#-speech-to-text): turning your voice messages into text
|
||||
- [🗣️ text-to-speech](../features.md#️-text-to-speech): turning bot or users text messages into voice messages
|
||||
- [🖌️ image-generation](../features.md#image-generation): generating images based on instructions
|
||||
|
||||
@@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
|
||||
|
||||
The bot supports the following use-purposes:
|
||||
|
||||
- [💬 text-generation](#-text-generation): communicating with you via text (though certain models may "see" images as well)
|
||||
- [💬 text-generation](#-text-generation): communicating with you via text (though certain models may also process images and files)
|
||||
- [🦻 speech-to-text](#-speech-to-text): turning your voice messages into text
|
||||
- [🗣️ text-to-speech](#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
|
||||
- [🖌️ image-generation](#%EF%B8%8F-image-generation): generating images based on instructions
|
||||
@@ -26,7 +26,7 @@ Text Generation is the bot's ability to **respond to users' messages with text**
|
||||
|
||||

|
||||
|
||||
Some models also support vision, so you may be able to mix text and images in the same conversation.
|
||||
Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation. Note that certain providers may not support all file types or may have issues with specific files (e.g. scanned/image-based PDFs). If a file is rejected by the provider, the conversation thread may become unusable — start a new thread to work around this.
|
||||
|
||||
In multi-user (group) rooms, to avoid disturbing the normal conversation between people, the bot is auto-configured to only respond to messages starting with the command prefix (`!bai`) or direct mentions via the [💬 Text Generation / 🗟 Prefix Requirement Type](./configuration/text-generation.md#-prefix-requirement-type) setting.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
base_url: https://api.openai.com/v1
|
||||
api_key: YOUR_API_KEY_HERE
|
||||
text_generation:
|
||||
model_id: gpt-5.2
|
||||
model_id: gpt-5.4
|
||||
prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
|
||||
temperature: 1.0
|
||||
# Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
|
||||
|
||||
@@ -11,7 +11,7 @@ This is related to the [💬 Text Generation](./features.md#-text-generation) fe
|
||||
|
||||
If there's a text-generation handler agent configured, the bot **may** respond to messages sent in the room.
|
||||
|
||||
Some models also support vision, so you may be able to mix text and images in the same conversation.
|
||||
Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation.
|
||||
|
||||
See screenshots of:
|
||||
|
||||
|
||||
@@ -5,8 +5,17 @@ homeserver:
|
||||
|
||||
user:
|
||||
mxid_localpart: baibot
|
||||
|
||||
# Authentication: set EITHER password OR access_token + device_id.
|
||||
#
|
||||
# Password-based login (traditional homeservers):
|
||||
password: baibot
|
||||
|
||||
# Access token login (for Matrix Authentication Service/OIDC-enabled homeservers):
|
||||
# Generate a token via: mas-cli manage issue-compatibility-token <username> [device_id]
|
||||
# access_token: null
|
||||
# device_id: null
|
||||
|
||||
# The name the bot uses as a display name and when it refers to itself.
|
||||
# Leave empty to use the default (baibot).
|
||||
name: baibot
|
||||
@@ -82,7 +91,7 @@ agents:
|
||||
# base_url: https://api.openai.com/v1
|
||||
# api_key: ""
|
||||
# text_generation:
|
||||
# model_id: gpt-5.2
|
||||
# model_id: gpt-5.4
|
||||
# prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
|
||||
# temperature: 1.0
|
||||
# # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
continuwuity:
|
||||
image: forgejo.ellis.link/continuwuation/continuwuity:v0.5.5
|
||||
image: forgejo.ellis.link/continuwuation/continuwuity:v0.5.6
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
cap_drop:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
element-web:
|
||||
image: ghcr.io/element-hq/element-web:v1.12.10
|
||||
image: ghcr.io/element-hq/element-web:v1.12.12
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
ollama:
|
||||
image: docker.io/ollama/ollama:0.16.2
|
||||
image: docker.io/ollama/ollama:0.18.2
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/postgres:18.2-alpine
|
||||
image: docker.io/postgres:18.3-alpine
|
||||
user: ${UID}:${GID}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
- /etc/passwd:/etc/passwd:ro
|
||||
|
||||
synapse:
|
||||
image: ghcr.io/element-hq/synapse:v1.147.1
|
||||
image: ghcr.io/element-hq/synapse:v1.149.1
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
entrypoint: python
|
||||
|
||||
4
rust-toolchain.toml
Normal file
4
rust-toolchain.toml
Normal file
@@ -0,0 +1,4 @@
|
||||
[toolchain]
|
||||
channel = "1.93.0"
|
||||
components = ["rustfmt", "clippy"]
|
||||
profile = "default"
|
||||
@@ -28,6 +28,13 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
|
||||
},
|
||||
}]
|
||||
}
|
||||
LLMMessageContent::File(file_details) => {
|
||||
tracing::warn!(
|
||||
"The Anthropic provider's library does not support file/document content. This file message ({}) will be skipped.",
|
||||
file_details.filename(),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let message = Message { role, content };
|
||||
|
||||
@@ -84,7 +84,7 @@ impl Default for TextGenerationConfig {
|
||||
}
|
||||
|
||||
fn default_text_model_id() -> String {
|
||||
"gpt-5.2".to_owned()
|
||||
"gpt-5.4".to_owned()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use async_openai::types::responses::{
|
||||
EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputImageContent, InputItem,
|
||||
InputParam, MessageType, Role,
|
||||
EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputFileArgs,
|
||||
InputImageContent, InputItem, InputParam, MessageType, Role,
|
||||
};
|
||||
|
||||
use crate::conversation::llm::{
|
||||
@@ -35,12 +35,28 @@ pub fn convert_llm_messages_to_openai_response_input(
|
||||
file_id: None,
|
||||
})])
|
||||
}
|
||||
LLMMessageContent::File(file_details) => {
|
||||
let file_data = format!(
|
||||
"data:{};base64,{}",
|
||||
file_details.mime,
|
||||
base64_encode(&file_details.data)
|
||||
);
|
||||
|
||||
let file_content = InputFileArgs::default()
|
||||
.file_data(file_data)
|
||||
.filename(file_details.filename())
|
||||
.build()
|
||||
.expect("Failed to build InputFileContent");
|
||||
|
||||
EasyInputContent::ContentList(vec![InputContent::InputFile(file_content)])
|
||||
}
|
||||
};
|
||||
|
||||
items.push(InputItem::EasyMessage(EasyInputMessage {
|
||||
r#type: MessageType::Message,
|
||||
role,
|
||||
content,
|
||||
phase: None,
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
@@ -40,6 +40,12 @@ fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option<Mess
|
||||
);
|
||||
None
|
||||
}
|
||||
LLMMessageContent::File(_file_details) => {
|
||||
tracing::warn!(
|
||||
"The OpenAI-compat provider's library does not support file content. This file message will be skipped."
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::agent::Manager as AgentManager;
|
||||
use crate::entity::catch_up_marker::{
|
||||
CatchUpMarker, CatchUpMarkerManager, DelayedCatchUpMarkerManager,
|
||||
};
|
||||
use crate::entity::cfg::{Avatar, Config};
|
||||
use crate::entity::cfg::{Avatar, Config, ConfigUserAuth};
|
||||
use crate::entity::globalconfig::{GlobalConfig, GlobalConfigurationManager};
|
||||
use crate::entity::roomconfig::{RoomConfig, RoomConfigurationManager};
|
||||
|
||||
@@ -395,10 +395,22 @@ async fn create_matrix_link(config: &Config) -> anyhow::Result<MatrixLink> {
|
||||
let session_encryption_key = config.persistence.session_encryption_key()?;
|
||||
let db_dir_path: std::path::PathBuf = config.persistence.db_dir_path()?;
|
||||
|
||||
let login_creds = LoginCredentials::UserPassword(
|
||||
config.user.mxid_localpart.to_owned(),
|
||||
config.user.password.to_owned(),
|
||||
);
|
||||
let user_auth = config.user.auth_config(&config.homeserver.server_name)?;
|
||||
|
||||
let login_creds = match user_auth {
|
||||
ConfigUserAuth::UserPassword { username, password } => {
|
||||
LoginCredentials::UserPassword(username, password)
|
||||
}
|
||||
ConfigUserAuth::AccessToken {
|
||||
user_id,
|
||||
device_id,
|
||||
access_token,
|
||||
} => LoginCredentials::AccessToken {
|
||||
user_id,
|
||||
device_id,
|
||||
access_token,
|
||||
},
|
||||
};
|
||||
|
||||
let login_encryption = LoginEncryption::new(
|
||||
config.user.encryption.recovery_passphrase.clone(),
|
||||
|
||||
@@ -29,7 +29,15 @@ pub fn load() -> anyhow::Result<Config> {
|
||||
cfg_env::BAIBOT_HOMESERVER_SERVER_NAME => config.homeserver.server_name = value,
|
||||
cfg_env::BAIBOT_HOMESERVER_URL => config.homeserver.url = value,
|
||||
cfg_env::BAIBOT_USER_MXID_LOCALPART => config.user.mxid_localpart = value,
|
||||
cfg_env::BAIBOT_USER_PASSWORD => config.user.password = value,
|
||||
cfg_env::BAIBOT_USER_PASSWORD => {
|
||||
config.user.password = optional_non_empty(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_ACCESS_TOKEN => {
|
||||
config.user.access_token = optional_non_empty(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_DEVICE_ID => {
|
||||
config.user.device_id = optional_non_empty(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE => {
|
||||
config.user.encryption.recovery_passphrase = Some(value);
|
||||
}
|
||||
@@ -120,3 +128,7 @@ pub fn load() -> anyhow::Result<Config> {
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn optional_non_empty(value: String) -> Option<String> {
|
||||
if value.is_empty() { None } else { Some(value) }
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ pub enum ChatCompletionControllerType {
|
||||
|
||||
Image,
|
||||
|
||||
File,
|
||||
|
||||
ThreadMention,
|
||||
ReplyMention,
|
||||
}
|
||||
@@ -419,7 +421,8 @@ async fn handle_stage_text_generation(
|
||||
| ChatCompletionControllerType::TextMention
|
||||
| ChatCompletionControllerType::TextDirect
|
||||
| ChatCompletionControllerType::Audio
|
||||
| ChatCompletionControllerType::Image => {
|
||||
| ChatCompletionControllerType::Image
|
||||
| ChatCompletionControllerType::File => {
|
||||
Some(message_context.combined_admin_and_user_regexes())
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,18 @@ pub fn determine_controller(
|
||||
)
|
||||
}
|
||||
}
|
||||
MessagePayload::File(_file_message_content) => {
|
||||
let prefix_requirement_type = message_context
|
||||
.room_config_context()
|
||||
.text_generation_prefix_requirement_type();
|
||||
|
||||
match prefix_requirement_type {
|
||||
TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore,
|
||||
TextGenerationPrefixRequirementType::No => {
|
||||
ControllerType::ChatCompletion(ChatCompletionControllerType::File)
|
||||
}
|
||||
}
|
||||
}
|
||||
MessagePayload::Audio(_) => {
|
||||
ControllerType::ChatCompletion(ChatCompletionControllerType::Audio)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::{
|
||||
FileMessageEventContent, ImageMessageEventContent,
|
||||
};
|
||||
use mxlink::mime::Mime;
|
||||
|
||||
use crate::agent::provider::ImageSource;
|
||||
@@ -48,10 +50,35 @@ impl From<ImageDetails> for ImageSource {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileDetails {
|
||||
pub event_content: FileMessageEventContent,
|
||||
pub mime: Mime,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl FileDetails {
|
||||
pub fn new(event_content: FileMessageEventContent, mime: Mime, data: Vec<u8>) -> Self {
|
||||
Self {
|
||||
event_content,
|
||||
mime,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn filename(&self) -> String {
|
||||
self.event_content
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or(self.event_content.body.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Image(ImageDetails),
|
||||
File(FileDetails),
|
||||
}
|
||||
|
||||
impl PartialEq for MessageContent {
|
||||
@@ -62,6 +89,7 @@ impl PartialEq for MessageContent {
|
||||
// We can probably do better than this by inspecting `.event_conten1t.source`, but for now this is good enough.
|
||||
a.filename() == b.filename()
|
||||
}
|
||||
(MessageContent::File(a), MessageContent::File(b)) => a.filename() == b.filename(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +74,7 @@ fn calculate_token_size_for_message(bpe: &CoreBPE, model: &str, message: &Messag
|
||||
let text_length = match &message.content {
|
||||
MessageContent::Text(text) => bpe.encode_with_special_tokens(text).len() as i32,
|
||||
MessageContent::Image(..) => 0,
|
||||
MessageContent::File(..) => 0,
|
||||
};
|
||||
|
||||
(text_length + role_length + tokens_per_message + tokens_per_name) as u32
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use mxlink::matrix_sdk::ruma::OwnedUserId;
|
||||
|
||||
use super::entity::{Author, ImageDetails, Message, MessageContent};
|
||||
use super::entity::{Author, FileDetails, ImageDetails, Message, MessageContent};
|
||||
use crate::conversation::matrix::{MatrixMessage, MatrixMessageContent};
|
||||
use crate::utils::text_to_speech as text_to_speech_utils;
|
||||
|
||||
@@ -32,6 +32,15 @@ fn convert_bot_message(matrix_message: &MatrixMessage) -> Option<Message> {
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message {
|
||||
author: Author::Assistant,
|
||||
content: MessageContent::File(FileDetails::new(
|
||||
file_content.clone(),
|
||||
mime_type.clone(),
|
||||
media_bytes.clone(),
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,5 +99,14 @@ fn convert_user_message(matrix_message: &MatrixMessage) -> Option<Message> {
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message {
|
||||
author: Author::User,
|
||||
content: MessageContent::File(FileDetails::new(
|
||||
file_content.clone(),
|
||||
mime_type.clone(),
|
||||
media_bytes.clone(),
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ use chrono::{DateTime, Utc};
|
||||
use regex::Regex;
|
||||
|
||||
use mxlink::matrix_sdk::ruma::OwnedUserId;
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::{
|
||||
FileMessageEventContent, ImageMessageEventContent,
|
||||
};
|
||||
use mxlink::mime::Mime;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -18,6 +20,7 @@ pub enum MatrixMessageContent {
|
||||
Text(String),
|
||||
Notice(String),
|
||||
Image(ImageMessageEventContent, Mime, Vec<u8>),
|
||||
File(FileMessageEventContent, Mime, Vec<u8>),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -234,6 +234,7 @@ pub async fn convert_matrix_native_event_to_matrix_message(
|
||||
MessageType::Text(text_content) => (text_content.body.clone(), false),
|
||||
MessageType::Notice(notice_content) => (notice_content.body.clone(), true),
|
||||
MessageType::Image(image_content) => (image_content.body.clone(), false),
|
||||
MessageType::File(file_content) => (file_content.body.clone(), false),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
@@ -291,6 +292,67 @@ pub async fn convert_matrix_native_event_to_matrix_message(
|
||||
}));
|
||||
}
|
||||
|
||||
if let MessageType::File(file_content) = &room_message.msgtype {
|
||||
let media_request = mxlink::matrix_sdk::media::MediaRequestParameters {
|
||||
source: file_content.source.to_owned(),
|
||||
format: mxlink::matrix_sdk::media::MediaFormat::File,
|
||||
};
|
||||
|
||||
let file_name = file_content
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or(file_content.body.clone());
|
||||
|
||||
let mime_type = file_content
|
||||
.info
|
||||
.as_ref()
|
||||
.and_then(|info| info.mimetype.clone())
|
||||
.and_then(|mimetype| mimetype.parse::<mxlink::mime::Mime>().ok())
|
||||
.unwrap_or_else(|| get_mime_type_from_file_name(&file_name));
|
||||
|
||||
tracing::debug!("Determined mime type {} for file {}", mime_type, file_name);
|
||||
|
||||
if mime_type == mxlink::mime::APPLICATION_OCTET_STREAM {
|
||||
tracing::debug!(
|
||||
"Skipping file {} with unsupported MIME type {}. It will be represented as a text message.",
|
||||
file_name,
|
||||
mime_type,
|
||||
);
|
||||
|
||||
return Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
content: MatrixMessageContent::Text(format!(
|
||||
"[A file ({}) was attached but skipped because its content type ({}) is not supported. Let the user know.]",
|
||||
file_name, mime_type,
|
||||
)),
|
||||
mentioned_users,
|
||||
timestamp,
|
||||
}));
|
||||
}
|
||||
|
||||
let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type);
|
||||
|
||||
let media_bytes = matrix_link
|
||||
.client()
|
||||
.media()
|
||||
.get_media_content(&media_request, true)
|
||||
.instrument(span)
|
||||
.await?;
|
||||
|
||||
tracing::debug!(
|
||||
"Downloaded {} bytes for file {}",
|
||||
media_bytes.len(),
|
||||
file_name
|
||||
);
|
||||
|
||||
return Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
content: MatrixMessageContent::File(file_content.clone(), mime_type, media_bytes),
|
||||
mentioned_users,
|
||||
timestamp,
|
||||
}));
|
||||
}
|
||||
|
||||
Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
content: if is_notice {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use mxlink::helpers::encryption::EncryptionKey;
|
||||
use mxlink::matrix_sdk::ruma::{OwnedDeviceId, OwnedUserId};
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
use crate::{
|
||||
@@ -38,7 +39,7 @@ pub struct Config {
|
||||
impl Config {
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
self.homeserver.validate()?;
|
||||
self.user.validate()?;
|
||||
self.user.validate(&self.homeserver.server_name)?;
|
||||
self.persistence.validate()?;
|
||||
self.room.validate()?;
|
||||
self.access.validate()?;
|
||||
@@ -57,6 +58,19 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ConfigUserAuth {
|
||||
UserPassword {
|
||||
username: String,
|
||||
password: String,
|
||||
},
|
||||
AccessToken {
|
||||
user_id: OwnedUserId,
|
||||
device_id: OwnedDeviceId,
|
||||
access_token: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct ConfigHomeserver {
|
||||
pub server_name: String,
|
||||
@@ -127,7 +141,15 @@ impl Avatar {
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct ConfigUser {
|
||||
pub mxid_localpart: String,
|
||||
pub password: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub password: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub access_token: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub device_id: Option<String>,
|
||||
|
||||
#[serde(default = "super::defaults::name")]
|
||||
pub name: String,
|
||||
@@ -140,7 +162,7 @@ pub struct ConfigUser {
|
||||
}
|
||||
|
||||
impl ConfigUser {
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
pub fn validate(&self, homeserver_server_name: &str) -> anyhow::Result<()> {
|
||||
if self.mxid_localpart.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"The user.mxid_localpart ({}) configuration must be set",
|
||||
@@ -148,12 +170,7 @@ impl ConfigUser {
|
||||
));
|
||||
}
|
||||
|
||||
if self.password.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"The user.password ({}) configuration must be set",
|
||||
super::env::BAIBOT_USER_PASSWORD
|
||||
));
|
||||
}
|
||||
self.auth_config(homeserver_server_name)?;
|
||||
|
||||
if self.name.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
@@ -166,6 +183,57 @@ impl ConfigUser {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn auth_config(&self, homeserver_server_name: &str) -> anyhow::Result<ConfigUserAuth> {
|
||||
let password = self.password.as_deref().filter(|value| !value.is_empty());
|
||||
let access_token = self
|
||||
.access_token
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty());
|
||||
|
||||
match (password, access_token) {
|
||||
(Some(_), Some(_)) => Err(anyhow::anyhow!(
|
||||
"Set exactly one authentication method: either user.password ({}) OR user.access_token ({}) + user.device_id ({})",
|
||||
super::env::BAIBOT_USER_PASSWORD,
|
||||
super::env::BAIBOT_USER_ACCESS_TOKEN,
|
||||
super::env::BAIBOT_USER_DEVICE_ID
|
||||
)),
|
||||
(None, None) => Err(anyhow::anyhow!(
|
||||
"Set one authentication method: either user.password ({}) OR user.access_token ({}) + user.device_id ({})",
|
||||
super::env::BAIBOT_USER_PASSWORD,
|
||||
super::env::BAIBOT_USER_ACCESS_TOKEN,
|
||||
super::env::BAIBOT_USER_DEVICE_ID
|
||||
)),
|
||||
(Some(password), None) => Ok(ConfigUserAuth::UserPassword {
|
||||
username: self.mxid_localpart.to_owned(),
|
||||
password: password.to_owned(),
|
||||
}),
|
||||
(None, Some(access_token)) => {
|
||||
let device_id = self
|
||||
.device_id
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"user.device_id ({}) must be set when using access token authentication",
|
||||
super::env::BAIBOT_USER_DEVICE_ID
|
||||
)
|
||||
})?;
|
||||
|
||||
let user_id = OwnedUserId::try_from(format!(
|
||||
"@{}:{}",
|
||||
self.mxid_localpart, homeserver_server_name
|
||||
))
|
||||
.map_err(|e| anyhow::anyhow!("Invalid user ID: {e}"))?;
|
||||
|
||||
Ok(ConfigUserAuth::AccessToken {
|
||||
user_id,
|
||||
device_id: OwnedDeviceId::from(device_id),
|
||||
access_token: access_token.to_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
@@ -468,3 +536,7 @@ impl TryInto<GlobalConfig> for ConfigInitialGlobalConfig {
|
||||
Ok(entity)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "config_tests.rs"]
|
||||
mod config_tests;
|
||||
|
||||
117
src/entity/cfg/config_tests.rs
Normal file
117
src/entity/cfg/config_tests.rs
Normal file
@@ -0,0 +1,117 @@
|
||||
use super::{Avatar, ConfigUser, ConfigUserAuth, ConfigUserEncryption};
|
||||
use crate::entity::cfg::env;
|
||||
|
||||
fn base_user() -> ConfigUser {
|
||||
ConfigUser {
|
||||
mxid_localpart: "baibot".to_owned(),
|
||||
password: None,
|
||||
access_token: None,
|
||||
device_id: None,
|
||||
name: "baibot".to_owned(),
|
||||
encryption: ConfigUserEncryption {
|
||||
recovery_passphrase: None,
|
||||
recovery_reset_allowed: false,
|
||||
},
|
||||
avatar: Avatar::Default,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_config_uses_password_mode() {
|
||||
let mut user = base_user();
|
||||
user.password = Some("secret".to_owned());
|
||||
|
||||
let auth = user
|
||||
.auth_config("example.com")
|
||||
.expect("password auth should be valid");
|
||||
|
||||
match auth {
|
||||
ConfigUserAuth::UserPassword { username, password } => {
|
||||
assert_eq!(username, "baibot");
|
||||
assert_eq!(password, "secret");
|
||||
}
|
||||
ConfigUserAuth::AccessToken { .. } => {
|
||||
panic!("expected password auth mode");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_config_uses_access_token_mode() {
|
||||
let mut user = base_user();
|
||||
user.access_token = Some("token123".to_owned());
|
||||
user.device_id = Some("DEVICE1".to_owned());
|
||||
|
||||
let auth = user
|
||||
.auth_config("example.com")
|
||||
.expect("access token auth should be valid");
|
||||
|
||||
match auth {
|
||||
ConfigUserAuth::AccessToken {
|
||||
user_id,
|
||||
device_id,
|
||||
access_token,
|
||||
} => {
|
||||
assert_eq!(user_id.as_str(), "@baibot:example.com");
|
||||
assert_eq!(device_id.as_str(), "DEVICE1");
|
||||
assert_eq!(access_token, "token123");
|
||||
}
|
||||
ConfigUserAuth::UserPassword { .. } => {
|
||||
panic!("expected access token auth mode");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_config_rejects_both_auth_methods() {
|
||||
let mut user = base_user();
|
||||
user.password = Some("secret".to_owned());
|
||||
user.access_token = Some("token123".to_owned());
|
||||
user.device_id = Some("DEVICE1".to_owned());
|
||||
|
||||
let err = user
|
||||
.auth_config("example.com")
|
||||
.expect_err("both auth methods should be rejected");
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("exactly one authentication method")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_config_rejects_missing_auth() {
|
||||
let user = base_user();
|
||||
|
||||
let err = user
|
||||
.auth_config("example.com")
|
||||
.expect_err("missing auth should be rejected");
|
||||
|
||||
assert!(err.to_string().contains("Set one authentication method"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_config_rejects_access_token_without_device_id() {
|
||||
let mut user = base_user();
|
||||
user.access_token = Some("token123".to_owned());
|
||||
|
||||
let err = user
|
||||
.auth_config("example.com")
|
||||
.expect_err("access token mode without device_id should be rejected");
|
||||
|
||||
assert!(err.to_string().contains(env::BAIBOT_USER_DEVICE_ID));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_config_treats_empty_strings_as_unset() {
|
||||
let mut user = base_user();
|
||||
user.password = Some(String::new());
|
||||
user.access_token = Some(String::new());
|
||||
user.device_id = Some(String::new());
|
||||
|
||||
let err = user
|
||||
.auth_config("example.com")
|
||||
.expect_err("empty auth values should be treated as unset");
|
||||
|
||||
assert!(err.to_string().contains("Set one authentication method"));
|
||||
}
|
||||
@@ -5,6 +5,8 @@ pub const BAIBOT_HOMESERVER_URL: &str = "BAIBOT_HOMESERVER_URL";
|
||||
|
||||
pub const BAIBOT_USER_MXID_LOCALPART: &str = "BAIBOT_USER_MXID_LOCALPART";
|
||||
pub const BAIBOT_USER_PASSWORD: &str = "BAIBOT_USER_PASSWORD";
|
||||
pub const BAIBOT_USER_ACCESS_TOKEN: &str = "BAIBOT_USER_ACCESS_TOKEN";
|
||||
pub const BAIBOT_USER_DEVICE_ID: &str = "BAIBOT_USER_DEVICE_ID";
|
||||
pub const BAIBOT_USER_NAME: &str = "BAIBOT_USER_NAME";
|
||||
pub const BAIBOT_USER_AVATAR: &str = "BAIBOT_USER_AVATAR";
|
||||
pub const BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE: &str =
|
||||
|
||||
@@ -2,4 +2,4 @@ mod config;
|
||||
pub mod defaults;
|
||||
pub mod env;
|
||||
|
||||
pub use config::{Avatar, Config};
|
||||
pub use config::{Avatar, Config, ConfigUserAuth};
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::{
|
||||
AudioMessageEventContent, ImageMessageEventContent, MessageType, TextMessageEventContent,
|
||||
AudioMessageEventContent, FileMessageEventContent, ImageMessageEventContent, MessageType,
|
||||
TextMessageEventContent,
|
||||
};
|
||||
use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId};
|
||||
|
||||
@@ -30,6 +31,7 @@ pub enum MessagePayload {
|
||||
Text(TextMessageEventContent),
|
||||
Audio(AudioMessageEventContent),
|
||||
Image(ImageMessageEventContent),
|
||||
File(FileMessageEventContent),
|
||||
|
||||
Reaction {
|
||||
key: String,
|
||||
@@ -57,6 +59,7 @@ impl TryInto<MessagePayload> for MessageType {
|
||||
MessagePayload::Audio(audio_content)
|
||||
}
|
||||
MessageType::Image(image_content) => MessagePayload::Image(image_content),
|
||||
MessageType::File(file_content) => MessagePayload::File(file_content),
|
||||
other => {
|
||||
return Err(format!("Unsupported message type: {:?}", other));
|
||||
}
|
||||
|
||||
@@ -17,18 +17,7 @@ pub fn get_file_extension(mime_type: &mime::Mime) -> String {
|
||||
}
|
||||
|
||||
pub fn get_mime_type_from_file_name(file_name: &str) -> mime::Mime {
|
||||
let extension = file_name.rsplit('.').next().unwrap_or("");
|
||||
|
||||
match extension.to_lowercase().as_str() {
|
||||
"jpg" | "jpeg" => mime::IMAGE_JPEG,
|
||||
"png" => mime::IMAGE_PNG,
|
||||
"gif" => mime::IMAGE_GIF,
|
||||
"webp" => "image/webp".parse().unwrap(),
|
||||
"svg" => mime::IMAGE_SVG,
|
||||
"tiff" | "tif" => "image/tiff".parse().unwrap(),
|
||||
"bmp" => "image/bmp".parse().unwrap(),
|
||||
"heic" | "heif" => "image/heic".parse().unwrap(),
|
||||
"avif" => "image/avif".parse().unwrap(),
|
||||
_ => mime::APPLICATION_OCTET_STREAM,
|
||||
}
|
||||
mime_guess::from_path(file_name)
|
||||
.first()
|
||||
.unwrap_or(mime::APPLICATION_OCTET_STREAM)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user