Compare commits
154 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e57778d2bd | ||
|
|
2d659964a7 | ||
|
|
661e7263fb | ||
|
|
1705c16762 | ||
|
|
2455117e41 | ||
|
|
3e9c110afc | ||
|
|
d9b5524c97 | ||
|
|
9b169a7d28 | ||
|
|
cb29419d75 | ||
|
|
51ca8c9948 | ||
|
|
12b938d2d1 | ||
|
|
f3d1b32ad7 | ||
|
|
3d3bd3c9f9 | ||
|
|
a25845e89e | ||
|
|
479f54b93d | ||
|
|
90ab6807ac | ||
|
|
5022f79bf5 | ||
|
|
2ba3b5a437 | ||
|
|
2819011b9d | ||
|
|
ede9065f77 | ||
|
|
5e61f5b3a3 | ||
|
|
8633e82f62 | ||
|
|
6aa3d70d57 | ||
|
|
75e29ee3fb | ||
|
|
eab7978b9f | ||
|
|
765e7c17b2 | ||
|
|
748d2b7fd4 | ||
|
|
527759dd02 | ||
|
|
3290255bad | ||
|
|
9b987395b3 | ||
|
|
4852d1fe92 | ||
|
|
8bd313f0d4 | ||
|
|
711e1099d6 | ||
|
|
91c8dd8f7d | ||
|
|
afc5572d6a | ||
|
|
73e13dcf2f | ||
|
|
2bebd109b1 | ||
|
|
7f7c58be1f | ||
|
|
47e5a464a0 | ||
|
|
85f751e514 | ||
|
|
95acad3558 | ||
|
|
304056c59a | ||
|
|
bedc0335f1 | ||
|
|
a8be8c3c1e | ||
|
|
826fa728a9 | ||
|
|
5aef8e8b2f | ||
|
|
f70f20181e | ||
|
|
fcdd4f39ee | ||
|
|
891adfec49 | ||
|
|
35ab79844b | ||
|
|
bbc122fbb1 | ||
|
|
2413c8b88b | ||
|
|
10c3c64469 | ||
|
|
b3307b404b | ||
|
|
7a0d1e830d | ||
|
|
b3bd241823 | ||
|
|
de3d8b054f | ||
|
|
0a55e276a2 | ||
|
|
1f2c65d2e6 | ||
|
|
3b5e4745f2 | ||
|
|
407bb022d9 | ||
|
|
faf92cac09 | ||
|
|
a82e9a1d1f | ||
|
|
8f87f05a08 | ||
|
|
61d18b2e13 | ||
|
|
d831c08306 | ||
|
|
c70387b0c3 | ||
|
|
38516f2e17 | ||
|
|
5481b5a763 | ||
|
|
26bc437678 | ||
|
|
ec93f1ee2a | ||
|
|
b920b6e556 | ||
|
|
7136d34843 | ||
|
|
e0b4a40dd8 | ||
|
|
691aeeb1c7 | ||
|
|
257ffae9e7 | ||
|
|
f7bf3d7b60 | ||
|
|
3a88b0d656 | ||
|
|
08c689a889 | ||
|
|
ae8e878817 | ||
|
|
edbd72ece6 | ||
|
|
22906aa2d3 | ||
|
|
99bde53ef6 | ||
|
|
b3fd8e548f | ||
|
|
062fbbb8ef | ||
|
|
2801c78ad9 | ||
|
|
f4c698ad33 | ||
|
|
bd39001417 | ||
|
|
2692d0322e | ||
|
|
8eb70f0f2c | ||
|
|
5c0a7be7a2 | ||
|
|
0a8f9fc3e5 | ||
|
|
1ac3b2e060 | ||
|
|
a3ef9fd1bf | ||
|
|
df507eb201 | ||
|
|
4dcd9eff40 | ||
|
|
ea760ce755 | ||
|
|
1528df6a55 | ||
|
|
b0fa024297 | ||
|
|
3ec203128a | ||
|
|
da97361e1b | ||
|
|
b430fe0189 | ||
|
|
f03126a9e1 | ||
|
|
7d46b926c1 | ||
|
|
6f3c048195 | ||
|
|
b47cf598b5 | ||
|
|
265ad7e1cb | ||
|
|
a159f67e45 | ||
|
|
624b9de35b | ||
|
|
941bf7ca42 | ||
|
|
ef0f1671da | ||
|
|
b43f61f5ff | ||
|
|
1967d2b34c | ||
|
|
bb3734ad24 | ||
|
|
eb6db34177 | ||
|
|
6e845caa2e | ||
|
|
1004966785 | ||
|
|
7ae1864c2e | ||
|
|
68a2fb161f | ||
|
|
3a3eb58d7b | ||
|
|
74d988e650 | ||
|
|
ed8bedcd7e | ||
|
|
2842632969 | ||
|
|
10a5bd2abb | ||
|
|
5308b75f52 | ||
|
|
dad61e1270 | ||
|
|
91986a129c | ||
|
|
264f683d6a | ||
|
|
62f0f4fa0d | ||
|
|
69627abd74 | ||
|
|
d2660be33c | ||
|
|
ce81fe69bd | ||
|
|
1162636b88 | ||
|
|
8c90e13a79 | ||
|
|
274b614d25 | ||
|
|
7bd46821dc | ||
|
|
a84135ff32 | ||
|
|
231528a0d8 | ||
|
|
d8e47b0578 | ||
|
|
96c1542f4a | ||
|
|
2f9c3dfce0 | ||
|
|
de958208b2 | ||
|
|
ac4f2080ce | ||
|
|
3ffa50b7b9 | ||
|
|
8f86289373 | ||
|
|
e0dcc39a72 | ||
|
|
c94376109c | ||
|
|
256ed05662 | ||
|
|
8222681e27 | ||
|
|
f304b93c68 | ||
|
|
889d8a1d04 | ||
|
|
6082bfaf56 | ||
|
|
1d629e0859 | ||
|
|
49471c1df0 |
16
.github/workflows/workflow.yml
vendored
16
.github/workflows/workflow.yml
vendored
@@ -16,8 +16,8 @@ jobs:
|
||||
name: Unit testing and linting
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dtolnay/rust-toolchain@1.93.0
|
||||
- name: Install SQLite3
|
||||
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
|
||||
- run: cargo test --all-features
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
steps:
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ github.repository }}
|
||||
@@ -56,16 +56,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
- name: Log in to the GitHub Container registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
ghcr.io/${{ github.repository }}
|
||||
|
||||
- name: Build and push Docker images
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Log in to the GitHub Container registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
36
.pre-commit-config.yaml
Normal file
36
.pre-commit-config.yaml
Normal file
@@ -0,0 +1,36 @@
|
||||
repos:
|
||||
# Fast built-in hooks (Rust-native, no dependencies)
|
||||
- repo: builtin
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-merge-conflict
|
||||
- id: check-added-large-files
|
||||
args: ['--maxkb=1024']
|
||||
|
||||
# Local hooks that run project-specific tools
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: cargo-fmt-check
|
||||
name: Cargo Format Check
|
||||
entry: cargo fmt --all -- --check
|
||||
language: system
|
||||
files: '\.rs$'
|
||||
pass_filenames: false
|
||||
|
||||
- id: cargo-clippy
|
||||
name: Cargo Clippy
|
||||
entry: cargo clippy -- -D warnings
|
||||
language: system
|
||||
files: '\.rs$'
|
||||
pass_filenames: false
|
||||
priority: 100
|
||||
|
||||
- id: test-unit
|
||||
name: Unit Tests
|
||||
entry: just test
|
||||
language: system
|
||||
files: '\.rs$'
|
||||
pass_filenames: false
|
||||
priority: 100
|
||||
169
CHANGELOG.md
169
CHANGELOG.md
@@ -1,3 +1,172 @@
|
||||
# (2026-03-25) Version 1.17.0
|
||||
|
||||
- (**Feature**) Add `text-generation sender-context-mode` for attaching sender metadata to conversation messages. See the [💬 Text Generation](./docs/configuration/text-generation.md#-sender-context-mode) documentation for details. Thanks to [kschwank](https://github.com/kschwank) for the contribution in [#104](https://github.com/etkecc/baibot/pull/104)!
|
||||
|
||||
|
||||
# (2026-03-24) Version 1.16.1
|
||||
|
||||
- (**Bugfix**) Fix compatibility with [async-openai](https://crates.io/crates/async-openai) 0.34.0 by populating the new `phase` field required for OpenAI Responses API message inputs. baibot does not currently distinguish between assistant `commentary` and `final_answer` turns, so using `None` preserves the previous behavior while remaining compatible with the updated crate.
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
|
||||
# (2026-03-20) Version 1.16.0
|
||||
|
||||
- (**Feature**) Add support for file attachments (`m.file` Matrix messages) in conversations. Files like PDFs, text documents, spreadsheets, code files, etc. are now downloaded and forwarded to the LLM alongside the conversation context, similar to how images (`m.image`) are already handled. See the [💬 Text Generation](./docs/features.md#-text-generation) documentation for details and known limitations.
|
||||
|
||||
- (**Improvement**) Use the [mime_guess](https://crates.io/crates/mime_guess) crate for MIME type detection from file extensions, replacing a hand-maintained mapping. This covers hundreds of file extensions out of the box.
|
||||
|
||||
|
||||
|
||||
# (2026-03-07) Version 1.15.0
|
||||
|
||||
- (**Feature**) Add support for authentication via access tokens (for [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service)/OIDC-enabled homeservers) as an alternative to password authentication. See [🔐 Authentication](./docs/configuration/authentication.md) for setup details. Thanks to [Taylor Southwick](https://github.com/twsouthwick) for the contribution in [#83](https://github.com/etkecc/baibot/pull/83)!
|
||||
|
||||
- (**Internal Improvement**) Pin the Rust toolchain to `1.93.0` in both CI and local development to avoid `matrix-sdk` build failures on newer stable toolchains.
|
||||
|
||||
- (**Internal Improvement**) Documentation updates.
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
|
||||
# (2026-02-18) Version 1.14.3
|
||||
|
||||
- (**Internal Improvement**) Add [Renovate](https://docs.renovatebot.com/) configuration for automated dependency updates
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
|
||||
# (2026-02-18) Version 1.14.2
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
- (**Internal Improvement**) Reorganize the development environment to support [Continuwuity](https://continuwuity.org/) as a homeserver choice (in addition to [Synapse](https://github.com/element-hq/synapse)). Continuwuity is now the default for its lighter footprint (no external database required). See [development docs](./docs/development.md) for details.
|
||||
|
||||
|
||||
# (2026-02-10) Version 1.14.1
|
||||
|
||||
- (**Security**) Dependency updates to fix security vulnerabilities ([time](https://crates.io/crates/time) stack exhaustion DoS, [bytes](https://crates.io/crates/bytes) integer overflow), via [mxlink](https://crates.io/crates/mxlink) 1.12.0
|
||||
|
||||
- (**Internal Improvement**) Switch from deprecated [serde_yaml](https://crates.io/crates/serde_yaml) to its maintained fork [serde_yaml_ng](https://crates.io/crates/serde_yaml_ng)
|
||||
|
||||
- (**Internal Improvement**) Add [prek](https://github.com/nicholasgasior/prek) pre-commit hooks via [mise](https://mise.jdx.dev/) for automated code quality checks (formatting, clippy, tests)
|
||||
|
||||
- (**Internal Improvement**) Fix clippy warnings and formatting issues
|
||||
|
||||
|
||||
# (2026-02-04) Version 1.14.0
|
||||
|
||||
- (**Feature**) The `openai` provider now uses OpenAI's [Responses API](https://platform.openai.com/docs/api-reference/responses) (instead of the older Chat Completions API), adding support for [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (`web_search` and `code_interpreter`). These tools are **disabled by default** and can be enabled via the `text_generation.tools` configuration (see the [sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15)). To enable tools on an existing agent, you need to [update the agent](./docs/agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need. Thanks to [Layla Manley](https://github.com/yeslayla) for the contribution in [#62](https://github.com/etkecc/baibot/pull/62)!
|
||||
|
||||
- (**Bugfix**) Fix sticker generation for newer GPT image models (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`) which don't support the previously hardcoded `256x256` size (minimum is `1024x1024`)
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
|
||||
# (2026-01-23) Version 1.13.0
|
||||
|
||||
- (**Improvement**) Extend auto-switching to support cheaper models (`gpt-image-1-mini`) for `gpt-image-1` and `gpt-image-1.5` when generating stickers ([e0b4a40](https://github.com/etkecc/baibot/commit/e0b4a40))
|
||||
|
||||
- (**Internal Improvement**) Upgrade Rust compiler (1.92.0 -> 1.93.0) ([691aeeb](https://github.com/etkecc/baibot/commit/691aeeb))
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
|
||||
# (2025-12-21) Version 1.12.0
|
||||
|
||||
- (**Improvement**) Upgrade [async-openai](https://crates.io/crates/async-openai) (0.31.1 -> 0.32.2) and add support for OpenAI's `gpt-image-1.5` model ([08c689a](https://github.com/etkecc/baibot/commit/08c689a), [f7bf3d7](https://github.com/etkecc/baibot/commit/f7bf3d7))
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
|
||||
# (2025-12-15) Version 1.11.0
|
||||
|
||||
- (**Feature**) Add support for custom avatars via file path and for keeping the already-set avatar (for those who wish to manage it by themselves via other means). See the [sample config](./etc/app/config.yml.dist) for details. ([062fbbb](https://github.com/etkecc/baibot/commit/062fbbb8ef9ad600db483a431c5c782402191023))
|
||||
|
||||
- (**Internal Improvement**) Dependency updates ([99bde53](https://github.com/etkecc/baibot/commit/99bde53ef648a5a9086a96778fde4a9dbc1ede58))
|
||||
|
||||
- (**Internal Improvement**) Documentation updates ([b3fd8e5](https://github.com/etkecc/baibot/commit/b3fd8e548f83fe46398ced4760d7e2bb7588c24d))
|
||||
|
||||
- (**Internal Improvement**) Upgrade Rust compiler (1.91.1 -> 1.92.0) ([22906aa](https://github.com/etkecc/baibot/commit/22906aa2d3cae51815fad2560a545eaa69c247b6))
|
||||
|
||||
|
||||
# (2025-12-06) Version 1.10.0
|
||||
|
||||
- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.11.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.16.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.16.0).
|
||||
|
||||
# (2025-11-30) Version 1.9.0
|
||||
|
||||
- (**Internal Improvement**) Upgrade [async-openai](https://crates.io/crates/async-openai) from our own etkecc fork (0.28.1-patched) to the official upstream version 0.31.1. This upgrade required some code adaptations to the new module structure, etc. While tested, regressions are possible.
|
||||
|
||||
# (2025-11-28) Version 1.8.3
|
||||
|
||||
- (**Improvement**) Add support for the `BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY` environment variable for configuring `persistence.session_encryption_key`
|
||||
|
||||
- (**Improvement**) Add support for the `BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED` environment variable for configuring `user.encryption.recovery_reset_allowed`
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
# (2025-11-20) Version 1.8.2
|
||||
|
||||
- (**Internal Improvement**) Dependency and compiler updates (Rust 1.89.0 -> 1.91.1).
|
||||
|
||||
# (2025-09-12) Version 1.8.1
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
# (2025-09-08) Version 1.8.0
|
||||
|
||||
- (**Internal Improvement**) Upgrade [mxlink](https://crates.io/crates/mxlink) (1.9.0 -> 1.10.0) and [matrix-sdk](https://crates.io/crates/matrix-sdk) (0.13.0 -> 0.14.0)
|
||||
|
||||
- (**Internal Improvement**) Upgrade [Rust](https://www.rust-lang.org/) (1.88.0 -> 1.89.0)
|
||||
|
||||
- (**Internal Improvement**) Upgrade Debian base for container images (12/bookworm -> 13/trixie)
|
||||
|
||||
# (2025-07-11) Version 1.7.6
|
||||
|
||||
- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.9.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.13.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.13.0), which contains fixes for some security vulnerabilities)
|
||||
|
||||
# (2025-06-10) Version 1.7.5
|
||||
|
||||
- (**Internal Improvement**) Dependency and compiler updates (Rust 1.86 -> 1.86).
|
||||
|
||||
# (2025-06-10) Version 1.7.4
|
||||
|
||||
- (**Internal Improvement**) Dependency updates.
|
||||
|
||||
# (2025-06-10) Version 1.7.3
|
||||
|
||||
- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.8.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.12.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.12.0), which contains fixes for important security vulnerabilities)
|
||||
|
||||
# (2025-05-11) Version 1.7.2
|
||||
|
||||
- (**Bugfix**) Allow `image_generation.size` configuration value for OpenAI to be `null` to allow the model to choose the size automatically and default to that
|
||||
|
||||
# (2025-05-11) Version 1.7.1
|
||||
|
||||
- (**Bugfix**) Fix lack of documentation for the new [image-editing](./docs/features.md#-image-editing) feature in the `!bai usage` command's output
|
||||
|
||||
# (2025-05-10) Version 1.7.0
|
||||
|
||||
- (**Feature**) Add vision support to the OpenAI and Anthropic providers. You can now mix text and images in your conversations - fixes [issue #5](https://github.com/etkecc/baibot/issues/5)
|
||||
|
||||
- (**Feature**) Add [image-editing](./docs/features.md#-image-editing) support to the OpenAI provider
|
||||
|
||||
- (**Improvement**) Add compatibility with OpenAI's `gpt-image-1` model - fixes [issue #40](https://github.com/etkecc/baibot/issues/40)
|
||||
|
||||
- (**Change**) Rework [image-creation](./docs/features.md#-image-creation) to avoid command conflicts with [image-editing](./docs/features.md#-image-editing). The image-creation command syntax is now `!bai image create <prompt>` (previously: `!bai image <prompt>`).
|
||||
|
||||
- (**Internal Improvement**) Dependency and compiler updates
|
||||
|
||||
> [!WARNING]
|
||||
> Unlike other releases, this release is not published to [crates.io](https://crates.io), because it relies on multiple library forks (`async-openai` and `anthropic-rs`) sourced from Github.
|
||||
|
||||
|
||||
# (2025-04-12) Version 1.6.0
|
||||
|
||||
- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.7.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.11.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.11.0))
|
||||
|
||||
|
||||
# (2025-03-31) Version 1.5.1
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
2608
Cargo.lock
generated
2608
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
21
Cargo.toml
21
Cargo.toml
@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
|
||||
readme = "README.md"
|
||||
keywords = ["matrix", "chat", "bot", "AI", "LLM"]
|
||||
include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
|
||||
version = "1.5.1"
|
||||
version = "1.17.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
@@ -15,25 +15,26 @@ name = "baibot"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
anthropic = "=0.0.8"
|
||||
anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" }
|
||||
anyhow = "1.0.*"
|
||||
async-openai = "0.28.*"
|
||||
async-openai = { version = "0.34.0", features = ["audio", "chat-completion", "image", "responses"] }
|
||||
base64 = "0.22.*"
|
||||
chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
|
||||
# We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
|
||||
# We add the `native-tls` feature, because of https://github.com/etkecc/rust-mxlink/issues/1
|
||||
matrix-sdk = { version = "0.10.0", default-features = false, features = ["native-tls"] }
|
||||
matrix-sdk = { version = "0.16.0", default-features = false, features = ["native-tls"] }
|
||||
mime_guess = "2.0.*"
|
||||
mxidwc = "1.0.*"
|
||||
mxlink = ">=1.6.0"
|
||||
mxlink = ">=1.13.0"
|
||||
etke_openai_api_rust = "0.1.*"
|
||||
quick_cache = "0.6.*"
|
||||
regex = "1.11.*"
|
||||
regex = "1.12.*"
|
||||
serde = { version = "1.0.*", features = ["derive"], default-features = false }
|
||||
serde_json = "1.0.*"
|
||||
serde_yaml = "0.9.*"
|
||||
tempfile = "3.19.*"
|
||||
tiktoken-rs = { version = "0.6.*", features = ["async-openai"] }
|
||||
tokio = { version = "1.44.*", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
serde_yaml_ng = "0.10.*"
|
||||
tempfile = "3.27.*"
|
||||
tiktoken-rs = { version = "0.9.*", default-features = false }
|
||||
tokio = { version = "1.50.*", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
tracing = "0.1.*"
|
||||
tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }
|
||||
url = "2.5.*"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# #
|
||||
#######################################
|
||||
|
||||
FROM docker.io/rust:1.85.1-slim-bookworm AS build
|
||||
FROM docker.io/rust:1.93.1-slim-trixie AS build
|
||||
|
||||
RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
|
||||
|
||||
@@ -39,7 +39,7 @@ RUN --mount=type=cache,target=/target,sharing=locked \
|
||||
# #
|
||||
#######################################
|
||||
|
||||
FROM docker.io/debian:bookworm-slim
|
||||
FROM docker.io/debian:trixie-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \
|
||||
apt-get clean && \
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# #
|
||||
#######################################
|
||||
|
||||
FROM docker.io/rust:1.85.1-slim-bookworm AS build
|
||||
FROM docker.io/rust:1.93.1-slim-trixie AS build
|
||||
|
||||
RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN cargo build --release
|
||||
# #
|
||||
#######################################
|
||||
|
||||
FROM docker.io/debian:bookworm-slim
|
||||
FROM docker.io/debian:trixie-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \
|
||||
apt-get clean && \
|
||||
|
||||
@@ -17,10 +17,10 @@ It's influenced by [chaz](https://github.com/arcuru/chaz), but does **not** use
|
||||
|
||||
- Supports **different use purposes** (depending on the [☁️ provider](./docs/providers.md) & model):
|
||||
|
||||
- [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text
|
||||
- [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text (though certain models may "see" images as well). The [OpenAI provider](./docs/providers.md#openai) also supports [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (web search, code interpreter)
|
||||
- [🦻 speech-to-text](./docs/features.md#-speech-to-text): turning your voice messages into text
|
||||
- [🗣️ text-to-speech](./docs/features.md#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
|
||||
- [🖌️ image-generation](./docs/features.md#%EF%B8%8F-image-generation): generating images based on instructions
|
||||
- [🖌️ image-generation](./docs/features.md#image-generation): creating and editing images based on instructions
|
||||
|
||||
- 🪄 Supports [seamless voice interaction](./docs/features.md#seamless-voice-interaction) (turning user voice messages into text, answering in text, then turning that text back into voice)
|
||||
|
||||
|
||||
@@ -43,7 +43,8 @@ Administrators cannot be changed without adjusting the bot's configuration on th
|
||||
|
||||
Room-local agent managers are users privileged to **create their own [agents](./agents.md)** (see `!bai agent`) in rooms.
|
||||
|
||||
**⚠️ WARNING**: Letting regular users create agents which contact arbitrary network services **may be a security issue**.
|
||||
> [!WARNING]
|
||||
> Letting regular users create agents which contact arbitrary network services **may be a security issue**.
|
||||
|
||||
The following commands are available:
|
||||
- **Show** the currently allowed users: `!bai access room-local-agent-managers`
|
||||
|
||||
@@ -35,7 +35,7 @@ Depending on where the agent is defined (within a room, globally, or [statically
|
||||
|
||||
When creating an agent, you will be given some sample [YAML](https://en.wikipedia.org/wiki/YAML) configuration which you can use to customize the agent's behavior.
|
||||
|
||||
This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Generation](./features.md#-image-generation)).
|
||||
This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Creation](./features.md#-image-creation), [🎨 Image Editing](./features.md#-image-editing), [🫵 Sticker Creation](./features.md#-sticker-creation)).
|
||||
|
||||
After making your modifications to the sample YAML, you submit it back to the bot and the new agent will be created.
|
||||
|
||||
|
||||
@@ -12,12 +12,17 @@ This file is created from the template found in [etc/app/config.yml.dist](../../
|
||||
|
||||
Certain keys can be left unset, in which case [📝 hardcoded defaults](../../src/entity/cfg/defaults.rs) would be used.
|
||||
|
||||
Each configuration key found in the YAML configuration can be overridden by setting an environment variable (dots should be replaced with `_`). Example:
|
||||
Some configuration keys found in the YAML configuration can be overridden by setting an environment variable (dots should be replaced with `_`). Example:
|
||||
|
||||
- to override `command_prefix`, set an environment variable `BAIBOT_COMMAND_PREFIX`
|
||||
- to override `homeserver.server_name`, set an environment variable `BAIBOT_HOMESERVER_SERVER_NAME`
|
||||
|
||||
The static configuration contains an `initial_global_config` key, which is used to populate the bot's global configuration (stored as [dynamic configuration](#dynamic-configuration)) the first time the bot starts. Modifying this subsequently will not have any effect. After initial global configuration creation, it's expected to be managed dynamically via chat commands.
|
||||
You can see the list of supported environment variables in the [🦀 src/entity/cfg/env.rs](../../src/entity/cfg/env.rs) file.
|
||||
|
||||
> [!WARNING]
|
||||
> The static configuration contains an `initial_global_config` key, which is used to populate the bot's global configuration (stored as [dynamic configuration](#dynamic-configuration)) the first time the bot starts. Modifying this subsequently will not have any effect. After initial global configuration creation, it's expected to be managed dynamically via chat commands.
|
||||
|
||||
For Matrix-account authentication setup, see [🔐 Authentication](./authentication.md).
|
||||
|
||||
|
||||
### Dynamic configuration
|
||||
@@ -40,7 +45,7 @@ You can adjust the following settings per room and/or globally:
|
||||
- [💬 Text Generation](text-generation.md)
|
||||
- [🦻 Speech-to-Text](speech-to-text.md)
|
||||
- [🗣️ Text-to-Speech](text-to-speech.md)
|
||||
- [🖌️ Image Generation](image-generation.md)
|
||||
- [🖌️ Image Creation](image-generation.md)
|
||||
- [🤝 Handlers](handlers.md)
|
||||
|
||||
Refer to the bot's help messages (as a response to a `!bai config` help command) for the most up-to-date information on what Room Settings can be configured.
|
||||
|
||||
23
docs/configuration/authentication.md
Normal file
23
docs/configuration/authentication.md
Normal file
@@ -0,0 +1,23 @@
|
||||
## 🔐 Authentication
|
||||
|
||||
baibot supports 2 authentication modes for the Matrix account (`user.*` keys in config).
|
||||
|
||||
Set **exactly one** mode. If both are set (or neither is set), startup validation fails.
|
||||
|
||||
### Password authentication
|
||||
|
||||
- Config key: `user.password`
|
||||
- Environment variable: `BAIBOT_USER_PASSWORD`
|
||||
|
||||
### Access token authentication
|
||||
|
||||
- Config keys: `user.access_token` + `user.device_id`
|
||||
- Environment variables: `BAIBOT_USER_ACCESS_TOKEN` + `BAIBOT_USER_DEVICE_ID`
|
||||
|
||||
Access-token authentication is useful for OIDC-enabled homeservers (e.g. those using [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service)).
|
||||
|
||||
Example token-generation command:
|
||||
|
||||
```sh
|
||||
mas-cli manage issue-compatibility-token <username> [device_id]
|
||||
```
|
||||
@@ -8,10 +8,10 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
|
||||
|
||||
The bot supports the following use-purposes:
|
||||
|
||||
- [💬 text-generation](../features.md#-text-generation): communicating with you via text
|
||||
- [💬 text-generation](../features.md#-text-generation): communicating with you via text (though certain models may also process images and files)
|
||||
- [🦻 speech-to-text](../features.md#-speech-to-text): turning your voice messages into text
|
||||
- [🗣️ text-to-speech](../features.md#️-text-to-speech): turning bot or users text messages into voice messages
|
||||
- [🖌️ image-generation](../features.md#-image-generation): generating images based on instructions
|
||||
- [🖌️ image-generation](../features.md#image-generation): generating images based on instructions
|
||||
|
||||
In a given room, each different purpose can be served by a different [provider](../providers.md) and model. This combination of provider and model configuration is called an [🤖 agent](../agents.md). Each purpose can be served by a different **handler** agent.
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
|
||||
## 🖌️ Image Generation
|
||||
## Image Generation
|
||||
|
||||
The Image Generation feature is not configurable at this moment.
|
||||
The Image Creation and Image Editing features are not configurable at this moment.
|
||||
|
||||
You may also wish to see:
|
||||
|
||||
- [🌟 Features / 🖌️ Image Generation](../features.md#-image-generation) for a higher-level introduction to the Image Generation features
|
||||
- [📖 Usage / 🖌️ Image Generation](../usage.md#-image-generation) section for more details on how to use the bot for Image Generation in a room
|
||||
- [🌟 Features / Image Generation / 🖌️ Image Creation](../features.md#-image-creation) for a higher-level introduction to the Image Creation features
|
||||
- [🌟 Features / Image Generation / 🎨 Image Editing](../features.md#-image-editing) for a higher-level introduction to the Image Editing features
|
||||
- [📖 Usage / Image Generation / 🖌️ Creating Images](../usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room
|
||||
- [📖 Usage / Image Generation / 🎨 Editing images](../usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room
|
||||
|
||||
@@ -57,6 +57,25 @@ This feature relies on [tokenization](https://en.wikipedia.org/wiki/Large_langua
|
||||
This setting is **disabled by default**, but can be enabled via `!bai config room text-generation set-context-management-enabled true` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)).
|
||||
|
||||
|
||||
### 👤 Sender Context Mode
|
||||
|
||||
In multi-user rooms, it may be useful for the model to know which participant sent each message in the conversation context.
|
||||
|
||||
To support this, the bot has a `text-generation sender-context-mode` setting, which can be set to:
|
||||
|
||||
- (default) `disabled`: do not attach sender metadata to messages before sending them to the model
|
||||
|
||||
- `matrix_user_id`: prefix text messages with the sender's Matrix user ID, for example: `[sender=@alice:example.com] Hello bot`
|
||||
|
||||
- `matrix_user_id_and_timestamp`: prefix text messages with the sender's Matrix user ID and the message timestamp, for example: `[sender=@alice:example.com sent_at=2026-03-23T14:30:00Z] Hello bot`
|
||||
|
||||
This sender metadata is attached to conversation messages before they are sent to the model provider. It applies to user and assistant text messages, but not to system prompts or non-text content.
|
||||
|
||||
⚠️ Enabling this sends Matrix user IDs, and optionally timestamps, to the model provider.
|
||||
|
||||
Example: `!bai config room text-generation set-sender-context-mode matrix_user_id` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
|
||||
|
||||
|
||||
### ⌨️ Prompt Override
|
||||
|
||||
You can override the [system prompt](https://huggingface.co/docs/transformers/en/tasks/prompting) configured at the [🤖 agent](../agents.md) level.
|
||||
|
||||
@@ -18,6 +18,27 @@ For local development, we run all dependency services in [🐋 Docker](https://w
|
||||
- (Optional) an API key for some Large Language Model [☁️ provider](./providers.md) (e.g. [OpenAI](./providers.md#openai)), though we recommend using [LocalAI](#localai) or [Ollama](#ollama) for local development
|
||||
|
||||
|
||||
### Choosing a homeserver
|
||||
|
||||
The development environment supports two homeserver implementations:
|
||||
|
||||
- **[Continuwuity](https://continuwuity.org/)** (default) — lightweight, no external database required. Good for most development needs.
|
||||
- **[Synapse](https://github.com/element-hq/synapse)** — the reference implementation, bundled with Postgres. Use this if you need Synapse-specific behavior.
|
||||
|
||||
To choose a homeserver (optional — defaults to Continuwuity if skipped):
|
||||
|
||||
```sh
|
||||
just homeserver-init continuwuity # or: just homeserver-init synapse
|
||||
```
|
||||
|
||||
The choice is stored in `var/homeserver` and affects all subsequent commands.
|
||||
|
||||
> **Note:** If you switch homeservers after initial setup, you will need to:
|
||||
> - Delete `var/app/local/` and/or `var/app/container/` (app config and data)
|
||||
> - Delete `var/services/element-web/` (to regenerate its config)
|
||||
> - Re-run the prepare and user registration steps
|
||||
|
||||
|
||||
### Getting started guide
|
||||
|
||||
Developing [locally](#running-locally) is possible, but requires a [Rust](https://www.rust-lang.org/) toolchain.
|
||||
@@ -28,11 +49,12 @@ In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency
|
||||
|
||||
#### Running locally
|
||||
|
||||
1. Start the core dependency services (Postgres, Synapse, Element Web): `just services-start`
|
||||
2. (Only the first time around) Prepare initial app configuration in `var/app/local/config.yml`: `just app-local-prepare`
|
||||
3. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
|
||||
4. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
|
||||
5. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
|
||||
1. (Optional) Choose a homeserver: `just homeserver-init continuwuity` (or `synapse`). Default is `continuwuity`.
|
||||
2. Start the homeserver and Element Web: `just services-start`
|
||||
3. (Only the first time around) Prepare initial app configuration in `var/app/local/config.yml`: `just app-local-prepare`
|
||||
4. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
|
||||
5. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
|
||||
6. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
|
||||
- for [LocalAI](#localai):
|
||||
- Start services: `just localai-start`
|
||||
- Wait a while for LocalAI to start up. It has a lot of models to download. Monitor progress using `just localai-tail-logs`
|
||||
@@ -40,12 +62,12 @@ In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency
|
||||
- for [Ollama](#ollama):
|
||||
- Start services: `just ollama-start`
|
||||
- (Only the first time around) Pull the model configured in `agents.static_definitions` in the configuration file: `just ollama-pull-model gemma2:2b`
|
||||
6. Start the bot: `just run-locally`
|
||||
7. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
|
||||
8. Create a new room and invite `@baibot:synapse.127.0.0.1.nip.io`
|
||||
9. When done, stop the bot (`Ctrl` + `C`)
|
||||
10. Stop the core dependency services: `just services-stop`
|
||||
11. (Optional) Stop additional services:
|
||||
7. Start the bot: `just run-locally`
|
||||
8. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
|
||||
9. Create a new room and invite `@baibot:continuwuity.127.0.0.1.nip.io` (or `@baibot:synapse.127.0.0.1.nip.io` if using Synapse)
|
||||
10. When done, stop the bot (`Ctrl` + `C`)
|
||||
11. Stop the services: `just services-stop`
|
||||
12. (Optional) Stop additional services:
|
||||
- for [LocalAI](#localai): `just localai-stop`
|
||||
- for [Ollama](#ollama): `just ollama-stop`
|
||||
|
||||
@@ -54,11 +76,12 @@ In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency
|
||||
|
||||
You can avoid having a [Rust](https://www.rust-lang.org/) toolchain installed locally and build/run this in a container.
|
||||
|
||||
1. Start the core dependency services (Postgres, Synapse, Element Web): `just services-start`
|
||||
2. (Only the first time around) Prepare initial app configuration in `var/app/container/config.yml`: `just app-container-prepare`
|
||||
3. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
|
||||
4. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
|
||||
5. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
|
||||
1. (Optional) Choose a homeserver: `just homeserver-init continuwuity` (or `synapse`). Default is `continuwuity`.
|
||||
2. Start the homeserver and Element Web: `just services-start`
|
||||
3. (Only the first time around) Prepare initial app configuration in `var/app/container/config.yml`: `just app-container-prepare`
|
||||
4. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
|
||||
5. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
|
||||
6. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
|
||||
- for [LocalAI](#localai):
|
||||
- Start services: `just localai-start`
|
||||
- Wait a while for LocalAI to start up. It has a lot of models to download. Monitor progress using `just localai-tail-logs`
|
||||
@@ -66,12 +89,12 @@ You can avoid having a [Rust](https://www.rust-lang.org/) toolchain installed lo
|
||||
- for [Ollama](#ollama):
|
||||
- Start services: `just ollama-start`
|
||||
- (Only the first time around) Pull the model configured in `agents.static_definitions` in the configuration file: `just ollama-pull-model gemma2:2b`
|
||||
6. Start the bot: `just run-in-container`
|
||||
7. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
|
||||
8. Create a new room and invite `@baibot:synapse.127.0.0.1.nip.io`
|
||||
9. When done, stop the bot (`Ctrl` + `C`)
|
||||
10. Stop the dependency services: `just services-stop`
|
||||
11. (Optional) Stop additional services:
|
||||
7. Start the bot: `just run-in-container`
|
||||
8. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
|
||||
9. Create a new room and invite `@baibot:continuwuity.127.0.0.1.nip.io` (or `@baibot:synapse.127.0.0.1.nip.io` if using Synapse)
|
||||
10. When done, stop the bot (`Ctrl` + `C`)
|
||||
11. Stop the services: `just services-stop`
|
||||
12. (Optional) Stop additional services:
|
||||
- for [LocalAI](#localai): `just localai-stop`
|
||||
- for [Ollama](#ollama): `just ollama-stop`
|
||||
|
||||
@@ -93,7 +116,7 @@ For getting started most quickly (and locally), we recommend using [LocalAI](#lo
|
||||
|
||||
**Ollama is most lightweight** (~2GB for the container image + ~1.6GB for the model), but supports only [💬 text-generation](./features.md#-text-generation).
|
||||
|
||||
**LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-generation).
|
||||
**LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-creation).
|
||||
|
||||
**OpenAI supports all of these capabilities** as well and does not require powerful hardware or lots of disk space. However, it requires signup and an API key.
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
|
||||
|
||||
The bot supports the following use-purposes:
|
||||
|
||||
- [💬 text-generation](#-text-generation): communicating with you via text
|
||||
- [💬 text-generation](#-text-generation): communicating with you via text (though certain models may also process images and files)
|
||||
- [🦻 speech-to-text](#-speech-to-text): turning your voice messages into text
|
||||
- [🗣️ text-to-speech](#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
|
||||
- [🖌️ image-generation](#%EF%B8%8F-image-generation): generating images based on instructions
|
||||
@@ -22,14 +22,18 @@ For more information about configuring handlers, see the [🤝 Handlers / Config
|
||||
|
||||
### 💬 Text Generation
|
||||
|
||||
Text Generation is the bot's ability to **respond to users' text messages with text**.
|
||||
Text Generation is the bot's ability to **respond to users' messages with text**.
|
||||
|
||||

|
||||
|
||||
Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation. Note that certain providers may not support all file types or may have issues with specific files (e.g. scanned/image-based PDFs). If a file is rejected by the provider, the conversation thread may become unusable — start a new thread to work around this.
|
||||
|
||||
In multi-user (group) rooms, to avoid disturbing the normal conversation between people, the bot is auto-configured to only respond to messages starting with the command prefix (`!bai`) or direct mentions via the [💬 Text Generation / 🗟 Prefix Requirement Type](./configuration/text-generation.md#-prefix-requirement-type) setting.
|
||||
|
||||
Normally, the bot only responds to allowed [👥 Users](./access.md#-users). In certain cases, it's useful for an allowed user to provoke the bot to respond even in foreign threads or reply chains. You can learn more about this feature in the [On-demand involvement](./features.md#on-demand-involvement) section below.
|
||||
|
||||
If needed, the bot can also attach sender metadata to conversation messages before sending them to the model, which can help the model distinguish between participants in multi-user rooms. See [🛠️ Configuration / 💬 Text Generation / 👤 Sender Context Mode](./configuration/text-generation.md#-sender-context-mode).
|
||||
|
||||
A few other features (like [🗣️ Text-to-Speech](#️-text-to-speech) and [🦻 Speech-to-Text](#-speech-to-text)) combine well with Text Generation, so you **don't necessarily need to communicate with the bot via text** (with [Seamless voice interaction](#seamless-voice-interaction), you can communicate only with voice).
|
||||
|
||||
You may also wish to see:
|
||||
@@ -38,6 +42,23 @@ You may also wish to see:
|
||||
- [📖 Usage / 💬 Text Generation](./usage.md#-text-generation) section for more details on how to use the bot for Text Generation in a room
|
||||
|
||||
|
||||
#### 🛠️ Built-in Tools (OpenAI only)
|
||||
|
||||
|
||||
|
||||
The [OpenAI provider](./providers.md#openai) supports built-in tools that extend the model's capabilities:
|
||||
|
||||
- [🔍 Web Search](https://platform.openai.com/docs/guides/tools-web-search) (`web_search`): allows the model to search the web for up-to-date information. [🖼️ Screenshot](./screenshots/text-generation-tools-web-search.webp)
|
||||
|
||||
- [💻 Code Interpreter](https://platform.openai.com/docs/guides/tools-code-interpreter) (`code_interpreter`): allows the model to write and execute Python code in a sandbox
|
||||
|
||||
These tools are **disabled by default** and need to be explicitly enabled in the agent's `text_generation.tools` configuration. See the [OpenAI sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15) for reference.
|
||||
|
||||
To enable tools on an existing dynamically-created agent, you need to [update the agent](./agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need
|
||||
|
||||
💡 **Note**: These tools run on OpenAI's infrastructure and may incur additional costs. Web search results include citations that are incorporated into the response.
|
||||
|
||||
|
||||
#### On-demand involvement
|
||||
|
||||
In the following 2 cases, it's useful to involve the bot in conversations on-demand:
|
||||
@@ -139,26 +160,42 @@ To operate in this mode, you can:
|
||||
- optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
|
||||
|
||||
|
||||
### 🖌️ Image Generation
|
||||
### Image Generation
|
||||
|
||||
Image generation is the bot's ability to **generate images** based on text prompts.
|
||||
#### 🖌️ Image Creation
|
||||
|
||||
See a [🖼️ Screenshot of the Image Generation feature](./screenshots/image-generation.webp).
|
||||
Image creation is the bot's ability to **create images** based on text prompts.
|
||||
|
||||
See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp).
|
||||
|
||||
You may also wish to see:
|
||||
|
||||
- [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation
|
||||
- [📖 Usage / 🖌️ Image Generation](./usage.md#-image-generation) section for more details on how to use the bot for Image Generation in a room
|
||||
- [🫵 Sticker Generation](#-sticker-generation) - a special case of Image Generation
|
||||
- [📖 Usage / Image Generation / 🖌️ Creating Images](./usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room
|
||||
- [🖌️ Image Editing](#️-image-editing) - another image generation feature
|
||||
- [🫵 Sticker Creation](#-sticker-creation) - a special case of Image Creation
|
||||
|
||||
|
||||
### 🫵 Sticker Generation
|
||||
#### 🎨 Image Editing
|
||||
|
||||
Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Generation](#️-image-generation).
|
||||
Image editing is the bot's ability to **edit images** based on a prompt and one or more existing images.
|
||||
|
||||
See a [🖼️ Screenshot of the Sticker Generation feature](./screenshots/sticker-generation.webp).
|
||||
See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp).
|
||||
|
||||
See [📖 Usage / 🖌️ Image Generation / Generating Stickers](./usage.md#generating-stickers) for details.
|
||||
You may also wish to see:
|
||||
|
||||
- [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation
|
||||
- [📖 Usage / Image Generation / 🎨 Editing images](./usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room
|
||||
- [🖌️ Image Creation](#️-image-creation) - another image generation feature
|
||||
|
||||
|
||||
#### 🫵 Sticker Creation
|
||||
|
||||
Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Creation](#️-image-creation).
|
||||
|
||||
See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp).
|
||||
|
||||
See [📖 Usage / Image Generation / 🫵 Creating Stickers](./usage.md#-creating-stickers) for details.
|
||||
|
||||
|
||||
### 🔒 Encryption
|
||||
|
||||
@@ -53,6 +53,7 @@ CONTAINER_IMAGE_NAME=ghcr.io/etkecc/baibot:v1.0.0
|
||||
--env BAIBOT_PERSISTENCE_DATA_DIR_PATH=/data \
|
||||
--mount type=bind,src=/path/to/config.yml,dst=/app/config.yml,ro \
|
||||
--mount type=bind,src=/path/to/data,dst=/data \
|
||||
--tmpfs=/tmp:rw,noexec,nosuid,size=1024m \
|
||||
$CONTAINER_IMAGE_NAME
|
||||
```
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ The list of supported providers is below.
|
||||
|
||||
### How to choose a provider
|
||||
|
||||
If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation), [🖌️ image-generation](./features.md#️-image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
|
||||
If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🖌️ image-generation](./features.md#️image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
|
||||
|
||||
You don't need to choose just one though. The bot supports [mixing & matching models](./features.md#-mixing--matching-models), so you can use multiple providers at the same time.
|
||||
|
||||
@@ -47,7 +47,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `anthropic`
|
||||
- 🔗 Links: [🏠 Home page](https://www.anthropic.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Anthropic), [👤 Sign up](https://console.anthropic.com/), [📋 Models list](https://docs.anthropic.com/en/docs/about-claude/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (incl. vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local anthropic my-anthropic-agent`
|
||||
- create a global agent: `!bai agent create-global anthropic my-anthropic-agent`
|
||||
@@ -61,7 +61,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `groq`
|
||||
- 🔗 Links: [🏠 Home page](https://groq.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Groq), [👤 Sign up](https://console.groq.com/login), [📋 Models list](https://console.groq.com/docs/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local groq my-groq-agent`
|
||||
- create a global agent: `!bai agent create-global groq my-groq-agent`
|
||||
@@ -75,7 +75,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `localai`
|
||||
- 🔗 Links: [🏠 Home page](https://localai.io/), [📋 Models list](https://localai.io/gallery.html)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local localai my-localai-agent`
|
||||
- create a global agent: `!bai agent create-global localai my-localai-agent`
|
||||
@@ -89,7 +89,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `mistral`
|
||||
- 🔗 Links: [🏠 Home page](https://mistral.ai/), [🌐 Wiki](https://en.wikipedia.org/wiki/Mistral_AI), [👤 Sign up](https://auth.mistral.ai/ui/registration), [📋 Models list](https://docs.mistral.ai/getting-started/models/)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local mistral my-mistral-agent`
|
||||
- create a global agent: `!bai agent create-global mistral my-mistral-agent`
|
||||
@@ -103,7 +103,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `ollama`
|
||||
- 🔗 Links: [🏠 Home page](https://ollama.com/), [📋 Models list](https://ollama.com/library)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local ollama my-ollama-agent`
|
||||
- create a global agent: `!bai agent create-global ollama my-ollama-agent`
|
||||
@@ -120,15 +120,12 @@ For services which are not fully compatible with the OpenAI API, consider using
|
||||
|
||||
- 🆔 Identifier: `openai`
|
||||
- 🔗 Links: [🏠 Home page](https://openai.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/OpenAI), [👤 Sign up](https://platform.openai.com/signup), [📋 Models list](https://platform.openai.com/docs/models)
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-generation), [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local openai my-openai-agent`
|
||||
- create a global agent: `!bai agent create-global openai my-openai-agent`
|
||||
|
||||
💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which:
|
||||
|
||||
- in the general case looks [like this](./sample-provider-configs/openai.yml)
|
||||
- for the [o1](https://platform.openai.com/docs/models/o1) models needs to look [like this](./sample-provider-configs/openai-o1.yml)
|
||||
💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/openai.yml).
|
||||
|
||||
|
||||
### OpenAI Compatible
|
||||
@@ -140,7 +137,7 @@ Some of these popular services already have **shortcut** providers (leading to t
|
||||
This provider is just as featureful as the [OpenAI](#openai) provider, but is more compatible with services which do not fully adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/).
|
||||
|
||||
- 🆔 Identifier: `openai-compatible`
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-generation), [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local openai-compatible my-openai-compatible-agent`
|
||||
- create a global agent: `!bai agent create-global openai-compatible my-openai-compatible-agent`
|
||||
@@ -154,7 +151,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
|
||||
|
||||
- 🆔 Identifier: `openrouter`
|
||||
- 🔗 Links: [🏠 Home page](https://openrouter.ai/), [👤 Sign up](https://openrouter.ai/), [📋 Models list](https://openrouter.ai/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local openrouter my-openrouter-agent`
|
||||
- create a global agent: `!bai agent create-global openrouter my-openrouter-agent`
|
||||
@@ -168,7 +165,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
|
||||
|
||||
- 🆔 Identifier: `together-ai`
|
||||
- 🔗 Links: [🏠 Home page](https://www.together.ai/), [👤 Sign up](https://api.together.ai/signup), [📋 Models list](https://api.together.xyz/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local together-ai my-together-ai-agent`
|
||||
- create a global agent: `!bai agent create-global together-ai my-together-ai-agent`
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
base_url: https://api.openai.com/v1
|
||||
api_key: YOUR_API_KEY_HERE
|
||||
text_generation:
|
||||
model_id: o1-mini
|
||||
# o1 models do not support a system prompt
|
||||
prompt: null
|
||||
temperature: 1.0
|
||||
# o1 models do not support max_response_tokens.
|
||||
# They use `max_completion_tokens` as an alternative
|
||||
max_response_tokens: null
|
||||
max_completion_tokens: 16384
|
||||
max_context_tokens: 128000
|
||||
speech_to_text:
|
||||
model_id: whisper-1
|
||||
text_to_speech:
|
||||
model_id: tts-1-hd
|
||||
voice: onyx
|
||||
speed: 1.0
|
||||
response_format: opus
|
||||
image_generation:
|
||||
model_id: dall-e-3
|
||||
style: vivid
|
||||
size: 1024x1024
|
||||
quality: standard
|
||||
@@ -1,11 +1,18 @@
|
||||
base_url: https://api.openai.com/v1
|
||||
api_key: YOUR_API_KEY_HERE
|
||||
text_generation:
|
||||
model_id: gpt-4o
|
||||
model_id: gpt-5.4
|
||||
prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
|
||||
temperature: 1.0
|
||||
max_response_tokens: 16384
|
||||
max_context_tokens: 128000
|
||||
# Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
|
||||
# If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`.
|
||||
max_response_tokens: null
|
||||
max_completion_tokens: 128000
|
||||
max_context_tokens: 400000
|
||||
# Built-in tools
|
||||
tools:
|
||||
web_search: false
|
||||
code_interpreter: false
|
||||
speech_to_text:
|
||||
model_id: whisper-1
|
||||
text_to_speech:
|
||||
@@ -14,7 +21,7 @@ text_to_speech:
|
||||
speed: 1.0
|
||||
response_format: opus
|
||||
image_generation:
|
||||
model_id: dall-e-3
|
||||
style: vivid
|
||||
size: 1024x1024
|
||||
quality: standard
|
||||
model_id: gpt-image-1.5
|
||||
style: null
|
||||
size: null
|
||||
quality: null
|
||||
|
||||
BIN
docs/screenshots/image-creation.webp
Normal file
BIN
docs/screenshots/image-creation.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 298 KiB |
BIN
docs/screenshots/image-editing-multiple-images.webp
Normal file
BIN
docs/screenshots/image-editing-multiple-images.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 339 KiB |
BIN
docs/screenshots/image-editing-single-image.webp
Normal file
BIN
docs/screenshots/image-editing-single-image.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 285 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 684 KiB |
BIN
docs/screenshots/text-generation-tools-web-search.webp
Normal file
BIN
docs/screenshots/text-generation-tools-web-search.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -11,6 +11,8 @@ This is related to the [💬 Text Generation](./features.md#-text-generation) fe
|
||||
|
||||
If there's a text-generation handler agent configured, the bot **may** respond to messages sent in the room.
|
||||
|
||||
Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation.
|
||||
|
||||
See screenshots of:
|
||||
|
||||
- 🖼️ [the default Text Generation flow](./screenshots/text-generation.webp) in 1:1 rooms
|
||||
@@ -64,34 +66,48 @@ The speech-to-text feature triggers automatically by default, but can be adjuste
|
||||
If all your messages are in the same language, you can improve accuracy & latency by configuring the language (see [🦻 Speech-to-Text / 🔤 Language](./configuration/speech-to-text.md#-language)).
|
||||
|
||||
|
||||
### 🖌️ Image Generation
|
||||
|
||||
This is related to the [🖌️ Image Generation](./features.md#️-image-generation) feature.
|
||||
### Image Generation
|
||||
|
||||
This feature is not configurable at the moment. The configuration (size, quality, style) specified at the [🤖 agent](./agents.md) level will be used.
|
||||
|
||||
Capabilities depend on the [☁️ provider](./providers.md) and model used.
|
||||
|
||||
#### Generating images
|
||||
|
||||
Simply send a command like `!bai image A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
|
||||
#### 🖌️ Creating images
|
||||
|
||||
See a [🖼️ Screenshot of the Image Generation feature](./screenshots/image-generation.webp).
|
||||
Simply send a command like `!bai image create A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
|
||||
|
||||
You can then, respond in the same message thread with:
|
||||
See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp).
|
||||
|
||||
You can then respond in the same message thread with:
|
||||
|
||||
- more messages, to add more criteria to your prompt.
|
||||
- a message saying `again`, to generate one more image with the current prompt.
|
||||
|
||||
|
||||
#### Generating stickers
|
||||
#### 🎨 Editing images
|
||||
|
||||
A variation of [generating images](#generating-images) is to generate "sticker images".
|
||||
Simply send a command like `!bai image edit Turn the following image into an anime-style drawing` and the bot will start a threaded conversation asking for more details.
|
||||
|
||||
See a [🖼️ Screenshot of the Sticker Generation feature](./screenshots/sticker-generation.webp).
|
||||
See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp).
|
||||
|
||||
To generate a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`.
|
||||
You can then respond in the same message thread with:
|
||||
|
||||
The difference from [generating images](#generating-images) is that the bot will:
|
||||
- more messages, to add more criteria to your prompt.
|
||||
- one or more images, to provide the images that the bot will operate on.
|
||||
- a message saying `go`, to start the image generation process.
|
||||
- a message saying `again`, to prompt the bot to generate one more image edit with the current prompt.
|
||||
|
||||
|
||||
#### 🫵 Creating stickers
|
||||
|
||||
A variation of [creating images](#creating-images) is creating "sticker images".
|
||||
|
||||
See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp).
|
||||
|
||||
To create a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`.
|
||||
|
||||
The difference from [creating images](#creating-images) is that the bot will:
|
||||
|
||||
- generate a smaller-resolution image (currently hardcoded to `256x256`) - smaller/quicker, but still good enough for a sticker
|
||||
- potentially switch to a different (cheaper or otherwise more suitable) model, if available
|
||||
|
||||
@@ -1,16 +1,31 @@
|
||||
homeserver:
|
||||
# The canonical homeserver domain name
|
||||
server_name: synapse.127.0.0.1.nip.io
|
||||
url: http://synapse.127.0.0.1.nip.io:42020
|
||||
server_name: __HOMESERVER_SERVER_NAME__
|
||||
url: __HOMESERVER_URL__
|
||||
|
||||
user:
|
||||
mxid_localpart: baibot
|
||||
|
||||
# Authentication: set EITHER password OR access_token + device_id.
|
||||
#
|
||||
# Password-based login (traditional homeservers):
|
||||
password: baibot
|
||||
|
||||
# Access token login (for Matrix Authentication Service/OIDC-enabled homeservers):
|
||||
# Generate a token via: mas-cli manage issue-compatibility-token <username> [device_id]
|
||||
# access_token: null
|
||||
# device_id: null
|
||||
|
||||
# The name the bot uses as a display name and when it refers to itself.
|
||||
# Leave empty to use the default (baibot).
|
||||
name: baibot
|
||||
|
||||
# An optional path to an image file to be used as a custom avatar image.
|
||||
# - null or empty string: use the default avatar
|
||||
# - "keep": don't touch the avatar, keep whatever is already set
|
||||
# - any other value: path to a custom avatar image file
|
||||
avatar: null
|
||||
|
||||
encryption:
|
||||
# An optional passphrase to use for backing up and recovering the bot's encryption keys.
|
||||
# You can use any string here.
|
||||
@@ -39,7 +54,7 @@ room:
|
||||
access:
|
||||
# Space-separated list of MXID patterns which specify who is an admin.
|
||||
admin_patterns:
|
||||
- "@admin:synapse.127.0.0.1.nip.io"
|
||||
- "@admin:__HOMESERVER_SERVER_NAME__"
|
||||
|
||||
persistence:
|
||||
# This is unset here, because we expect the configuration to come from an environment variable (BAIBOT_PERSISTENCE_DATA_DIR_PATH).
|
||||
@@ -76,13 +91,18 @@ agents:
|
||||
# base_url: https://api.openai.com/v1
|
||||
# api_key: ""
|
||||
# text_generation:
|
||||
# model_id: gpt-4o
|
||||
# model_id: gpt-5.4
|
||||
# prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
|
||||
# temperature: 1.0
|
||||
# max_response_tokens: 16384
|
||||
# # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
|
||||
# max_completion_tokens: ~
|
||||
# max_context_tokens: 128000
|
||||
# # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`.
|
||||
# max_response_tokens: null
|
||||
# max_completion_tokens: 128000
|
||||
# max_context_tokens: 400000
|
||||
# # Built-in tools
|
||||
# tools:
|
||||
# web_search: false
|
||||
# code_interpreter: false
|
||||
# speech_to_text:
|
||||
# model_id: whisper-1
|
||||
# text_to_speech:
|
||||
@@ -91,10 +111,10 @@ agents:
|
||||
# speed: 1.0
|
||||
# response_format: opus
|
||||
# image_generation:
|
||||
# model_id: dall-e-3
|
||||
# style: vivid
|
||||
# size: 1024x1024
|
||||
# quality: standard
|
||||
# model_id: gpt-image-1.5
|
||||
# style: null
|
||||
# size: null
|
||||
# quality: null
|
||||
#
|
||||
# - id: localai
|
||||
# provider: localai
|
||||
@@ -146,7 +166,7 @@ initial_global_config:
|
||||
# Space-separated list of MXID patterns which specify who can use the bot.
|
||||
# By default, we let anyone on the homeserver use the bot.
|
||||
user_patterns:
|
||||
- "@*:synapse.127.0.0.1.nip.io"
|
||||
- "@*:__HOMESERVER_SERVER_NAME__"
|
||||
|
||||
# Controls logging.
|
||||
#
|
||||
|
||||
23
etc/services/continuwuity/compose.yml
Normal file
23
etc/services/continuwuity/compose.yml
Normal file
@@ -0,0 +1,23 @@
|
||||
services:
|
||||
continuwuity:
|
||||
image: forgejo.ellis.link/continuwuation/continuwuity:v0.5.6
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
cap_drop:
|
||||
- ALL
|
||||
read_only: true
|
||||
environment:
|
||||
CONDUWUIT_CONFIG: /etc/continuwuity/continuwuity.toml
|
||||
CONDUWUIT_DATABASE_PATH: /var/lib/continuwuity
|
||||
ports:
|
||||
- "${SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API}:6167"
|
||||
volumes:
|
||||
- ../../etc/services/continuwuity/config:/etc/continuwuity:ro
|
||||
- ./continuwuity/data:/var/lib/continuwuity
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=500m
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ${NETWORK_NAME}
|
||||
external: true
|
||||
19
etc/services/continuwuity/config/continuwuity.toml
Normal file
19
etc/services/continuwuity/config/continuwuity.toml
Normal file
@@ -0,0 +1,19 @@
|
||||
[global]
|
||||
server_name = "continuwuity.127.0.0.1.nip.io"
|
||||
|
||||
address = "0.0.0.0"
|
||||
port = 6167
|
||||
|
||||
database_path = "/var/lib/continuwuity"
|
||||
|
||||
allow_registration = true
|
||||
yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true
|
||||
|
||||
new_user_displayname_suffix = ""
|
||||
|
||||
max_request_size = 20_000_000
|
||||
|
||||
allow_federation = false
|
||||
trusted_servers = ["matrix.org"]
|
||||
|
||||
log = "info,state_res=warn,rocket=off,_=off,sled=off"
|
||||
48
etc/services/continuwuity/register-user.sh
Executable file
48
etc/services/continuwuity/register-user.sh
Executable file
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "Usage: $0 <env-file> <username> <password>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ENV_FILE="$1"
|
||||
USERNAME="$2"
|
||||
PASSWORD="$3"
|
||||
|
||||
SERVER="http://$(grep '^SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API=' "${ENV_FILE}" | cut -d= -f2)"
|
||||
REGISTER_URL="${SERVER}/_matrix/client/v3/register"
|
||||
|
||||
echo "Registering user '${USERNAME}' on ${SERVER}..."
|
||||
|
||||
SESSION_RESPONSE=$(curl -s -X POST "${REGISTER_URL}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\"}")
|
||||
|
||||
SESSION_ID=$(echo "${SESSION_RESPONSE}" | grep -o '"session":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||||
if [ -z "${SESSION_ID}" ]; then
|
||||
echo "Error: Could not get session ID. Response: ${SESSION_RESPONSE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Determine the required auth flow from the server response.
|
||||
# The first user requires m.login.registration_token (bootstrap token from logs).
|
||||
# Subsequent users use m.login.dummy (open registration).
|
||||
if echo "${SESSION_RESPONSE}" | grep -q 'm.login.registration_token'; then
|
||||
CONTAINER_ID=$(docker ps -q --filter name=baibot-continuwuity-continuwuity)
|
||||
REG_TOKEN=$(docker logs "${CONTAINER_ID}" 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep 'using the registration token' | grep -oP 'registration token \K[A-Za-z0-9]+' | head -1)
|
||||
AUTH_BODY="{\"type\": \"m.login.registration_token\", \"token\": \"${REG_TOKEN}\", \"session\": \"${SESSION_ID}\"}"
|
||||
else
|
||||
AUTH_BODY="{\"type\": \"m.login.dummy\", \"session\": \"${SESSION_ID}\"}"
|
||||
fi
|
||||
|
||||
RESULT=$(curl -s -X POST "${REGISTER_URL}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\", \"auth\": ${AUTH_BODY}}")
|
||||
|
||||
if echo "${RESULT}" | grep -q '"user_id"'; then
|
||||
echo "Successfully registered user: $(echo "${RESULT}" | grep -o '"user_id":"[^"]*"' | cut -d'"' -f4)"
|
||||
else
|
||||
echo "Registration failed. Response: ${RESULT}"
|
||||
exit 1
|
||||
fi
|
||||
21
etc/services/element-web/compose.yml
Normal file
21
etc/services/element-web/compose.yml
Normal file
@@ -0,0 +1,21 @@
|
||||
services:
|
||||
element-web:
|
||||
image: ghcr.io/element-hq/element-web:v1.12.13
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
ELEMENT_WEB_PORT: 8080
|
||||
ports:
|
||||
- "${SERVICE_ELEMENT_WEB_BIND_PORT_HTTP}:8080"
|
||||
volumes:
|
||||
- ./element-web/config.json:/app/config.json:ro
|
||||
tmpfs:
|
||||
- /var/cache/nginx:rw,mode=777
|
||||
- /var/run:rw,mode=777
|
||||
- /tmp/element-web-config:rw,mode=777
|
||||
- /etc/nginx/conf.d:rw,mode=777
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ${NETWORK_NAME}
|
||||
external: true
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"default_hs_url": "http://synapse.127.0.0.1.nip.io:42020",
|
||||
"default_hs_url": "__HOMESERVER_CLIENT_URL__",
|
||||
"default_is_url": "https://vector.im",
|
||||
"integrations_ui_url": "https://scalar.vector.im/",
|
||||
"integrations_rest_url": "https://scalar.vector.im/api",
|
||||
@@ -3,6 +3,8 @@ SERVICE_SYNAPSE_BIND_PORT_FEDERATION_API=127.0.0.1:42028
|
||||
|
||||
SERVICE_ELEMENT_WEB_BIND_PORT_HTTP=127.0.0.1:42025
|
||||
|
||||
SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API=127.0.0.1:42030
|
||||
|
||||
SERVICE_OLLAMA_BIND_PORT_HTTP=127.0.0.1:42026
|
||||
|
||||
# See https://localai.io/basics/container/#all-in-one-images for the list of available images
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
ollama:
|
||||
image: docker.io/ollama/ollama:0.6.3
|
||||
image: docker.io/ollama/ollama:0.18.2
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/postgres:16.8-alpine
|
||||
image: docker.io/postgres:18.3-alpine
|
||||
user: ${UID}:${GID}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
@@ -8,12 +8,13 @@ services:
|
||||
POSTGRES_PASSWORD: synapse-password
|
||||
POSTGRES_DB: homeserver
|
||||
POSTGRES_INITDB_ARGS: --lc-collate C --lc-ctype C --encoding UTF8
|
||||
PGDATA: /data
|
||||
volumes:
|
||||
- ./postgres:/var/lib/postgresql/data
|
||||
- ./postgres:/data
|
||||
- /etc/passwd:/etc/passwd:ro
|
||||
|
||||
synapse:
|
||||
image: ghcr.io/element-hq/synapse:v1.127.1
|
||||
image: ghcr.io/element-hq/synapse:v1.150.0
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
entrypoint: python
|
||||
@@ -22,25 +23,9 @@ services:
|
||||
- "${SERVICE_SYNAPSE_BIND_PORT_CLIENT_API}:8008"
|
||||
- "${SERVICE_SYNAPSE_BIND_PORT_FEDERATION_API}:8008"
|
||||
volumes:
|
||||
- ../../etc/services/core/synapse/config:/config:ro
|
||||
- ../../etc/services/synapse/config:/config:ro
|
||||
- ./synapse/media-store:/media-store
|
||||
|
||||
element-web:
|
||||
image: ghcr.io/element-hq/element-web:v1.11.96
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
ELEMENT_WEB_PORT: 8080
|
||||
ports:
|
||||
- "${SERVICE_ELEMENT_WEB_BIND_PORT_HTTP}:8080"
|
||||
volumes:
|
||||
- ../../etc/services/core/element-web/config.json:/app/config.json:ro
|
||||
tmpfs:
|
||||
- /var/cache/nginx:rw,mode=777
|
||||
- /var/run:rw,mode=777
|
||||
- /tmp/element-web-config:rw,mode=777
|
||||
- /etc/nginx/conf.d:rw,mode=777
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ${NETWORK_NAME}
|
||||
214
justfile
214
justfile
@@ -2,10 +2,33 @@ project_name := "baibot"
|
||||
container_image_name := "localhost/baibot"
|
||||
project_container_network := "baibot"
|
||||
|
||||
admin_username := "admin"
|
||||
admin_password := "admin"
|
||||
bot_username := "baibot"
|
||||
bot_password := "baibot"
|
||||
|
||||
homeserver := `cat var/homeserver 2>/dev/null || echo continuwuity`
|
||||
|
||||
mise_data_dir := env("MISE_DATA_DIR", justfile_directory() / "var/mise")
|
||||
mise_trusted_config_paths := justfile_directory() / "mise.toml"
|
||||
|
||||
# Show help by default
|
||||
default:
|
||||
@just --list --justfile {{ justfile() }}
|
||||
|
||||
# Selects which homeserver implementation to use (continuwuity or synapse)
|
||||
homeserver-init value:
|
||||
#!/bin/sh
|
||||
mkdir -p {{ justfile_directory() }}/var
|
||||
echo {{ value }} > {{ justfile_directory() }}/var/homeserver
|
||||
echo ""
|
||||
echo "⚠️ If you had already prepared your app configuration (var/app/local/config.yml or var/app/container/config.yml),"
|
||||
echo " you will need to update it manually or delete it and re-run the prepare step."
|
||||
echo " You should also delete var/app/local/data and/or var/app/container/data,"
|
||||
echo " as old application state is not compatible across homeserver implementations."
|
||||
echo ""
|
||||
echo "⚠️ If Element Web was already prepared, delete var/services/element-web/ to regenerate its config."
|
||||
|
||||
# Builds and runs a development binary
|
||||
run-locally *extra_args: app-local-prepare
|
||||
RUST_BACKTRACE=1 \
|
||||
@@ -65,9 +88,13 @@ docker-compose services_type *extra_args:
|
||||
-p {{ project_name }}-{{ services_type }} \
|
||||
{{ extra_args }}
|
||||
|
||||
# Runs a docker-compose command against the core services
|
||||
docker-compose-core *extra_args:
|
||||
just docker-compose core {{ extra_args }}
|
||||
# Runs a docker-compose command against the synapse services
|
||||
docker-compose-synapse *extra_args:
|
||||
just docker-compose synapse {{ extra_args }}
|
||||
|
||||
# Runs a docker-compose command against the element-web services
|
||||
docker-compose-element-web *extra_args:
|
||||
just docker-compose element-web {{ extra_args }}
|
||||
|
||||
# Runs a docker-compose command against the localai services
|
||||
docker-compose-localai *extra_args:
|
||||
@@ -77,17 +104,52 @@ docker-compose-localai *extra_args:
|
||||
docker-compose-ollama *extra_args:
|
||||
just docker-compose ollama {{ extra_args }}
|
||||
|
||||
# Runs all core dependency components (in the background)
|
||||
services-start: services-prepare (docker-compose-core "up" "-d")
|
||||
# Runs a docker-compose command against the continuwuity services
|
||||
docker-compose-continuwuity *extra_args:
|
||||
just docker-compose continuwuity {{ extra_args }}
|
||||
|
||||
# Stops all core dependency components
|
||||
services-stop: (docker-compose-core "down")
|
||||
# Runs the homeserver and Element Web (in the background)
|
||||
services-start: services-prepare
|
||||
just -f {{ justfile_directory() }}/justfile {{ homeserver }}-start
|
||||
just -f {{ justfile_directory() }}/justfile element-web-start
|
||||
|
||||
# Tails the logs for all running core services
|
||||
services-tail-logs: (docker-compose-core "logs" "-f")
|
||||
# Stops Element Web and the homeserver
|
||||
services-stop:
|
||||
just -f {{ justfile_directory() }}/justfile element-web-stop
|
||||
just -f {{ justfile_directory() }}/justfile {{ homeserver }}-stop
|
||||
|
||||
# Prepares the core services for running
|
||||
services-prepare: _prepare-var-services-env _prepare-var-services-postgres _prepare-var-services-synapse _prepare-container-network
|
||||
# Tails the logs for the homeserver and Element Web
|
||||
services-tail-logs:
|
||||
just -f {{ justfile_directory() }}/justfile {{ homeserver }}-tail-logs
|
||||
|
||||
# Prepares the homeserver and Element Web for running
|
||||
services-prepare:
|
||||
just -f {{ justfile_directory() }}/justfile {{ homeserver }}-prepare
|
||||
just -f {{ justfile_directory() }}/justfile element-web-prepare
|
||||
|
||||
# Runs Synapse (in the background)
|
||||
synapse-start: synapse-prepare (docker-compose-synapse "up" "-d")
|
||||
|
||||
# Stops Synapse
|
||||
synapse-stop: (docker-compose-synapse "down")
|
||||
|
||||
# Tails the logs for Synapse
|
||||
synapse-tail-logs: (docker-compose-synapse "logs" "-f")
|
||||
|
||||
# Prepares Synapse for running
|
||||
synapse-prepare: _prepare-var-services-env _prepare-var-services-postgres _prepare-var-services-synapse _prepare-container-network
|
||||
|
||||
# Runs Element Web (in the background)
|
||||
element-web-start: element-web-prepare (docker-compose-element-web "up" "-d")
|
||||
|
||||
# Stops Element Web
|
||||
element-web-stop: (docker-compose-element-web "down")
|
||||
|
||||
# Tails the logs for Element Web
|
||||
element-web-tail-logs: (docker-compose-element-web "logs" "-f")
|
||||
|
||||
# Prepares Element Web for running
|
||||
element-web-prepare: _prepare-var-services-env _prepare-var-services-element-web _prepare-container-network
|
||||
|
||||
# Runs LocalAI (in the background)
|
||||
localai-start: localai-prepare (docker-compose-localai "up" "-d")
|
||||
@@ -113,6 +175,27 @@ ollama-tail-logs: (docker-compose-ollama "logs" "-f")
|
||||
# Prepares Ollama for running
|
||||
ollama-prepare: _prepare-var-services-env _prepare-var-services-ollama _prepare-container-network
|
||||
|
||||
# Runs Continuwuity (in the background)
|
||||
continuwuity-start: continuwuity-prepare (docker-compose-continuwuity "up" "-d")
|
||||
|
||||
# Stops Continuwuity
|
||||
continuwuity-stop: (docker-compose-continuwuity "down")
|
||||
|
||||
# Tails the logs for Continuwuity
|
||||
continuwuity-tail-logs: (docker-compose-continuwuity "logs" "-f")
|
||||
|
||||
# Prepares Continuwuity for running
|
||||
continuwuity-prepare: _prepare-var-services-env _prepare-var-services-continuwuity _prepare-container-network
|
||||
|
||||
# Registers a user on Continuwuity via the Matrix Client-Server API
|
||||
continuwuity-register-user username password:
|
||||
{{ justfile_directory() }}/etc/services/continuwuity/register-user.sh {{ justfile_directory() }}/var/services/env {{ username }} {{ password }}
|
||||
|
||||
# Prepares the Continuwuity user accounts
|
||||
continuwuity-users-prepare: continuwuity-prepare
|
||||
just -f {{ justfile_directory() }}/justfile continuwuity-register-user "{{ admin_username }}" "{{ admin_password }}"
|
||||
just -f {{ justfile_directory() }}/justfile continuwuity-register-user "{{ bot_username }}" "{{ bot_password }}"
|
||||
|
||||
# Pulls an Ollama model
|
||||
ollama-pull-model model_id:
|
||||
just -f {{ justfile_directory() }}/justfile docker-compose-ollama \
|
||||
@@ -126,16 +209,20 @@ app-local-prepare: _prepare-var-app-local-config_yml _prepare-var-app-local-data
|
||||
app-container-prepare: _prepare-var-app-container-config_yml _prepare-var-app-container-data
|
||||
|
||||
# Prepares the user accounts
|
||||
users-prepare: services-prepare
|
||||
just -f {{ justfile_directory() }}/justfile synapse-register-admin-user "admin" "admin"
|
||||
just -f {{ justfile_directory() }}/justfile synapse-register-regular-user "baibot" "baibot"
|
||||
users-prepare:
|
||||
just -f {{ justfile_directory() }}/justfile {{ homeserver }}-users-prepare
|
||||
|
||||
# Prepares the Synapse user accounts
|
||||
synapse-users-prepare: synapse-prepare
|
||||
just -f {{ justfile_directory() }}/justfile synapse-register-admin-user "{{ admin_username }}" "{{ admin_password }}"
|
||||
just -f {{ justfile_directory() }}/justfile synapse-register-regular-user "{{ bot_username }}" "{{ bot_password }}"
|
||||
|
||||
# Starts a Postgres CLI (psql)
|
||||
postgres-cli: services-prepare (docker-compose-core "exec" "postgres" "/bin/sh" "-c" "'PGUSER=synapse PGPASSWORD=synapse-password PGDATABASE=homeserver psql -h postgres'")
|
||||
postgres-cli: synapse-prepare (docker-compose-synapse "exec" "postgres" "/bin/sh" "-c" "'PGUSER=synapse PGPASSWORD=synapse-password PGDATABASE=homeserver psql -h postgres'")
|
||||
|
||||
# Creates an administrator user
|
||||
synapse-register-admin-user username password: services-prepare
|
||||
just -f {{ justfile_directory() }}/justfile docker-compose-core \
|
||||
# Creates an administrator user on Synapse
|
||||
synapse-register-admin-user username password: synapse-prepare
|
||||
just -f {{ justfile_directory() }}/justfile docker-compose-synapse \
|
||||
exec synapse \
|
||||
register_new_matrix_user \
|
||||
--admin \
|
||||
@@ -144,9 +231,9 @@ synapse-register-admin-user username password: services-prepare
|
||||
-c /config/homeserver.yaml \
|
||||
http://localhost:8008
|
||||
|
||||
# Create a regular user
|
||||
synapse-register-regular-user username password: services-prepare
|
||||
just -f {{ justfile_directory() }}/justfile docker-compose-core \
|
||||
# Creates a regular user on Synapse
|
||||
synapse-register-regular-user username password: synapse-prepare
|
||||
just -f {{ justfile_directory() }}/justfile docker-compose-synapse \
|
||||
exec synapse \
|
||||
register_new_matrix_user \
|
||||
--no-admin \
|
||||
@@ -159,6 +246,44 @@ synapse-register-regular-user username password: services-prepare
|
||||
clippy *extra_args:
|
||||
cargo clippy {{ extra_args }}
|
||||
|
||||
# Checks that the code compiles without building
|
||||
check:
|
||||
cargo check
|
||||
|
||||
# Invokes mise with the project-local data directory
|
||||
mise *args: _ensure_mise_data_directory
|
||||
#!/bin/sh
|
||||
export MISE_DATA_DIR="{{ mise_data_dir }}"
|
||||
export MISE_TRUSTED_CONFIG_PATHS="{{ mise_trusted_config_paths }}"
|
||||
mise {{ args }}
|
||||
|
||||
# Runs prek (pre-commit hooks manager) with the given arguments
|
||||
prek *args: _ensure_mise_tools_installed
|
||||
@just --justfile {{ justfile() }} mise exec -- prek {{ args }}
|
||||
|
||||
# Runs pre-commit hooks on staged files
|
||||
prek-run-on-staged *args: _ensure_mise_tools_installed
|
||||
@just --justfile {{ justfile() }} mise exec -- prek run {{ args }}
|
||||
|
||||
# Runs pre-commit hooks on all files
|
||||
prek-run-on-all *args: _ensure_mise_tools_installed
|
||||
@just --justfile {{ justfile() }} mise exec -- prek run --all-files {{ args }}
|
||||
|
||||
# Installs the git pre-commit hook (runs prek automatically before each commit)
|
||||
prek-install-git-pre-commit-hook: _ensure_mise_tools_installed
|
||||
@just --justfile {{ justfile() }} mise exec -- prek install
|
||||
|
||||
# Internal - ensures var/mise directory exists
|
||||
_ensure_mise_data_directory:
|
||||
#!/bin/sh
|
||||
if [ ! -d "{{ mise_data_dir }}" ]; then
|
||||
mkdir -p "{{ mise_data_dir }}"
|
||||
fi
|
||||
|
||||
# Internal - ensures mise tools are installed
|
||||
_ensure_mise_tools_installed: _ensure_mise_data_directory
|
||||
@just --justfile {{ justfile() }} mise install --quiet
|
||||
|
||||
_prepare-var-services-env:
|
||||
#!/bin/sh
|
||||
cd {{ justfile_directory() }};
|
||||
@@ -188,6 +313,22 @@ _prepare-var-services-synapse:
|
||||
mkdir -p var/services/synapse/media-store
|
||||
fi
|
||||
|
||||
_prepare-var-services-element-web:
|
||||
#!/bin/sh
|
||||
cd {{ justfile_directory() }};
|
||||
|
||||
if [ ! -f var/services/element-web/config.json ]; then
|
||||
mkdir -p var/services/element-web
|
||||
cp {{ justfile_directory() }}/etc/services/element-web/config.json.dist var/services/element-web/config.json
|
||||
|
||||
homeserver="{{ homeserver }}"
|
||||
if [ "$homeserver" = "continuwuity" ]; then
|
||||
sed --in-place 's|__HOMESERVER_CLIENT_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/services/element-web/config.json
|
||||
elif [ "$homeserver" = "synapse" ]; then
|
||||
sed --in-place 's|__HOMESERVER_CLIENT_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/services/element-web/config.json
|
||||
fi
|
||||
fi
|
||||
|
||||
_prepare-var-services-ollama:
|
||||
#!/bin/sh
|
||||
cd {{ justfile_directory() }};
|
||||
@@ -196,6 +337,14 @@ _prepare-var-services-ollama:
|
||||
mkdir -p var/services/ollama
|
||||
fi
|
||||
|
||||
_prepare-var-services-continuwuity:
|
||||
#!/bin/sh
|
||||
cd {{ justfile_directory() }};
|
||||
|
||||
if [ ! -f var/services/continuwuity ]; then
|
||||
mkdir -p var/services/continuwuity/data
|
||||
fi
|
||||
|
||||
_prepare-var-services-localai:
|
||||
#!/bin/sh
|
||||
cd {{ justfile_directory() }};
|
||||
@@ -219,6 +368,15 @@ _prepare-var-app-local-config_yml:
|
||||
if [ ! -f var/app/local/config.yml ]; then
|
||||
mkdir -p var/app/local
|
||||
cp {{ justfile_directory() }}/etc/app/config.yml.dist var/app/local/config.yml
|
||||
|
||||
homeserver="{{ homeserver }}"
|
||||
if [ "$homeserver" = "continuwuity" ]; then
|
||||
sed --in-place 's/__HOMESERVER_SERVER_NAME__/continuwuity.127.0.0.1.nip.io/g' var/app/local/config.yml
|
||||
sed --in-place 's|__HOMESERVER_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/app/local/config.yml
|
||||
elif [ "$homeserver" = "synapse" ]; then
|
||||
sed --in-place 's/__HOMESERVER_SERVER_NAME__/synapse.127.0.0.1.nip.io/g' var/app/local/config.yml
|
||||
sed --in-place 's|__HOMESERVER_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/app/local/config.yml
|
||||
fi
|
||||
fi
|
||||
|
||||
_prepare-var-app-local-data:
|
||||
@@ -236,7 +394,18 @@ _prepare-var-app-container-config_yml:
|
||||
if [ ! -f var/app/container/config.yml ]; then
|
||||
mkdir -p var/app/container
|
||||
cp {{ justfile_directory() }}/etc/app/config.yml.dist var/app/container/config.yml
|
||||
sed --in-place 's/synapse.127.0.0.1.nip.io:42020/synapse:8008/g' var/app/container/config.yml
|
||||
|
||||
homeserver="{{ homeserver }}"
|
||||
if [ "$homeserver" = "continuwuity" ]; then
|
||||
sed --in-place 's/__HOMESERVER_SERVER_NAME__/continuwuity.127.0.0.1.nip.io/g' var/app/container/config.yml
|
||||
sed --in-place 's|__HOMESERVER_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/app/container/config.yml
|
||||
sed --in-place 's/continuwuity.127.0.0.1.nip.io:42030/continuwuity:6167/g' var/app/container/config.yml
|
||||
elif [ "$homeserver" = "synapse" ]; then
|
||||
sed --in-place 's/__HOMESERVER_SERVER_NAME__/synapse.127.0.0.1.nip.io/g' var/app/container/config.yml
|
||||
sed --in-place 's|__HOMESERVER_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/app/container/config.yml
|
||||
sed --in-place 's/synapse.127.0.0.1.nip.io:42020/synapse:8008/g' var/app/container/config.yml
|
||||
fi
|
||||
|
||||
sed --in-place 's/127.0.0.1:42026/ollama:11434/g' var/app/container/config.yml
|
||||
sed --in-place 's/127.0.0.1:42027/localai:8080/g' var/app/container/config.yml
|
||||
fi
|
||||
@@ -248,4 +417,3 @@ _prepare-var-app-container-data:
|
||||
if [ ! -f var/app/container/data ]; then
|
||||
mkdir -p var/app/container/data
|
||||
fi
|
||||
|
||||
|
||||
6
mise.toml
Normal file
6
mise.toml
Normal file
@@ -0,0 +1,6 @@
|
||||
[tools]
|
||||
prek = "0.3.2"
|
||||
|
||||
[settings]
|
||||
# Disable automatic trust prompts - we trust this config
|
||||
yes = true
|
||||
9
renovate.json
Normal file
9
renovate.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:recommended"
|
||||
],
|
||||
"labels": [
|
||||
"dependencies"
|
||||
]
|
||||
}
|
||||
4
rust-toolchain.toml
Normal file
4
rust-toolchain.toml
Normal file
@@ -0,0 +1,4 @@
|
||||
[toolchain]
|
||||
channel = "1.93.0"
|
||||
components = ["rustfmt", "clippy"]
|
||||
profile = "default"
|
||||
@@ -33,11 +33,11 @@ pub struct AgentDefinition {
|
||||
)]
|
||||
pub provider: AgentProvider,
|
||||
|
||||
pub config: serde_yaml::Value,
|
||||
pub config: serde_yaml_ng::Value,
|
||||
}
|
||||
|
||||
impl AgentDefinition {
|
||||
pub fn new(id: String, provider: AgentProvider, config: serde_yaml::Value) -> Self {
|
||||
pub fn new(id: String, provider: AgentProvider, config: serde_yaml_ng::Value) -> Self {
|
||||
Self {
|
||||
id,
|
||||
provider,
|
||||
|
||||
@@ -15,7 +15,7 @@ pub enum Error {
|
||||
// Contains the error from the constructor function
|
||||
ConstructionFailed(anyhow::Error),
|
||||
// Contains the error from the YAML deserialization function
|
||||
Yaml(serde_yaml::Error),
|
||||
Yaml(serde_yaml_ng::Error),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
@@ -69,7 +69,7 @@ pub(super) fn create(
|
||||
pub fn create_from_provider_and_yaml_value_config(
|
||||
provider: &AgentProvider,
|
||||
identifier: &PublicIdentifier,
|
||||
config: serde_yaml::Value,
|
||||
config: serde_yaml_ng::Value,
|
||||
) -> Result<AgentInstance> {
|
||||
let definition = AgentDefinition::new(identifier.prefixless(), provider.to_owned(), config);
|
||||
|
||||
@@ -79,7 +79,7 @@ pub fn create_from_provider_and_yaml_value_config(
|
||||
fn create_controller_from_provider_and_json_value_config(
|
||||
agent_id: &str,
|
||||
provider: &AgentProvider,
|
||||
config: serde_yaml::Value,
|
||||
config: serde_yaml_ng::Value,
|
||||
) -> Result<ControllerType> {
|
||||
match provider {
|
||||
AgentProvider::Anthropic => {
|
||||
@@ -112,43 +112,43 @@ fn create_controller_from_provider_and_json_value_config(
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_config_for_provider(provider: &AgentProvider) -> serde_yaml::Value {
|
||||
pub fn default_config_for_provider(provider: &AgentProvider) -> serde_yaml_ng::Value {
|
||||
match provider {
|
||||
AgentProvider::Anthropic => {
|
||||
let config = super::provider::anthropic::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::Groq => {
|
||||
let config = super::provider::groq::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::LocalAI => {
|
||||
let config = super::provider::localai::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::Mistral => {
|
||||
let config = super::provider::mistral::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::Ollama => {
|
||||
let config = super::provider::ollama::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::OpenAI => {
|
||||
let config = super::provider::openai::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::OpenAICompat => {
|
||||
let config = super::provider::openai_compat::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::OpenRouter => {
|
||||
let config = super::provider::openrouter::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
AgentProvider::TogetherAI => {
|
||||
let config = super::provider::togetherai::default_config();
|
||||
serde_yaml::to_value(config).expect("Failed to serialize config")
|
||||
serde_yaml_ng::to_value(config).expect("Failed to serialize config")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,13 +7,15 @@ use anthropic::types::ContentBlock;
|
||||
use super::super::ControllerTrait;
|
||||
use crate::agent::AgentPurpose;
|
||||
use crate::agent::provider::entity::{
|
||||
ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
|
||||
TextToSpeechParams, TextToSpeechResult,
|
||||
ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams,
|
||||
TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
|
||||
};
|
||||
use crate::agent::provider::{
|
||||
ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
|
||||
};
|
||||
use crate::agent::provider::{ImageGenerationParams, SpeechToTextParams, SpeechToTextResult};
|
||||
use crate::conversation::llm::{
|
||||
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
|
||||
shorten_messages_list_to_context_size,
|
||||
MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
|
||||
};
|
||||
use crate::strings;
|
||||
|
||||
@@ -69,7 +71,8 @@ impl ControllerTrait for Controller {
|
||||
|
||||
let messages = vec![LLMMessage {
|
||||
author: LLMAuthor::User,
|
||||
message_text: "Hello!".to_string(),
|
||||
sender_id: None,
|
||||
content: LLMMessageContent::Text("Hello!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
}];
|
||||
|
||||
@@ -106,7 +109,8 @@ impl ControllerTrait for Controller {
|
||||
} else {
|
||||
Some(LLMMessage {
|
||||
author: LLMAuthor::Prompt,
|
||||
message_text: prompt_text,
|
||||
sender_id: None,
|
||||
content: LLMMessageContent::Text(prompt_text),
|
||||
timestamp: chrono::Utc::now(),
|
||||
})
|
||||
};
|
||||
@@ -144,8 +148,10 @@ impl ControllerTrait for Controller {
|
||||
.temperature_override
|
||||
.unwrap_or(text_generation_config.temperature);
|
||||
|
||||
if let Some(prompt_message) = prompt_message {
|
||||
request.system = prompt_message.message_text;
|
||||
if let Some(prompt_message) = prompt_message
|
||||
&& let LLMMessageContent::Text(text) = &prompt_message.content
|
||||
{
|
||||
request.system = text.clone();
|
||||
}
|
||||
|
||||
request.model = text_generation_config.model_id.clone();
|
||||
@@ -172,15 +178,8 @@ impl ControllerTrait for Controller {
|
||||
ContentBlock::Text { text } => {
|
||||
text_parts.push(text);
|
||||
}
|
||||
ContentBlock::Image {
|
||||
source,
|
||||
media_type,
|
||||
data: _,
|
||||
} => {
|
||||
text_parts.push(format!(
|
||||
"The model responded with an image of type {}: {}",
|
||||
media_type, source
|
||||
));
|
||||
ContentBlock::Image { .. } => {
|
||||
text_parts.push("The model responded with an image".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -213,6 +212,15 @@ impl ControllerTrait for Controller {
|
||||
Err(anyhow::anyhow!("Image generation not supported"))
|
||||
}
|
||||
|
||||
async fn create_image_edit(
|
||||
&self,
|
||||
_prompt: &str,
|
||||
_images: Vec<ImageSource>,
|
||||
_params: ImageEditParams,
|
||||
) -> anyhow::Result<ImageEditResult> {
|
||||
Err(anyhow::anyhow!("Image editing is not supported"))
|
||||
}
|
||||
|
||||
async fn text_to_speech(
|
||||
&self,
|
||||
_input: &str,
|
||||
|
||||
@@ -12,12 +12,12 @@ use super::controller::ControllerType;
|
||||
|
||||
pub fn create_controller_from_yaml_value_config(
|
||||
agent_id: &str,
|
||||
config: serde_yaml::Value,
|
||||
config: serde_yaml_ng::Value,
|
||||
) -> AgentInstantiationResult<ControllerType> {
|
||||
let config = match &config {
|
||||
serde_yaml::Value::Mapping(_) => {
|
||||
serde_yaml_ng::Value::Mapping(_) => {
|
||||
let config: Config =
|
||||
serde_yaml::from_value(config).map_err(AgentInstantiationError::Yaml)?;
|
||||
serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?;
|
||||
|
||||
config
|
||||
.validate()
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use anthropic::types::{ContentBlock, Message, MessagesRequest, MessagesRequestBuilder, Role};
|
||||
use anthropic::types::{
|
||||
ContentBlock, ImageSource, Message, MessagesRequest, MessagesRequestBuilder, Role,
|
||||
};
|
||||
|
||||
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
|
||||
use crate::conversation::llm::{
|
||||
Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
|
||||
};
|
||||
|
||||
pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessagesRequest {
|
||||
let mut messages = vec![];
|
||||
@@ -14,9 +18,24 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
|
||||
}
|
||||
};
|
||||
|
||||
let content = vec![ContentBlock::Text {
|
||||
text: message.message_text,
|
||||
}];
|
||||
let content = match &message.content {
|
||||
LLMMessageContent::Text(text) => vec![ContentBlock::Text { text: text.clone() }],
|
||||
LLMMessageContent::Image(image_details) => {
|
||||
vec![ContentBlock::Image {
|
||||
source: ImageSource::Base64 {
|
||||
media_type: image_details.mime.to_string(),
|
||||
data: crate::utils::base64::base64_encode(&image_details.data),
|
||||
},
|
||||
}]
|
||||
}
|
||||
LLMMessageContent::File(file_details) => {
|
||||
tracing::warn!(
|
||||
"The Anthropic provider's library does not support file/document content. This file message ({}) will be skipped.",
|
||||
file_details.filename(),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let message = Message { role, content };
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use crate::{agent::AgentPurpose, conversation::llm::Conversation};
|
||||
|
||||
use super::{
|
||||
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
|
||||
ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
|
||||
entity::{
|
||||
ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
|
||||
TextToSpeechParams, TextToSpeechResult,
|
||||
ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams,
|
||||
TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -42,6 +42,13 @@ pub trait ControllerTrait {
|
||||
params: ImageGenerationParams,
|
||||
) -> impl std::future::Future<Output = anyhow::Result<ImageGenerationResult>> + Send;
|
||||
|
||||
fn create_image_edit(
|
||||
&self,
|
||||
prompt: &str,
|
||||
images: Vec<ImageSource>,
|
||||
params: ImageEditParams,
|
||||
) -> impl std::future::Future<Output = anyhow::Result<ImageEditResult>> + Send;
|
||||
|
||||
fn text_to_speech(
|
||||
&self,
|
||||
text: &str,
|
||||
@@ -166,6 +173,25 @@ impl ControllerTrait for ControllerType {
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_image_edit(
|
||||
&self,
|
||||
prompt: &str,
|
||||
images: Vec<ImageSource>,
|
||||
params: ImageEditParams,
|
||||
) -> anyhow::Result<ImageEditResult> {
|
||||
match &self {
|
||||
ControllerType::OpenAI(controller) => {
|
||||
controller.create_image_edit(prompt, images, params).await
|
||||
}
|
||||
ControllerType::OpenAICompat(controller) => {
|
||||
controller.create_image_edit(prompt, images, params).await
|
||||
}
|
||||
ControllerType::Anthropic(controller) => {
|
||||
controller.create_image_edit(prompt, images, params).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn text_to_speech(
|
||||
&self,
|
||||
text: &str,
|
||||
|
||||
@@ -68,6 +68,8 @@ impl AgentProvider {
|
||||
sign_up_url: Some("https://console.anthropic.com/"),
|
||||
models_list_url: Some("https://docs.anthropic.com/en/docs/about-claude/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: true,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::Groq => AgentProviderInfo {
|
||||
id: Self::Groq.to_static_str(),
|
||||
@@ -78,11 +80,13 @@ impl AgentProvider {
|
||||
sign_up_url: Some("https://console.groq.com/login"),
|
||||
models_list_url: Some("https://console.groq.com/docs/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration, AgentPurpose::SpeechToText],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::LocalAI => AgentProviderInfo {
|
||||
id: Self::LocalAI.to_static_str(),
|
||||
name: "LocalAI",
|
||||
description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that’s compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.",
|
||||
description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that's compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.",
|
||||
homepage_url: Some("https://localai.io/"),
|
||||
wiki_url: None,
|
||||
sign_up_url: None,
|
||||
@@ -92,6 +96,8 @@ impl AgentProvider {
|
||||
AgentPurpose::TextToSpeech,
|
||||
AgentPurpose::SpeechToText,
|
||||
],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::Mistral => AgentProviderInfo {
|
||||
id: Self::Mistral.to_static_str(),
|
||||
@@ -102,6 +108,8 @@ impl AgentProvider {
|
||||
sign_up_url: Some("https://auth.mistral.ai/ui/registration"),
|
||||
models_list_url: Some("https://docs.mistral.ai/getting-started/models/"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::Ollama => AgentProviderInfo {
|
||||
id: Self::Ollama.to_static_str(),
|
||||
@@ -112,6 +120,8 @@ impl AgentProvider {
|
||||
sign_up_url: None,
|
||||
models_list_url: Some("https://ollama.com/library"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::OpenAI => AgentProviderInfo {
|
||||
id: Self::OpenAI.to_static_str(),
|
||||
@@ -127,6 +137,8 @@ impl AgentProvider {
|
||||
AgentPurpose::TextToSpeech,
|
||||
AgentPurpose::SpeechToText,
|
||||
],
|
||||
text_generation_supports_vision: true,
|
||||
text_generation_supports_tools: true,
|
||||
},
|
||||
Self::OpenAICompat => AgentProviderInfo {
|
||||
id: Self::OpenAICompat.to_static_str(),
|
||||
@@ -142,6 +154,8 @@ impl AgentProvider {
|
||||
AgentPurpose::TextToSpeech,
|
||||
AgentPurpose::SpeechToText,
|
||||
],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::OpenRouter => AgentProviderInfo {
|
||||
id: Self::OpenRouter.to_static_str(),
|
||||
@@ -152,6 +166,8 @@ impl AgentProvider {
|
||||
sign_up_url: Some("https://openrouter.ai/"),
|
||||
models_list_url: Some("https://openrouter.ai/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::TogetherAI => AgentProviderInfo {
|
||||
id: Self::TogetherAI.to_static_str(),
|
||||
@@ -162,6 +178,8 @@ impl AgentProvider {
|
||||
sign_up_url: Some("https://api.together.ai/signup"),
|
||||
models_list_url: Some("https://api.together.xyz/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -182,4 +200,6 @@ pub struct AgentProviderInfo {
|
||||
pub sign_up_url: Option<&'static str>,
|
||||
pub models_list_url: Option<&'static str>,
|
||||
pub supported_purposes: Vec<AgentPurpose>,
|
||||
pub text_generation_supports_vision: bool,
|
||||
pub text_generation_supports_tools: bool,
|
||||
}
|
||||
|
||||
63
src/agent/provider/entity/image.rs
Normal file
63
src/agent/provider/entity/image.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
use mxlink::mime;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ImageGenerationParams {
|
||||
pub smallest_size_possible: bool,
|
||||
|
||||
pub cheaper_model_switching_allowed: bool,
|
||||
|
||||
pub cheaper_quality_switching_allowed: bool,
|
||||
}
|
||||
|
||||
impl ImageGenerationParams {
|
||||
pub fn with_smallest_size_possible(mut self, value: bool) -> Self {
|
||||
self.smallest_size_possible = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_model_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_quality_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ImageGenerationResult {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mime::Mime,
|
||||
pub revised_prompt: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ImageEditParams {}
|
||||
|
||||
pub struct ImageEditResult {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mime::Mime,
|
||||
}
|
||||
|
||||
pub struct ImageSource {
|
||||
pub filename: String,
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mime::Mime,
|
||||
}
|
||||
|
||||
impl ImageSource {
|
||||
pub fn new(filename: String, bytes: Vec<u8>, mime_type: mime::Mime) -> Self {
|
||||
Self {
|
||||
filename,
|
||||
bytes,
|
||||
mime_type,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ImageSource> for async_openai::types::images::ImageInput {
|
||||
fn from(value: ImageSource) -> Self {
|
||||
async_openai::types::images::ImageInput::from_vec_u8(value.filename, value.bytes)
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
#[derive(Default)]
|
||||
pub struct ImageGenerationParams {
|
||||
pub size_override: Option<String>,
|
||||
|
||||
pub cheaper_model_switching_allowed: bool,
|
||||
|
||||
pub cheaper_quality_switching_allowed: bool,
|
||||
}
|
||||
|
||||
impl ImageGenerationParams {
|
||||
pub fn with_size_override(mut self, value: Option<String>) -> Self {
|
||||
self.size_override = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_model_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_quality_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ImageGenerationResult {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mxlink::mime::Mime,
|
||||
pub revised_prompt: Option<String>,
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
mod agent_provider;
|
||||
mod image_generation;
|
||||
mod image;
|
||||
mod ping;
|
||||
mod speech_to_text;
|
||||
mod text_generation;
|
||||
mod text_to_speech;
|
||||
|
||||
pub use agent_provider::{AgentProvider, AgentProviderInfo};
|
||||
pub use image_generation::{ImageGenerationParams, ImageGenerationResult};
|
||||
pub use image::{
|
||||
ImageEditParams, ImageEditResult, ImageGenerationParams, ImageGenerationResult, ImageSource,
|
||||
};
|
||||
pub use ping::PingResult;
|
||||
pub use speech_to_text::{SpeechToTextParams, SpeechToTextResult};
|
||||
pub use text_generation::{
|
||||
|
||||
@@ -20,6 +20,7 @@ pub use controller::{ControllerTrait, ControllerType};
|
||||
pub use config::ConfigTrait;
|
||||
|
||||
pub use entity::{
|
||||
AgentProvider, AgentProviderInfo, ImageGenerationParams, PingResult, SpeechToTextParams,
|
||||
SpeechToTextResult, TextGenerationParams, TextGenerationPromptVariables, TextToSpeechParams,
|
||||
AgentProvider, AgentProviderInfo, ImageEditParams, ImageGenerationParams, ImageSource,
|
||||
PingResult, SpeechToTextParams, SpeechToTextResult, TextGenerationParams,
|
||||
TextGenerationPromptVariables, TextToSpeechParams,
|
||||
};
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5;
|
||||
use crate::agent::{default_prompt, provider::ConfigTrait};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -63,6 +64,9 @@ pub struct TextGenerationConfig {
|
||||
|
||||
#[serde(default)]
|
||||
pub max_context_tokens: u32,
|
||||
|
||||
#[serde(default)]
|
||||
pub tools: ToolsConfig,
|
||||
}
|
||||
|
||||
impl Default for TextGenerationConfig {
|
||||
@@ -71,15 +75,25 @@ impl Default for TextGenerationConfig {
|
||||
model_id: default_text_model_id(),
|
||||
prompt: Some(default_prompt().to_owned()),
|
||||
temperature: super::super::default_temperature(),
|
||||
max_response_tokens: Some(16_384),
|
||||
max_completion_tokens: None,
|
||||
max_context_tokens: 128_000,
|
||||
max_response_tokens: None,
|
||||
max_completion_tokens: Some(128_000),
|
||||
max_context_tokens: 400_000,
|
||||
tools: ToolsConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_text_model_id() -> String {
|
||||
"gpt-4o".to_owned()
|
||||
"gpt-5.4".to_owned()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ToolsConfig {
|
||||
#[serde(default)]
|
||||
pub web_search: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub code_interpreter: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -103,16 +117,16 @@ fn default_speech_to_text_model_id() -> String {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TextToSpeechConfig {
|
||||
#[serde(default = "default_text_to_speech_model_id")]
|
||||
pub model_id: async_openai::types::SpeechModel,
|
||||
pub model_id: async_openai::types::audio::SpeechModel,
|
||||
|
||||
#[serde(default = "default_text_to_speech_voice")]
|
||||
pub voice: async_openai::types::Voice,
|
||||
pub voice: async_openai::types::audio::Voice,
|
||||
|
||||
#[serde(default = "default_text_to_speech_speed")]
|
||||
pub speed: f32,
|
||||
|
||||
#[serde(default = "default_text_to_speech_response_format")]
|
||||
pub response_format: async_openai::types::SpeechResponseFormat,
|
||||
pub response_format: async_openai::types::audio::SpeechResponseFormat,
|
||||
}
|
||||
|
||||
impl Default for TextToSpeechConfig {
|
||||
@@ -126,22 +140,22 @@ impl Default for TextToSpeechConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn default_text_to_speech_model_id() -> async_openai::types::SpeechModel {
|
||||
async_openai::types::SpeechModel::Tts1Hd
|
||||
fn default_text_to_speech_model_id() -> async_openai::types::audio::SpeechModel {
|
||||
async_openai::types::audio::SpeechModel::Tts1Hd
|
||||
}
|
||||
|
||||
fn default_text_to_speech_voice() -> async_openai::types::Voice {
|
||||
async_openai::types::Voice::Onyx
|
||||
fn default_text_to_speech_voice() -> async_openai::types::audio::Voice {
|
||||
async_openai::types::audio::Voice::Onyx
|
||||
}
|
||||
|
||||
fn default_text_to_speech_speed() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_text_to_speech_response_format() -> async_openai::types::SpeechResponseFormat {
|
||||
fn default_text_to_speech_response_format() -> async_openai::types::audio::SpeechResponseFormat {
|
||||
// The API defaults to mp3, but we prefer Opus because it's smaller.
|
||||
// Our clients should all have support for it.
|
||||
async_openai::types::SpeechResponseFormat::Opus
|
||||
async_openai::types::audio::SpeechResponseFormat::Opus
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -149,19 +163,19 @@ pub struct ImageGenerationConfig {
|
||||
pub model_id: String,
|
||||
|
||||
#[serde(default = "default_image_style")]
|
||||
pub style: async_openai::types::ImageStyle,
|
||||
pub style: Option<async_openai::types::images::ImageStyle>,
|
||||
|
||||
#[serde(default = "default_image_size")]
|
||||
pub size: async_openai::types::ImageSize,
|
||||
pub size: Option<async_openai::types::images::ImageSize>,
|
||||
|
||||
#[serde(default = "default_image_quality")]
|
||||
pub quality: async_openai::types::ImageQuality,
|
||||
pub quality: Option<async_openai::types::images::ImageQuality>,
|
||||
}
|
||||
|
||||
impl Default for ImageGenerationConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
model_id: "dall-e-3".to_owned(),
|
||||
model_id: OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5.to_owned(),
|
||||
style: default_image_style(),
|
||||
size: default_image_size(),
|
||||
quality: default_image_quality(),
|
||||
@@ -172,23 +186,28 @@ impl Default for ImageGenerationConfig {
|
||||
impl ImageGenerationConfig {
|
||||
pub fn model_id_as_openai_image_model(
|
||||
&self,
|
||||
) -> Result<async_openai::types::ImageModel, String> {
|
||||
) -> Result<async_openai::types::images::ImageModel, String> {
|
||||
match self.model_id.as_str() {
|
||||
"dall-e-2" => Ok(async_openai::types::ImageModel::DallE2),
|
||||
"dall-e-3" => Ok(async_openai::types::ImageModel::DallE3),
|
||||
other => Ok(async_openai::types::ImageModel::Other(other.to_owned())),
|
||||
"dall-e-2" => Ok(async_openai::types::images::ImageModel::DallE2),
|
||||
"dall-e-3" => Ok(async_openai::types::images::ImageModel::DallE3),
|
||||
"gpt-image-1" => Ok(async_openai::types::images::ImageModel::GptImage1),
|
||||
"gpt-image-1.5" => Ok(async_openai::types::images::ImageModel::GptImage1dot5),
|
||||
"gpt-image-1-mini" => Ok(async_openai::types::images::ImageModel::GptImage1Mini),
|
||||
other => Ok(async_openai::types::images::ImageModel::Other(
|
||||
other.to_owned(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_image_style() -> async_openai::types::ImageStyle {
|
||||
async_openai::types::ImageStyle::Vivid
|
||||
fn default_image_style() -> Option<async_openai::types::images::ImageStyle> {
|
||||
None
|
||||
}
|
||||
|
||||
fn default_image_size() -> async_openai::types::ImageSize {
|
||||
async_openai::types::ImageSize::S1024x1024
|
||||
fn default_image_size() -> Option<async_openai::types::images::ImageSize> {
|
||||
None
|
||||
}
|
||||
|
||||
fn default_image_quality() -> async_openai::types::ImageQuality {
|
||||
async_openai::types::ImageQuality::Standard
|
||||
fn default_image_quality() -> Option<async_openai::types::images::ImageQuality> {
|
||||
None
|
||||
}
|
||||
|
||||
@@ -4,34 +4,39 @@ use async_openai::{
|
||||
Client as OpenAIClient,
|
||||
config::OpenAIConfig,
|
||||
types::{
|
||||
ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageRequestArgs,
|
||||
CreateSpeechRequestArgs, CreateTranscriptionRequestArgs,
|
||||
audio::{AudioInput, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs},
|
||||
images::{
|
||||
CreateImageEditRequestArgs, CreateImageRequestArgs, Image, ImageInput, ImageModel,
|
||||
ImageResponseFormat,
|
||||
},
|
||||
responses::{
|
||||
CodeInterpreterContainerAuto, CodeInterpreterTool, CodeInterpreterToolContainer,
|
||||
CreateResponseArgs, OutputItem, OutputMessageContent, Tool, WebSearchTool,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
use super::super::ControllerTrait;
|
||||
use crate::{
|
||||
agent::{
|
||||
AgentPurpose,
|
||||
provider::{
|
||||
entity::{ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult},
|
||||
openai::utils::convert_string_to_enum,
|
||||
},
|
||||
},
|
||||
strings,
|
||||
};
|
||||
use crate::{
|
||||
agent::{
|
||||
provider::{
|
||||
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
|
||||
entity::{TextGenerationParams, TextGenerationResult},
|
||||
},
|
||||
utils::base64_decode,
|
||||
agent::provider::{
|
||||
ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
|
||||
entity::{TextGenerationParams, TextGenerationResult},
|
||||
},
|
||||
conversation::llm::{
|
||||
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
|
||||
shorten_messages_list_to_context_size,
|
||||
MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
|
||||
},
|
||||
utils::base64::base64_decode,
|
||||
};
|
||||
use crate::{
|
||||
agent::{
|
||||
AgentPurpose,
|
||||
provider::entity::{
|
||||
ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextToSpeechParams,
|
||||
TextToSpeechResult,
|
||||
},
|
||||
},
|
||||
strings,
|
||||
};
|
||||
|
||||
use super::config::Config;
|
||||
@@ -62,7 +67,8 @@ impl ControllerTrait for Controller {
|
||||
|
||||
let messages = vec![LLMMessage {
|
||||
author: LLMAuthor::User,
|
||||
message_text: "Hello!".to_string(),
|
||||
sender_id: None,
|
||||
content: LLMMessageContent::Text("Hello!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
}];
|
||||
|
||||
@@ -99,7 +105,8 @@ impl ControllerTrait for Controller {
|
||||
} else {
|
||||
Some(LLMMessage {
|
||||
author: LLMAuthor::Prompt,
|
||||
message_text: prompt_text,
|
||||
sender_id: None,
|
||||
content: LLMMessageContent::Text(prompt_text),
|
||||
timestamp: chrono::Utc::now(),
|
||||
})
|
||||
};
|
||||
@@ -124,28 +131,45 @@ impl ControllerTrait for Controller {
|
||||
conversation_messages.insert(0, prompt_message);
|
||||
}
|
||||
|
||||
let openai_conversation_messages: Vec<ChatCompletionRequestMessage> =
|
||||
super::utils::convert_llm_messages_to_openai_messages(conversation_messages);
|
||||
let input =
|
||||
super::utils::convert_llm_messages_to_openai_response_input(conversation_messages);
|
||||
|
||||
let messages_count = openai_conversation_messages.len();
|
||||
let messages_count = match &input {
|
||||
async_openai::types::responses::InputParam::Items(items) => items.len(),
|
||||
_ => 1,
|
||||
};
|
||||
|
||||
let temperature = params
|
||||
.temperature_override
|
||||
.unwrap_or(text_generation_config.temperature);
|
||||
|
||||
let mut request_builder = CreateChatCompletionRequestArgs::default();
|
||||
let mut request_builder = CreateResponseArgs::default();
|
||||
|
||||
request_builder
|
||||
.model(&text_generation_config.model_id)
|
||||
.temperature(temperature)
|
||||
.messages(openai_conversation_messages);
|
||||
.input(input);
|
||||
|
||||
if let Some(max_response_tokens) = text_generation_config.max_response_tokens {
|
||||
request_builder.max_tokens(max_response_tokens);
|
||||
let mut tools = Vec::new();
|
||||
if text_generation_config.tools.web_search {
|
||||
tools.push(Tool::WebSearch(WebSearchTool::default()));
|
||||
}
|
||||
if text_generation_config.tools.code_interpreter {
|
||||
tools.push(Tool::CodeInterpreter(CodeInterpreterTool {
|
||||
container: CodeInterpreterToolContainer::Auto(
|
||||
CodeInterpreterContainerAuto::default(),
|
||||
),
|
||||
}));
|
||||
}
|
||||
|
||||
if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens {
|
||||
request_builder.max_completion_tokens(max_completion_tokens);
|
||||
if !tools.is_empty() {
|
||||
request_builder.tools(tools);
|
||||
}
|
||||
|
||||
if let Some(max_response_tokens) = text_generation_config.max_response_tokens {
|
||||
request_builder.max_output_tokens(max_response_tokens);
|
||||
} else if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens {
|
||||
request_builder.max_output_tokens(max_completion_tokens);
|
||||
}
|
||||
|
||||
let request = request_builder.build()?;
|
||||
@@ -155,33 +179,28 @@ impl ControllerTrait for Controller {
|
||||
model = format!("{:?}", request.model),
|
||||
?messages_count,
|
||||
request = request_as_json,
|
||||
"Sending OpenAI chat completion API request"
|
||||
"Sending OpenAI response API request"
|
||||
);
|
||||
}
|
||||
|
||||
let response = self.client.chat().create(request).await?;
|
||||
let response = self.client.responses().create(request).await?;
|
||||
|
||||
tracing::trace!(
|
||||
?response,
|
||||
"Got response from the OpenAI chat completion API"
|
||||
);
|
||||
tracing::trace!(?response, "Got response from the OpenAI response API");
|
||||
|
||||
// We only request 1 result, so there should only be 1 choice.
|
||||
if let Some(choice) = response.choices.into_iter().next() {
|
||||
match choice.message.content {
|
||||
Some(text) => {
|
||||
return Ok(TextGenerationResult { text });
|
||||
}
|
||||
None => {
|
||||
return Err(anyhow::anyhow!(
|
||||
"No content was found in the response choice from the OpenAI chat completion API"
|
||||
));
|
||||
for item in response.output {
|
||||
if let OutputItem::Message(message) = item {
|
||||
for content in message.content {
|
||||
if let OutputMessageContent::OutputText(text_content) = content {
|
||||
return Ok(TextGenerationResult {
|
||||
text: text_content.text,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow::anyhow!(
|
||||
"No response messages choices were returned from the OpenAI chat completion API"
|
||||
"No response messages choices were returned from the OpenAI response API"
|
||||
))
|
||||
}
|
||||
|
||||
@@ -205,12 +224,7 @@ impl ControllerTrait for Controller {
|
||||
|
||||
let request = CreateTranscriptionRequestArgs::default()
|
||||
.model(&speech_to_text_config.model_id)
|
||||
.file(async_openai::types::AudioInput {
|
||||
source: async_openai::types::InputSource::VecU8 {
|
||||
filename,
|
||||
vec: media,
|
||||
},
|
||||
})
|
||||
.file(AudioInput::from_vec_u8(filename, media))
|
||||
.language(language.clone())
|
||||
.build()?;
|
||||
|
||||
@@ -220,7 +234,7 @@ impl ControllerTrait for Controller {
|
||||
"Sending OpenAI speech-to-text API request"
|
||||
);
|
||||
|
||||
let response = self.client.audio().transcribe(request).await?;
|
||||
let response = self.client.audio().transcription().create(request).await?;
|
||||
|
||||
tracing::trace!(
|
||||
?response,
|
||||
@@ -252,11 +266,12 @@ impl ControllerTrait for Controller {
|
||||
let model = if params.cheaper_model_switching_allowed {
|
||||
// Switch to a cheaper model
|
||||
match original_model {
|
||||
async_openai::types::ImageModel::DallE2 => async_openai::types::ImageModel::DallE2,
|
||||
async_openai::types::ImageModel::DallE3 => async_openai::types::ImageModel::DallE2,
|
||||
async_openai::types::ImageModel::Other(_) => {
|
||||
async_openai::types::ImageModel::DallE2
|
||||
}
|
||||
ImageModel::DallE2 => ImageModel::DallE2,
|
||||
ImageModel::DallE3 => ImageModel::DallE2,
|
||||
ImageModel::GptImage1 => ImageModel::GptImage1Mini,
|
||||
ImageModel::GptImage1dot5 => ImageModel::GptImage1Mini,
|
||||
ImageModel::GptImage1Mini => ImageModel::GptImage1Mini,
|
||||
ImageModel::Other(_) => ImageModel::DallE2,
|
||||
}
|
||||
} else {
|
||||
original_model
|
||||
@@ -265,33 +280,71 @@ impl ControllerTrait for Controller {
|
||||
let quality = if params.cheaper_quality_switching_allowed {
|
||||
// Switch to a cheaper quality
|
||||
match &image_generation_config.quality {
|
||||
async_openai::types::ImageQuality::Standard => {
|
||||
async_openai::types::ImageQuality::Standard
|
||||
}
|
||||
async_openai::types::ImageQuality::HD => {
|
||||
async_openai::types::ImageQuality::Standard
|
||||
}
|
||||
Some(quality) => match quality {
|
||||
async_openai::types::images::ImageQuality::Standard => {
|
||||
Some(async_openai::types::images::ImageQuality::Standard)
|
||||
}
|
||||
async_openai::types::images::ImageQuality::HD => {
|
||||
Some(async_openai::types::images::ImageQuality::Standard)
|
||||
}
|
||||
// New quality levels - keep as-is or downgrade to Standard
|
||||
async_openai::types::images::ImageQuality::High => {
|
||||
Some(async_openai::types::images::ImageQuality::Standard)
|
||||
}
|
||||
async_openai::types::images::ImageQuality::Medium => {
|
||||
Some(async_openai::types::images::ImageQuality::Medium)
|
||||
}
|
||||
async_openai::types::images::ImageQuality::Low => {
|
||||
Some(async_openai::types::images::ImageQuality::Low)
|
||||
}
|
||||
async_openai::types::images::ImageQuality::Auto => {
|
||||
Some(async_openai::types::images::ImageQuality::Auto)
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
}
|
||||
} else {
|
||||
image_generation_config.quality.clone()
|
||||
};
|
||||
|
||||
let size = params
|
||||
.size_override
|
||||
.map(|s| {
|
||||
convert_string_to_enum::<async_openai::types::ImageSize>(&s)
|
||||
.unwrap_or(image_generation_config.size)
|
||||
})
|
||||
.unwrap_or(image_generation_config.size);
|
||||
let size = if params.smallest_size_possible {
|
||||
Some(get_sticker_size(&model))
|
||||
} else {
|
||||
image_generation_config.size
|
||||
};
|
||||
|
||||
let request = CreateImageRequestArgs::default()
|
||||
.model(model)
|
||||
.prompt(prompt.to_owned())
|
||||
.response_format(async_openai::types::ImageResponseFormat::B64Json)
|
||||
.size(size)
|
||||
.style(image_generation_config.style.clone())
|
||||
.quality(quality)
|
||||
.build()?;
|
||||
let response_format = match model.clone() {
|
||||
ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
|
||||
ImageModel::DallE3 => Some(ImageResponseFormat::B64Json),
|
||||
// gpt-image-1 only outputs base64 and we don't need to specify the response format.
|
||||
// In fact, specifying the response format results in an error.
|
||||
ImageModel::GptImage1 => None,
|
||||
ImageModel::GptImage1Mini => None,
|
||||
ImageModel::GptImage1dot5 => None,
|
||||
ImageModel::Other(_) => Some(ImageResponseFormat::B64Json),
|
||||
};
|
||||
|
||||
let mut request_builder = CreateImageRequestArgs::default();
|
||||
|
||||
request_builder.model(model).prompt(prompt.to_owned());
|
||||
|
||||
if let Some(response_format) = response_format {
|
||||
request_builder.response_format(response_format);
|
||||
}
|
||||
|
||||
if let Some(style) = &image_generation_config.style {
|
||||
request_builder.style(style.clone());
|
||||
}
|
||||
|
||||
if let Some(quality) = quality {
|
||||
request_builder.quality(quality.clone());
|
||||
}
|
||||
|
||||
if let Some(size) = size {
|
||||
request_builder.size(size);
|
||||
}
|
||||
|
||||
let request = request_builder.build()?;
|
||||
|
||||
tracing::trace!(
|
||||
?prompt,
|
||||
@@ -302,15 +355,15 @@ impl ControllerTrait for Controller {
|
||||
"Sending OpenAI image generation API request"
|
||||
);
|
||||
|
||||
let response = self.client.images().create(request).await?;
|
||||
let response = self.client.images().generate(request).await?;
|
||||
|
||||
if let Some(image) = response.data.into_iter().next() {
|
||||
match image.deref() {
|
||||
async_openai::types::Image::B64Json {
|
||||
Image::B64Json {
|
||||
b64_json,
|
||||
revised_prompt,
|
||||
} => {
|
||||
let bytes = base64_decode(b64_json)?;
|
||||
let bytes = base64_decode(b64_json.as_ref())?;
|
||||
|
||||
return Ok(ImageGenerationResult {
|
||||
bytes,
|
||||
@@ -329,6 +382,108 @@ impl ControllerTrait for Controller {
|
||||
))
|
||||
}
|
||||
|
||||
async fn create_image_edit(
|
||||
&self,
|
||||
prompt: &str,
|
||||
images: Vec<ImageSource>,
|
||||
_params: ImageEditParams,
|
||||
) -> anyhow::Result<ImageEditResult> {
|
||||
let Some(image_generation_config) = &self.config.image_generation else {
|
||||
return Err(anyhow::anyhow!(
|
||||
strings::agent::no_configuration_for_purpose_so_cannot_be_used(
|
||||
&AgentPurpose::ImageGeneration
|
||||
),
|
||||
));
|
||||
};
|
||||
|
||||
if images.is_empty() {
|
||||
return Err(anyhow::anyhow!("No image sources provided"));
|
||||
}
|
||||
|
||||
let mut image_inputs: Vec<ImageInput> = Vec::new();
|
||||
for image in images {
|
||||
image_inputs.push(image.into());
|
||||
}
|
||||
|
||||
let dalle2_size = match image_generation_config.size {
|
||||
Some(async_openai::types::images::ImageSize::S256x256) => {
|
||||
Some(async_openai::types::images::ImageSize::S256x256)
|
||||
}
|
||||
Some(async_openai::types::images::ImageSize::S512x512) => {
|
||||
Some(async_openai::types::images::ImageSize::S512x512)
|
||||
}
|
||||
Some(async_openai::types::images::ImageSize::S1024x1024) => {
|
||||
Some(async_openai::types::images::ImageSize::S1024x1024)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let model = image_generation_config
|
||||
.model_id_as_openai_image_model()
|
||||
.map_err(|err| anyhow::anyhow!(err))?;
|
||||
|
||||
let response_format = match model.clone() {
|
||||
ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
|
||||
ImageModel::DallE3 => Some(ImageResponseFormat::B64Json),
|
||||
// gpt-image-1 only outputs base64 and we don't need to specify the response format.
|
||||
// In fact, specifying the response format results in an error.
|
||||
ImageModel::GptImage1 => None,
|
||||
ImageModel::GptImage1Mini => None,
|
||||
ImageModel::GptImage1dot5 => None,
|
||||
ImageModel::Other(_) => Some(ImageResponseFormat::B64Json),
|
||||
};
|
||||
|
||||
let mut request_builder = CreateImageEditRequestArgs::default();
|
||||
|
||||
request_builder
|
||||
.image(image_inputs)
|
||||
.prompt(prompt.to_owned())
|
||||
.model(model);
|
||||
|
||||
if let Some(size) = dalle2_size {
|
||||
request_builder.size(size);
|
||||
}
|
||||
|
||||
if let Some(response_format) = response_format {
|
||||
request_builder.response_format(response_format);
|
||||
}
|
||||
|
||||
let request = request_builder
|
||||
.build()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to build CreateImageEditRequest: {}", e))?;
|
||||
|
||||
tracing::trace!(
|
||||
model = format!("{:?}", request.model),
|
||||
size = format!("{:?}", request.size),
|
||||
response_format = format!("{:?}", request.response_format),
|
||||
"Sending OpenAI image edit API request"
|
||||
);
|
||||
|
||||
let response = self.client.images().edit(request).await?;
|
||||
|
||||
if let Some(image_data) = response.data.into_iter().next() {
|
||||
match image_data.deref() {
|
||||
Image::B64Json { b64_json, .. } => {
|
||||
let bytes = base64_decode(b64_json.as_ref())?;
|
||||
return Ok(ImageEditResult {
|
||||
bytes,
|
||||
mime_type: mxlink::mime::IMAGE_PNG,
|
||||
});
|
||||
}
|
||||
Image::Url { url, .. } => {
|
||||
tracing::warn!(?url, "Received URL instead of B64Json for image edit");
|
||||
return Err(anyhow::anyhow!(
|
||||
"Unexpected image type (URL) when B64Json was requested"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow::anyhow!(
|
||||
"The OpenAI image edit API returned no images"
|
||||
))
|
||||
}
|
||||
|
||||
async fn text_to_speech(
|
||||
&self,
|
||||
input: &str,
|
||||
@@ -346,7 +501,7 @@ impl ControllerTrait for Controller {
|
||||
|
||||
let voice = if let Some(voice_string) = params.voice_override {
|
||||
// This is a hacky way to construct a Voice enum from the string we have.
|
||||
let voice: serde_json::Result<async_openai::types::Voice> =
|
||||
let voice: serde_json::Result<async_openai::types::audio::Voice> =
|
||||
serde_json::from_str(&format!("\"{}\"", voice_string));
|
||||
match voice {
|
||||
Ok(voice) => voice,
|
||||
@@ -386,7 +541,7 @@ impl ControllerTrait for Controller {
|
||||
"Sending OpenAI text-to-speech API request"
|
||||
);
|
||||
|
||||
let result = self.client.audio().speech(request).await?;
|
||||
let result = self.client.audio().speech().create(request).await?;
|
||||
|
||||
Ok(TextToSpeechResult {
|
||||
bytes: result.bytes.into(),
|
||||
@@ -445,15 +600,15 @@ impl ControllerTrait for Controller {
|
||||
}
|
||||
|
||||
fn response_format_to_mime_type(
|
||||
response_format: &async_openai::types::SpeechResponseFormat,
|
||||
response_format: &async_openai::types::audio::SpeechResponseFormat,
|
||||
) -> Option<mxlink::mime::Mime> {
|
||||
let content_type = match response_format {
|
||||
async_openai::types::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(),
|
||||
async_openai::types::SpeechResponseFormat::Wav => "audio/wav".to_owned(),
|
||||
async_openai::types::SpeechResponseFormat::Opus => "audio/ogg".to_owned(),
|
||||
async_openai::types::SpeechResponseFormat::Aac => "audio/aac".to_owned(),
|
||||
async_openai::types::SpeechResponseFormat::Flac => "audio/flac".to_owned(),
|
||||
async_openai::types::SpeechResponseFormat::Pcm => "audio/L8".to_owned(),
|
||||
async_openai::types::audio::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(),
|
||||
async_openai::types::audio::SpeechResponseFormat::Wav => "audio/wav".to_owned(),
|
||||
async_openai::types::audio::SpeechResponseFormat::Opus => "audio/ogg".to_owned(),
|
||||
async_openai::types::audio::SpeechResponseFormat::Aac => "audio/aac".to_owned(),
|
||||
async_openai::types::audio::SpeechResponseFormat::Flac => "audio/flac".to_owned(),
|
||||
async_openai::types::audio::SpeechResponseFormat::Pcm => "audio/L8".to_owned(),
|
||||
};
|
||||
|
||||
match content_type.parse() {
|
||||
@@ -481,3 +636,17 @@ fn audio_mime_type_to_file_name(mime_type: &mxlink::mime::Mime) -> Option<String
|
||||
|
||||
Some(format!("audio.{}", file_extension))
|
||||
}
|
||||
|
||||
/// Returns the smallest supported size for stickers based on what the image model supports.
|
||||
fn get_sticker_size(model: &ImageModel) -> async_openai::types::images::ImageSize {
|
||||
use async_openai::types::images::ImageSize;
|
||||
|
||||
match model {
|
||||
ImageModel::DallE2 => ImageSize::S256x256,
|
||||
ImageModel::DallE3 => ImageSize::S1024x1024,
|
||||
ImageModel::GptImage1 => ImageSize::S1024x1024,
|
||||
ImageModel::GptImage1Mini => ImageSize::S1024x1024,
|
||||
ImageModel::GptImage1dot5 => ImageSize::S1024x1024,
|
||||
ImageModel::Other(_) => ImageSize::S1024x1024,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,14 +16,16 @@ use super::super::AgentInstantiationResult;
|
||||
use super::ConfigTrait;
|
||||
use super::controller::ControllerType;
|
||||
|
||||
pub const OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5: &str = "gpt-image-1.5";
|
||||
|
||||
pub fn create_controller_from_yaml_value_config(
|
||||
agent_id: &str,
|
||||
config: serde_yaml::Value,
|
||||
config: serde_yaml_ng::Value,
|
||||
) -> AgentInstantiationResult<ControllerType> {
|
||||
let config = match &config {
|
||||
serde_yaml::Value::Mapping(_) => {
|
||||
serde_yaml_ng::Value::Mapping(_) => {
|
||||
let config: Config =
|
||||
serde_yaml::from_value(config).map_err(AgentInstantiationError::Yaml)?;
|
||||
serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?;
|
||||
|
||||
config
|
||||
.validate()
|
||||
|
||||
@@ -1,55 +1,64 @@
|
||||
use async_openai::types::{
|
||||
ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage,
|
||||
ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs,
|
||||
use async_openai::types::responses::{
|
||||
EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputFileArgs,
|
||||
InputImageContent, InputItem, InputParam, MessageType, Role,
|
||||
};
|
||||
|
||||
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
|
||||
use crate::conversation::llm::{
|
||||
Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
|
||||
};
|
||||
use crate::utils::base64::base64_encode;
|
||||
|
||||
pub fn convert_llm_messages_to_openai_messages(
|
||||
pub fn convert_llm_messages_to_openai_response_input(
|
||||
conversation_messages: Vec<LLMMessage>,
|
||||
) -> Vec<ChatCompletionRequestMessage> {
|
||||
let mut openai_conversation_messages: Vec<ChatCompletionRequestMessage> =
|
||||
Vec::with_capacity(conversation_messages.len());
|
||||
) -> InputParam {
|
||||
let mut items = Vec::with_capacity(conversation_messages.len());
|
||||
|
||||
for message in conversation_messages {
|
||||
openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
|
||||
let role = match message.author {
|
||||
LLMAuthor::Prompt => Role::System,
|
||||
LLMAuthor::Assistant => Role::Assistant,
|
||||
LLMAuthor::User => Role::User,
|
||||
};
|
||||
|
||||
let content = match message.content {
|
||||
LLMMessageContent::Text(text) => EasyInputContent::Text(text),
|
||||
LLMMessageContent::Image(image_details) => {
|
||||
let image_url = format!(
|
||||
"data:{};base64,{}",
|
||||
image_details.mime,
|
||||
base64_encode(&image_details.data)
|
||||
);
|
||||
|
||||
EasyInputContent::ContentList(vec![InputContent::InputImage(InputImageContent {
|
||||
image_url: Some(image_url),
|
||||
detail: ImageDetail::Auto,
|
||||
file_id: None,
|
||||
})])
|
||||
}
|
||||
LLMMessageContent::File(file_details) => {
|
||||
let file_data = format!(
|
||||
"data:{};base64,{}",
|
||||
file_details.mime,
|
||||
base64_encode(&file_details.data)
|
||||
);
|
||||
|
||||
let file_content = InputFileArgs::default()
|
||||
.file_data(file_data)
|
||||
.filename(file_details.filename())
|
||||
.build()
|
||||
.expect("Failed to build InputFileContent");
|
||||
|
||||
EasyInputContent::ContentList(vec![InputContent::InputFile(file_content)])
|
||||
}
|
||||
};
|
||||
|
||||
items.push(InputItem::EasyMessage(EasyInputMessage {
|
||||
r#type: MessageType::Message,
|
||||
role,
|
||||
content,
|
||||
phase: None,
|
||||
}));
|
||||
}
|
||||
|
||||
openai_conversation_messages
|
||||
}
|
||||
|
||||
fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> ChatCompletionRequestMessage {
|
||||
match llm_message.author {
|
||||
LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
|
||||
.content(llm_message.message_text)
|
||||
.build()
|
||||
.expect("Failed building OpenAI system message")
|
||||
.into(),
|
||||
LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
|
||||
.content(llm_message.message_text)
|
||||
.build()
|
||||
.expect("Failed building OpenAI assistant message")
|
||||
.into(),
|
||||
LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
|
||||
.content(llm_message.message_text)
|
||||
.build()
|
||||
.expect("Failed building OpenAI user message")
|
||||
.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn convert_string_to_enum<T>(value: &str) -> Result<T, String>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
// This is a hacky way to construct an enum from the string we have.
|
||||
let enum_result: serde_json::Result<T> = serde_json::from_str(&format!("\"{}\"", value));
|
||||
match enum_result {
|
||||
Ok(enum_result) => Ok(enum_result),
|
||||
Err(err) => {
|
||||
tracing::debug!(?err, "Failed to parse into enum");
|
||||
|
||||
Err(format!("The value ({}) is not supported.", value))
|
||||
}
|
||||
}
|
||||
InputParam::Items(items)
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ impl TryInto<OpenAITextGenerationConfig> for TextGenerationConfig {
|
||||
max_response_tokens: self.max_response_tokens,
|
||||
max_completion_tokens: None,
|
||||
max_context_tokens: self.max_context_tokens,
|
||||
tools: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -161,13 +162,14 @@ impl TryInto<OpenAITextToSpeechConfig> for TextToSpeechConfig {
|
||||
type Error = String;
|
||||
|
||||
fn try_into(self) -> Result<OpenAITextToSpeechConfig, Self::Error> {
|
||||
let model_id = convert_string_to_enum::<async_openai::types::SpeechModel>(&self.model_id)?;
|
||||
let model_id =
|
||||
convert_string_to_enum::<async_openai::types::audio::SpeechModel>(&self.model_id)?;
|
||||
|
||||
let voice = convert_string_to_enum::<async_openai::types::Voice>(&self.voice)?;
|
||||
let voice = convert_string_to_enum::<async_openai::types::audio::Voice>(&self.voice)?;
|
||||
|
||||
let response_format = convert_string_to_enum::<async_openai::types::SpeechResponseFormat>(
|
||||
&self.response_format,
|
||||
)?;
|
||||
let response_format = convert_string_to_enum::<
|
||||
async_openai::types::audio::SpeechResponseFormat,
|
||||
>(&self.response_format)?;
|
||||
|
||||
Ok(OpenAITextToSpeechConfig {
|
||||
model_id,
|
||||
@@ -224,21 +226,27 @@ impl TryInto<OpenAIImageGenerationConfig> for ImageGenerationConfig {
|
||||
|
||||
fn try_into(self) -> Result<OpenAIImageGenerationConfig, Self::Error> {
|
||||
let size = if let Some(size) = &self.size {
|
||||
convert_string_to_enum::<async_openai::types::ImageSize>(size)?
|
||||
Some(convert_string_to_enum::<
|
||||
async_openai::types::images::ImageSize,
|
||||
>(size)?)
|
||||
} else {
|
||||
async_openai::types::ImageSize::S1024x1024
|
||||
None
|
||||
};
|
||||
|
||||
let style = if let Some(style) = &self.style {
|
||||
convert_string_to_enum::<async_openai::types::ImageStyle>(style)?
|
||||
Some(convert_string_to_enum::<
|
||||
async_openai::types::images::ImageStyle,
|
||||
>(style)?)
|
||||
} else {
|
||||
async_openai::types::ImageStyle::Vivid
|
||||
None
|
||||
};
|
||||
|
||||
let quality = if let Some(quality) = &self.quality {
|
||||
convert_string_to_enum::<async_openai::types::ImageQuality>(quality)?
|
||||
Some(convert_string_to_enum::<
|
||||
async_openai::types::images::ImageQuality,
|
||||
>(quality)?)
|
||||
} else {
|
||||
async_openai::types::ImageQuality::Standard
|
||||
None
|
||||
};
|
||||
|
||||
Ok(OpenAIImageGenerationConfig {
|
||||
|
||||
@@ -3,23 +3,27 @@ use etke_openai_api_rust::chat::{ChatApi, ChatBody};
|
||||
use etke_openai_api_rust::images::{ImagesApi, ImagesBody};
|
||||
use etke_openai_api_rust::{Auth, Message, OpenAI};
|
||||
|
||||
const SMALLEST_IMAGE_SIZE: &str = "256x256";
|
||||
|
||||
use super::super::ControllerTrait;
|
||||
use crate::agent::utils::base64_decode;
|
||||
use crate::utils::base64::base64_decode;
|
||||
use crate::{
|
||||
agent::provider::{
|
||||
ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
|
||||
ImageEditParams, ImageGenerationParams, ImageSource, SpeechToTextParams,
|
||||
SpeechToTextResult,
|
||||
entity::{TextGenerationParams, TextGenerationResult},
|
||||
},
|
||||
conversation::llm::{
|
||||
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
|
||||
shorten_messages_list_to_context_size,
|
||||
MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
|
||||
},
|
||||
};
|
||||
use crate::{
|
||||
agent::{
|
||||
AgentPurpose,
|
||||
provider::entity::{
|
||||
ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult,
|
||||
ImageEditResult, ImageGenerationResult, PingResult, TextToSpeechParams,
|
||||
TextToSpeechResult,
|
||||
},
|
||||
},
|
||||
strings,
|
||||
@@ -60,7 +64,8 @@ impl ControllerTrait for Controller {
|
||||
|
||||
let messages = vec![LLMMessage {
|
||||
author: LLMAuthor::User,
|
||||
message_text: "Hello!".to_string(),
|
||||
sender_id: None,
|
||||
content: LLMMessageContent::Text("Hello!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
}];
|
||||
|
||||
@@ -97,7 +102,8 @@ impl ControllerTrait for Controller {
|
||||
} else {
|
||||
Some(LLMMessage {
|
||||
author: LLMAuthor::Prompt,
|
||||
message_text: prompt_text,
|
||||
sender_id: None,
|
||||
content: LLMMessageContent::Text(prompt_text),
|
||||
timestamp: chrono::Utc::now(),
|
||||
})
|
||||
};
|
||||
@@ -301,9 +307,11 @@ impl ControllerTrait for Controller {
|
||||
// when they span multiple lines.
|
||||
let prompt = prompt.replace("\n", " ");
|
||||
|
||||
let size: Option<String> = params
|
||||
.size_override
|
||||
.or_else(|| image_generation_config.size.clone());
|
||||
let size: Option<String> = if params.smallest_size_possible {
|
||||
Some(SMALLEST_IMAGE_SIZE.to_owned())
|
||||
} else {
|
||||
image_generation_config.size.clone()
|
||||
};
|
||||
|
||||
let request = ImagesBody {
|
||||
model: Some(image_generation_config.model_id.to_owned()),
|
||||
@@ -366,6 +374,17 @@ impl ControllerTrait for Controller {
|
||||
))
|
||||
}
|
||||
|
||||
async fn create_image_edit(
|
||||
&self,
|
||||
_prompt: &str,
|
||||
_images: Vec<ImageSource>,
|
||||
_params: ImageEditParams,
|
||||
) -> anyhow::Result<ImageEditResult> {
|
||||
Err(anyhow::anyhow!(
|
||||
"The OpenAI image edit API is not supported by the OpenAI-compat provider"
|
||||
))
|
||||
}
|
||||
|
||||
async fn text_to_speech(
|
||||
&self,
|
||||
input: &str,
|
||||
|
||||
@@ -26,12 +26,12 @@ use super::controller::ControllerType;
|
||||
|
||||
pub fn create_controller_from_yaml_value_config(
|
||||
agent_id: &str,
|
||||
config: serde_yaml::Value,
|
||||
config: serde_yaml_ng::Value,
|
||||
) -> AgentInstantiationResult<ControllerType> {
|
||||
let config = match &config {
|
||||
serde_yaml::Value::Mapping(_) => {
|
||||
serde_yaml_ng::Value::Mapping(_) => {
|
||||
let config: Config =
|
||||
serde_yaml::from_value(config).map_err(AgentInstantiationError::Yaml)?;
|
||||
serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?;
|
||||
|
||||
config
|
||||
.validate()
|
||||
|
||||
@@ -2,7 +2,9 @@ use etke_openai_api_rust::{Message, Role};
|
||||
|
||||
use crate::agent::provider::openai::Config as OpenAIConfig;
|
||||
|
||||
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
|
||||
use crate::conversation::llm::{
|
||||
Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
|
||||
};
|
||||
|
||||
pub fn convert_llm_messages_to_openai_messages(
|
||||
conversation_messages: Vec<LLMMessage>,
|
||||
@@ -11,22 +13,39 @@ pub fn convert_llm_messages_to_openai_messages(
|
||||
Vec::with_capacity(conversation_messages.len());
|
||||
|
||||
for message in conversation_messages {
|
||||
openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
|
||||
let openai_message = convert_llm_message_to_openai_message(message);
|
||||
if let Some(openai_message) = openai_message {
|
||||
openai_conversation_messages.push(openai_message);
|
||||
}
|
||||
}
|
||||
|
||||
openai_conversation_messages
|
||||
}
|
||||
|
||||
fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Message {
|
||||
fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option<Message> {
|
||||
let role = match llm_message.author {
|
||||
LLMAuthor::Prompt => Role::System,
|
||||
LLMAuthor::Assistant => Role::Assistant,
|
||||
LLMAuthor::User => Role::User,
|
||||
};
|
||||
|
||||
Message {
|
||||
role,
|
||||
content: llm_message.message_text,
|
||||
match &llm_message.content {
|
||||
LLMMessageContent::Text(text) => Some(Message {
|
||||
role,
|
||||
content: text.clone(),
|
||||
}),
|
||||
LLMMessageContent::Image(_image_details) => {
|
||||
tracing::warn!(
|
||||
"The OpenAI-compat provider's library does not support image content. This image message will be skipped."
|
||||
);
|
||||
None
|
||||
}
|
||||
LLMMessageContent::File(_file_details) => {
|
||||
tracing::warn!(
|
||||
"The OpenAI-compat provider's library does not support file content. This file message will be skipped."
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
|
||||
use crate::{
|
||||
agent::{
|
||||
AgentInstance, AgentPurpose, ControllerTrait, Manager as AgentManager, PublicIdentifier,
|
||||
@@ -140,7 +138,3 @@ async fn get_global_agent_id_for_purpose(
|
||||
.handler
|
||||
.get_by_purpose_with_catch_all_fallback(purpose)
|
||||
}
|
||||
|
||||
pub(crate) fn base64_decode(base64_string: &str) -> Result<Vec<u8>, base64::DecodeError> {
|
||||
STANDARD.decode(base64_string)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use std::fs;
|
||||
use std::sync::Arc;
|
||||
use std::{future::Future, pin::Pin};
|
||||
|
||||
use mxlink::matrix_sdk::Room;
|
||||
use mxlink::matrix_sdk::media::{MediaFormat, MediaRequestParameters};
|
||||
use mxlink::matrix_sdk::ruma::api::client::profile::{AvatarUrl, DisplayName};
|
||||
use mxlink::matrix_sdk::ruma::{
|
||||
MilliSecondsSinceUnixEpoch, OwnedUserId, events::room::MediaSource,
|
||||
};
|
||||
@@ -17,12 +19,13 @@ use mxlink::helpers::account_data_config::{
|
||||
RoomConfigManager as AccountDataRoomConfigManager,
|
||||
};
|
||||
use mxlink::helpers::encryption::Manager as EncryptionManager;
|
||||
use mxlink::mime::Mime;
|
||||
|
||||
use crate::agent::Manager as AgentManager;
|
||||
use crate::entity::catch_up_marker::{
|
||||
CatchUpMarker, CatchUpMarkerManager, DelayedCatchUpMarkerManager,
|
||||
};
|
||||
use crate::entity::cfg::Config;
|
||||
use crate::entity::cfg::{Avatar, Config, ConfigUserAuth};
|
||||
use crate::entity::globalconfig::{GlobalConfig, GlobalConfigurationManager};
|
||||
use crate::entity::roomconfig::{RoomConfig, RoomConfigurationManager};
|
||||
|
||||
@@ -285,24 +288,27 @@ impl Bot {
|
||||
async fn do_prepare_profile(&self) -> anyhow::Result<()> {
|
||||
tracing::debug!("Preparing profile..");
|
||||
|
||||
let desired_display_name = self.inner.config.user.name.clone();
|
||||
|
||||
let account = self.inner.matrix_link.client().account();
|
||||
let media = self.inner.matrix_link.client().media();
|
||||
|
||||
let desired_display_name = self.inner.config.user.name.clone();
|
||||
|
||||
let profile = account
|
||||
.fetch_user_profile()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed fetching profile: {:?}", e))?;
|
||||
|
||||
let should_update_display_name = match &profile.displayname {
|
||||
let current_display_name = profile.get_static::<DisplayName>()?;
|
||||
let current_avatar_url = profile.get_static::<AvatarUrl>()?;
|
||||
|
||||
let should_update_display_name = match ¤t_display_name {
|
||||
Some(displayname) => displayname != &desired_display_name,
|
||||
None => true,
|
||||
};
|
||||
|
||||
if should_update_display_name {
|
||||
tracing::info!(
|
||||
?profile.displayname,
|
||||
?current_display_name,
|
||||
?desired_display_name,
|
||||
"Updating display name.."
|
||||
);
|
||||
@@ -312,34 +318,72 @@ impl Bot {
|
||||
}
|
||||
}
|
||||
|
||||
let should_update_avatar = match &profile.avatar_url {
|
||||
Some(avatar_url) => {
|
||||
let request = MediaRequestParameters {
|
||||
source: MediaSource::Plain(avatar_url.to_owned()),
|
||||
format: MediaFormat::File,
|
||||
};
|
||||
|
||||
let content = media
|
||||
.get_media_content(&request, true)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed fetching existing avatar: {:?}", e))?;
|
||||
|
||||
content.as_slice() != LOGO_BYTES
|
||||
let desired_avatar: Option<(Vec<u8>, Mime)> = match &self.inner.config.user.avatar {
|
||||
Avatar::Keep => {
|
||||
tracing::info!("Avatar configured to keep current, skipping avatar management");
|
||||
None
|
||||
}
|
||||
Avatar::Default => {
|
||||
tracing::info!("Avatar configured to use default");
|
||||
Some((
|
||||
LOGO_BYTES.to_vec(),
|
||||
LOGO_MIME_TYPE
|
||||
.parse()
|
||||
.expect("Failed parsing mime type for logo"),
|
||||
))
|
||||
}
|
||||
Avatar::Custom(avatar_path) => {
|
||||
tracing::info!(?avatar_path, "Avatar configured to use custom path");
|
||||
let bytes = fs::read(avatar_path).map_err(|e| {
|
||||
anyhow::anyhow!("Failed reading avatar from {:?}: {:?}", avatar_path, e)
|
||||
})?;
|
||||
let mime = mime_guess::from_path(avatar_path).first_or_octet_stream();
|
||||
tracing::debug!(?mime, bytes_len = bytes.len(), "Loaded custom avatar");
|
||||
Some((bytes, mime))
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
|
||||
if should_update_avatar {
|
||||
tracing::info!("Updating avatar..");
|
||||
if let Some((desired_bytes, mime_type)) = desired_avatar {
|
||||
let should_update_avatar = match ¤t_avatar_url {
|
||||
Some(avatar_url) => {
|
||||
tracing::debug!(?avatar_url, "Fetching current avatar to compare");
|
||||
let request = MediaRequestParameters {
|
||||
source: MediaSource::Plain(avatar_url.to_owned()),
|
||||
format: MediaFormat::File,
|
||||
};
|
||||
|
||||
let mime_type = LOGO_MIME_TYPE
|
||||
.parse()
|
||||
.expect("Failed parsing mime type for logo");
|
||||
let content = media
|
||||
.get_media_content(&request, true)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed fetching existing avatar: {:?}", e))?;
|
||||
|
||||
account
|
||||
.upload_avatar(&mime_type, LOGO_BYTES.to_vec())
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed uploading avatar: {:?}", e))?;
|
||||
let needs_update = content.as_slice() != desired_bytes;
|
||||
|
||||
tracing::debug!(
|
||||
current_bytes_len = content.len(),
|
||||
desired_bytes_len = desired_bytes.len(),
|
||||
?needs_update,
|
||||
"Compared current and desired avatar"
|
||||
);
|
||||
|
||||
needs_update
|
||||
}
|
||||
None => {
|
||||
tracing::debug!("No current avatar set, will upload");
|
||||
true
|
||||
}
|
||||
};
|
||||
|
||||
if should_update_avatar {
|
||||
tracing::info!("Updating avatar..");
|
||||
account
|
||||
.upload_avatar(&mime_type, desired_bytes)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed uploading avatar: {:?}", e))?;
|
||||
tracing::info!("Avatar updated successfully");
|
||||
} else {
|
||||
tracing::debug!("Avatar already up to date, skipping upload");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -351,10 +395,22 @@ async fn create_matrix_link(config: &Config) -> anyhow::Result<MatrixLink> {
|
||||
let session_encryption_key = config.persistence.session_encryption_key()?;
|
||||
let db_dir_path: std::path::PathBuf = config.persistence.db_dir_path()?;
|
||||
|
||||
let login_creds = LoginCredentials::UserPassword(
|
||||
config.user.mxid_localpart.to_owned(),
|
||||
config.user.password.to_owned(),
|
||||
);
|
||||
let user_auth = config.user.auth_config(&config.homeserver.server_name)?;
|
||||
|
||||
let login_creds = match user_auth {
|
||||
ConfigUserAuth::UserPassword { username, password } => {
|
||||
LoginCredentials::UserPassword(username, password)
|
||||
}
|
||||
ConfigUserAuth::AccessToken {
|
||||
user_id,
|
||||
device_id,
|
||||
access_token,
|
||||
} => LoginCredentials::AccessToken {
|
||||
user_id,
|
||||
device_id,
|
||||
access_token,
|
||||
},
|
||||
};
|
||||
|
||||
let login_encryption = LoginEncryption::new(
|
||||
config.user.encryption.recovery_passphrase.clone(),
|
||||
|
||||
@@ -5,7 +5,7 @@ use anyhow::anyhow;
|
||||
|
||||
use crate::agent::AgentPurpose;
|
||||
|
||||
pub use crate::entity::cfg::{Config, defaults as cfg_defaults, env as cfg_env};
|
||||
pub use crate::entity::cfg::{Avatar, Config, defaults as cfg_defaults, env as cfg_env};
|
||||
|
||||
pub fn load() -> anyhow::Result<Config> {
|
||||
let config_file_path = env::var(cfg_env::BAIBOT_CONFIG_FILE_PATH)
|
||||
@@ -21,7 +21,7 @@ pub fn load() -> anyhow::Result<Config> {
|
||||
}
|
||||
|
||||
let config_str = std::fs::read_to_string(config_file_path)?;
|
||||
let mut config: Config = serde_yaml::from_str(&config_str)?;
|
||||
let mut config: Config = serde_yaml_ng::from_str(&config_str)?;
|
||||
|
||||
// Allow environment variables to override some configuration keys
|
||||
for (key, value) in env::vars() {
|
||||
@@ -29,11 +29,25 @@ pub fn load() -> anyhow::Result<Config> {
|
||||
cfg_env::BAIBOT_HOMESERVER_SERVER_NAME => config.homeserver.server_name = value,
|
||||
cfg_env::BAIBOT_HOMESERVER_URL => config.homeserver.url = value,
|
||||
cfg_env::BAIBOT_USER_MXID_LOCALPART => config.user.mxid_localpart = value,
|
||||
cfg_env::BAIBOT_USER_PASSWORD => config.user.password = value,
|
||||
cfg_env::BAIBOT_USER_PASSWORD => {
|
||||
config.user.password = optional_non_empty(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_ACCESS_TOKEN => {
|
||||
config.user.access_token = optional_non_empty(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_DEVICE_ID => {
|
||||
config.user.device_id = optional_non_empty(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE => {
|
||||
config.user.encryption.recovery_passphrase = Some(value);
|
||||
}
|
||||
cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED => {
|
||||
config.user.encryption.recovery_reset_allowed = value.parse::<bool>()?;
|
||||
}
|
||||
cfg_env::BAIBOT_USER_NAME => config.user.name = value,
|
||||
cfg_env::BAIBOT_USER_AVATAR => {
|
||||
config.user.avatar = Avatar::from_string(value);
|
||||
}
|
||||
cfg_env::BAIBOT_COMMAND_PREFIX => config.command_prefix = value,
|
||||
cfg_env::BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED => {
|
||||
config.room.post_join_self_introduction_enabled = value.parse::<bool>()?;
|
||||
@@ -51,6 +65,9 @@ pub fn load() -> anyhow::Result<Config> {
|
||||
cfg_env::BAIBOT_PERSISTENCE_DATA_DIR_PATH => {
|
||||
config.persistence.data_dir_path = Some(value);
|
||||
}
|
||||
cfg_env::BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY => {
|
||||
config.persistence.session_encryption_key = Some(value);
|
||||
}
|
||||
cfg_env::BAIBOT_PERSISTENCE_CONFIG_ENCRYPTION_KEY => {
|
||||
config.persistence.config_encryption_key = Some(value);
|
||||
}
|
||||
@@ -111,3 +128,7 @@ pub fn load() -> anyhow::Result<Config> {
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn optional_non_empty(value: String) -> Option<String> {
|
||||
if value.is_empty() { None } else { Some(value) }
|
||||
}
|
||||
|
||||
@@ -28,18 +28,18 @@ pub async fn handle_set(
|
||||
message_context: &MessageContext,
|
||||
patterns: &Option<Vec<String>>,
|
||||
) -> anyhow::Result<()> {
|
||||
if let Some(patterns) = patterns {
|
||||
if let Err(err) = mxidwc::parse_patterns_vector(patterns) {
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::access::failed_to_parse_patterns(&err.to_string()),
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
|
||||
)
|
||||
.await;
|
||||
if let Some(patterns) = patterns
|
||||
&& let Err(err) = mxidwc::parse_patterns_vector(patterns)
|
||||
{
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::access::failed_to_parse_patterns(&err.to_string()),
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut global_config_manager_guard = bot.global_config_manager().lock().await;
|
||||
|
||||
@@ -24,18 +24,18 @@ pub async fn handle_set(
|
||||
message_context: &MessageContext,
|
||||
patterns: &Option<Vec<String>>,
|
||||
) -> anyhow::Result<()> {
|
||||
if let Some(patterns) = patterns {
|
||||
if let Err(err) = mxidwc::parse_patterns_vector(patterns) {
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::access::failed_to_parse_patterns(&err.to_string()),
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
|
||||
)
|
||||
.await;
|
||||
if let Some(patterns) = patterns
|
||||
&& let Err(err) = mxidwc::parse_patterns_vector(patterns)
|
||||
{
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::access::failed_to_parse_patterns(&err.to_string()),
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut global_config_manager_guard = bot.global_config_manager().lock().await;
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{Bot, entity::MessageContext};
|
||||
|
||||
struct ParsedAgentConfig {
|
||||
agent: AgentInstance,
|
||||
config: serde_yaml::Value,
|
||||
config: serde_yaml_ng::Value,
|
||||
}
|
||||
|
||||
pub async fn handle_room_local(
|
||||
@@ -250,7 +250,7 @@ async fn send_guide(
|
||||
provider: &AgentProvider,
|
||||
) -> anyhow::Result<()> {
|
||||
let sample_config = crate::agent::default_config_for_provider(provider);
|
||||
let sample_config_pretty_yaml = serde_yaml::to_string(&sample_config)?;
|
||||
let sample_config_pretty_yaml = serde_yaml_ng::to_string(&sample_config)?;
|
||||
|
||||
bot.messaging()
|
||||
.send_text_markdown_no_fail(
|
||||
@@ -263,7 +263,7 @@ async fn send_guide(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_from_message_to_yaml_value(text: &str) -> Result<serde_yaml::Value, String> {
|
||||
fn parse_from_message_to_yaml_value(text: &str) -> Result<serde_yaml_ng::Value, String> {
|
||||
let mut text = text.trim();
|
||||
|
||||
if text.starts_with("```") {
|
||||
@@ -274,10 +274,10 @@ fn parse_from_message_to_yaml_value(text: &str) -> Result<serde_yaml::Value, Str
|
||||
text = text.trim_end_matches("```");
|
||||
}
|
||||
|
||||
let config: serde_yaml::Value = serde_yaml::from_str(text).map_err(|e| e.to_string())?;
|
||||
let config: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).map_err(|e| e.to_string())?;
|
||||
|
||||
match config {
|
||||
serde_yaml::Value::Mapping(_) => {}
|
||||
serde_yaml_ng::Value::Mapping(_) => {}
|
||||
_ => {
|
||||
return Err("Not a valid YAML hashmap".to_owned());
|
||||
}
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
fn agent_config_parsing_works() {
|
||||
struct TestCase {
|
||||
input: String,
|
||||
expected: Option<serde_yaml::Value>,
|
||||
expected: Option<serde_yaml_ng::Value>,
|
||||
}
|
||||
|
||||
let provider = crate::agent::AgentProvider::OpenAI;
|
||||
let sample_config = crate::agent::default_config_for_provider(&provider);
|
||||
let sample_config_pretty_yaml = serde_yaml::to_string(&sample_config).unwrap();
|
||||
let sample_config_pretty_yaml = serde_yaml_ng::to_string(&sample_config).unwrap();
|
||||
|
||||
let test_cases = vec![
|
||||
// Invalid input
|
||||
|
||||
@@ -64,7 +64,7 @@ pub async fn handle(
|
||||
PublicIdentifier::Static(_) => {}
|
||||
};
|
||||
|
||||
let config_yaml_pretty = serde_yaml::to_string(&agent.definition().config)?;
|
||||
let config_yaml_pretty = serde_yaml_ng::to_string(&agent.definition().config)?;
|
||||
|
||||
bot.messaging()
|
||||
.send_text_markdown_no_fail(
|
||||
|
||||
@@ -3,7 +3,8 @@ use crate::{
|
||||
entity::roomconfig::{
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType,
|
||||
TextToSpeechUserMessagesFlowType,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -48,6 +49,9 @@ pub enum ConfigTextGenerationSettingRelatedControllerType {
|
||||
|
||||
GetTemperatureOverride,
|
||||
SetTemperatureOverride(Option<f32>),
|
||||
|
||||
GetSenderContextMode,
|
||||
SetSenderContextMode(Option<TextGenerationSenderContextMode>),
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
|
||||
@@ -163,6 +163,26 @@ fn determine_controller() {
|
||||
),
|
||||
)),
|
||||
},
|
||||
TestCase {
|
||||
name: "per-room text-generation/sender-context-mode getter",
|
||||
input: "room text-generation sender-context-mode",
|
||||
expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated(
|
||||
controller_type::SettingsStorageSource::Room,
|
||||
controller_type::ConfigSettingRelatedControllerType::TextGeneration(
|
||||
controller_type::ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode,
|
||||
),
|
||||
)),
|
||||
},
|
||||
TestCase {
|
||||
name: "global text-generation/sender-context-mode getter",
|
||||
input: "global text-generation sender-context-mode",
|
||||
expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated(
|
||||
controller_type::SettingsStorageSource::Global,
|
||||
controller_type::ConfigSettingRelatedControllerType::TextGeneration(
|
||||
controller_type::ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode,
|
||||
),
|
||||
)),
|
||||
},
|
||||
TestCase {
|
||||
name: "per-room text-to-speech/speed-override getter",
|
||||
input: "room text-to-speech speed-override",
|
||||
|
||||
@@ -3,7 +3,10 @@ mod tests;
|
||||
|
||||
use crate::{
|
||||
controller::ControllerType,
|
||||
entity::roomconfig::{TextGenerationAutoUsage, TextGenerationPrefixRequirementType},
|
||||
entity::roomconfig::{
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextGenerationSenderContextMode,
|
||||
},
|
||||
strings,
|
||||
};
|
||||
|
||||
@@ -197,5 +200,43 @@ pub(super) fn determine(
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(remaining_text) = text.strip_prefix("sender-context-mode") {
|
||||
let remaining_text = remaining_text.trim();
|
||||
|
||||
if !remaining_text.is_empty() {
|
||||
return Err(ControllerType::Error(
|
||||
strings::cfg::configuration_getter_used_with_extra_text(
|
||||
"sender-context-mode",
|
||||
remaining_text,
|
||||
)
|
||||
.to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
return Ok(ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode);
|
||||
}
|
||||
|
||||
if let Some(value_string) = text.strip_prefix("set-sender-context-mode") {
|
||||
let value_string = value_string.trim().to_owned();
|
||||
let value_choice = if value_string.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let value_choice =
|
||||
TextGenerationSenderContextMode::from_str(&value_string.to_lowercase());
|
||||
|
||||
if value_choice.is_none() {
|
||||
return Err(ControllerType::Error(
|
||||
strings::cfg::configuration_value_unrecognized(&value_string).to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
value_choice
|
||||
};
|
||||
|
||||
return Ok(
|
||||
ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(value_choice),
|
||||
);
|
||||
}
|
||||
|
||||
Err(ControllerType::Unknown)
|
||||
}
|
||||
|
||||
@@ -90,6 +90,74 @@ fn determine_controller_context_management() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn determine_controller_sender_context() {
|
||||
use super::ConfigTextGenerationSettingRelatedControllerType;
|
||||
use super::ControllerType;
|
||||
use crate::entity::roomconfig::TextGenerationSenderContextMode;
|
||||
|
||||
struct TestCase {
|
||||
name: &'static str,
|
||||
input: &'static str,
|
||||
expected: Result<ConfigTextGenerationSettingRelatedControllerType, ControllerType>,
|
||||
}
|
||||
|
||||
let test_cases = vec![
|
||||
TestCase {
|
||||
name: "sender-context-mode getter ok",
|
||||
input: "sender-context-mode",
|
||||
expected: Ok(ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode),
|
||||
},
|
||||
TestCase {
|
||||
name: "sender-context-mode getter extra args",
|
||||
input: "sender-context-mode some values here",
|
||||
expected: Err(ControllerType::Error(
|
||||
crate::strings::cfg::configuration_getter_used_with_extra_text(
|
||||
"sender-context-mode",
|
||||
"some values here",
|
||||
),
|
||||
)),
|
||||
},
|
||||
TestCase {
|
||||
name: "sender-context-mode setter matrix_user_id",
|
||||
input: "set-sender-context-mode matrix_user_id",
|
||||
expected: Ok(
|
||||
ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(Some(
|
||||
TextGenerationSenderContextMode::MatrixUserId,
|
||||
)),
|
||||
),
|
||||
},
|
||||
TestCase {
|
||||
name: "sender-context-mode setter uppercase",
|
||||
input: "set-sender-context-mode MATRIX_USER_ID_AND_TIMESTAMP",
|
||||
expected: Ok(
|
||||
ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(Some(
|
||||
TextGenerationSenderContextMode::MatrixUserIdAndTimestamp,
|
||||
)),
|
||||
),
|
||||
},
|
||||
TestCase {
|
||||
name: "sender-context-mode setter invalid",
|
||||
input: "set-sender-context-mode non-Enum-Value",
|
||||
expected: Err(ControllerType::Error(
|
||||
crate::strings::cfg::configuration_value_unrecognized("non-Enum-Value"),
|
||||
)),
|
||||
},
|
||||
TestCase {
|
||||
name: "sender-context-mode unsetter",
|
||||
input: "set-sender-context-mode",
|
||||
expected: Ok(
|
||||
ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(None),
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
for test_case in test_cases {
|
||||
let result = super::determine(test_case.input);
|
||||
assert_eq!(result, test_case.expected, "Test case: {}", test_case.name);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn determine_controller_prefix_requirement_type() {
|
||||
use super::ConfigTextGenerationSettingRelatedControllerType;
|
||||
|
||||
@@ -39,18 +39,18 @@ async fn dispatch_config_related_handler(
|
||||
message_context: &MessageContext,
|
||||
bot: &Bot,
|
||||
) -> anyhow::Result<()> {
|
||||
if let SettingsStorageSource::Global = config_type {
|
||||
if !message_context.sender_can_manage_global_config() {
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
strings::global_config::no_permissions_to_administrate(),
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
if let SettingsStorageSource::Global = config_type
|
||||
&& !message_context.sender_can_manage_global_config()
|
||||
{
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
strings::global_config::no_permissions_to_administrate(),
|
||||
MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let room_settings = match config_type {
|
||||
SettingsStorageSource::Room => &message_context.room_config().settings,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::entity::roomconfig::{
|
||||
RoomSettings, TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextGenerationSenderContextMode,
|
||||
};
|
||||
use crate::{Bot, entity::MessageContext};
|
||||
|
||||
@@ -151,5 +152,38 @@ pub(super) async fn dispatch(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode => {
|
||||
let value = &room_settings.text_generation.sender_context_mode;
|
||||
setting_get::<TextGenerationSenderContextMode>(bot, message_context, value).await
|
||||
}
|
||||
ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(value) => {
|
||||
let value = value.to_owned();
|
||||
|
||||
let setter_callback = Box::new(move |room_settings: &mut RoomSettings| {
|
||||
room_settings.text_generation.sender_context_mode = value;
|
||||
});
|
||||
|
||||
match config_type {
|
||||
SettingsStorageSource::Room => {
|
||||
room_setting_set::<TextGenerationSenderContextMode>(
|
||||
bot,
|
||||
message_context,
|
||||
&value,
|
||||
setter_callback,
|
||||
)
|
||||
.await
|
||||
}
|
||||
SettingsStorageSource::Global => {
|
||||
global_setting_set::<TextGenerationSenderContextMode>(
|
||||
bot,
|
||||
message_context,
|
||||
&value,
|
||||
setter_callback,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,8 @@ use crate::{
|
||||
roomconfig::{
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextGenerationAutoUsage, TextGenerationPrefixRequirementType,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType,
|
||||
TextToSpeechUserMessagesFlowType,
|
||||
},
|
||||
},
|
||||
strings,
|
||||
@@ -233,6 +234,46 @@ fn build_section_text_generation(command_prefix: &str, bot_username: &str) -> St
|
||||
));
|
||||
message.push_str("\n\n");
|
||||
|
||||
// Sender Context
|
||||
|
||||
message.push_str(&format!(
|
||||
"#### {}",
|
||||
strings::help::cfg::text_generation_sender_context_heading()
|
||||
));
|
||||
message.push_str("\n\n");
|
||||
message.push_str(&strings::help::cfg::text_generation_sender_context_intro());
|
||||
message.push('\n');
|
||||
message.push_str(
|
||||
&strings::help::cfg::the_following_configuration_values_are_recognized(
|
||||
TextGenerationSenderContextMode::choices(),
|
||||
),
|
||||
);
|
||||
message.push_str("\n\n");
|
||||
message.push_str(&format!(
|
||||
"- {}",
|
||||
&strings::help::cfg::current_setting_show(
|
||||
command_prefix,
|
||||
"text-generation sender-context-mode"
|
||||
)
|
||||
));
|
||||
message.push('\n');
|
||||
message.push_str(&format!(
|
||||
"- {}",
|
||||
&strings::help::cfg::current_setting_set(
|
||||
command_prefix,
|
||||
"text-generation set-sender-context-mode VALUE"
|
||||
)
|
||||
));
|
||||
message.push('\n');
|
||||
message.push_str(&format!(
|
||||
"- {}",
|
||||
&strings::help::cfg::current_setting_unset(
|
||||
command_prefix,
|
||||
"text-generation set-sender-context-mode"
|
||||
)
|
||||
));
|
||||
message.push_str("\n\n");
|
||||
|
||||
// Prompt override
|
||||
|
||||
message.push_str(&format!(
|
||||
|
||||
@@ -69,7 +69,7 @@ pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Resu
|
||||
);
|
||||
message.push_str("\n\n");
|
||||
|
||||
// Image Generation
|
||||
// Image Creation
|
||||
message.push_str(
|
||||
&generate_image_generation_section(agent_manager, message_context.room_config_context())
|
||||
.await,
|
||||
@@ -359,6 +359,33 @@ async fn generate_text_generation_section(
|
||||
),
|
||||
);
|
||||
|
||||
// Sender Context
|
||||
|
||||
let effective_sender_context = room_config_context.text_generation_sender_context_mode();
|
||||
let room_config_sender_context = room_config_context
|
||||
.room_config
|
||||
.settings
|
||||
.text_generation
|
||||
.sender_context_mode;
|
||||
let global_config_sender_context = room_config_context
|
||||
.global_config
|
||||
.fallback_room_settings
|
||||
.text_generation
|
||||
.sender_context_mode;
|
||||
|
||||
let sender_context_set_where = if room_config_sender_context.is_some() {
|
||||
strings::cfg::status_badge_set_in_room_config()
|
||||
} else if global_config_sender_context.is_some() {
|
||||
strings::cfg::status_badge_set_in_global_config()
|
||||
} else {
|
||||
strings::cfg::status_badge_using_hardcoded_default()
|
||||
};
|
||||
|
||||
message.push_str(&strings::cfg::status_text_generation_entry_sender_context(
|
||||
effective_sender_context,
|
||||
sender_context_set_where,
|
||||
));
|
||||
|
||||
// Prompt override
|
||||
|
||||
let text_agent_prompt = if let Some(text_generation_agent) = &text_generation_agent {
|
||||
|
||||
@@ -15,7 +15,8 @@ use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||
use crate::entity::MessagePayload;
|
||||
use crate::entity::roomconfig::{
|
||||
SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages,
|
||||
TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType,
|
||||
TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType,
|
||||
TextToSpeechUserMessagesFlowType,
|
||||
};
|
||||
use crate::strings;
|
||||
use crate::utils::text_to_speech::create_transcribed_message_text;
|
||||
@@ -23,6 +24,7 @@ use crate::{
|
||||
Bot,
|
||||
conversation::{
|
||||
create_llm_conversation_for_matrix_reply_chain, create_llm_conversation_for_matrix_thread,
|
||||
llm::{Author, Conversation, MessageContent},
|
||||
matrix::create_list_of_bot_user_prefixes_to_strip,
|
||||
},
|
||||
entity::MessageContext,
|
||||
@@ -39,6 +41,10 @@ pub enum ChatCompletionControllerType {
|
||||
|
||||
Audio,
|
||||
|
||||
Image,
|
||||
|
||||
File,
|
||||
|
||||
ThreadMention,
|
||||
ReplyMention,
|
||||
}
|
||||
@@ -416,7 +422,9 @@ async fn handle_stage_text_generation(
|
||||
ChatCompletionControllerType::TextCommand
|
||||
| ChatCompletionControllerType::TextMention
|
||||
| ChatCompletionControllerType::TextDirect
|
||||
| ChatCompletionControllerType::Audio => {
|
||||
| ChatCompletionControllerType::Audio
|
||||
| ChatCompletionControllerType::Image
|
||||
| ChatCompletionControllerType::File => {
|
||||
Some(message_context.combined_admin_and_user_regexes())
|
||||
}
|
||||
|
||||
@@ -438,6 +446,7 @@ async fn handle_stage_text_generation(
|
||||
// When we're triggered via a reply mention, the context is the whole reply chain upward of the message that triggered us.
|
||||
ChatCompletionControllerType::ReplyMention => {
|
||||
create_llm_conversation_for_matrix_reply_chain(
|
||||
&matrix_link,
|
||||
&bot.room_event_fetcher().clone(),
|
||||
message_context.room(),
|
||||
message_context.thread_info().last_event_id.clone(),
|
||||
@@ -449,7 +458,7 @@ async fn handle_stage_text_generation(
|
||||
// Everything else is happening in a thread, so the context is the whole thread.
|
||||
_ => {
|
||||
create_llm_conversation_for_matrix_thread(
|
||||
matrix_link.clone(),
|
||||
&matrix_link,
|
||||
message_context.room(),
|
||||
message_context.thread_info().root_event_id.clone(),
|
||||
¶ms,
|
||||
@@ -479,6 +488,13 @@ async fn handle_stage_text_generation(
|
||||
}
|
||||
};
|
||||
|
||||
let conversation = inject_sender_context(
|
||||
conversation,
|
||||
message_context
|
||||
.room_config_context()
|
||||
.text_generation_sender_context_mode(),
|
||||
);
|
||||
|
||||
tracing::debug!(
|
||||
agent_id = agent.identifier().as_string(),
|
||||
provider = format!("{}", agent.definition().provider.clone()),
|
||||
@@ -754,3 +770,238 @@ async fn generate_and_send_tts_for_message(
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn inject_sender_context(
|
||||
conversation: Conversation,
|
||||
sender_context_mode: TextGenerationSenderContextMode,
|
||||
) -> Conversation {
|
||||
if sender_context_mode == TextGenerationSenderContextMode::Disabled {
|
||||
return conversation;
|
||||
}
|
||||
|
||||
let include_timestamp =
|
||||
sender_context_mode == TextGenerationSenderContextMode::MatrixUserIdAndTimestamp;
|
||||
|
||||
let messages = conversation
|
||||
.messages
|
||||
.into_iter()
|
||||
.map(|mut message| {
|
||||
if message.author == Author::Prompt {
|
||||
return message;
|
||||
}
|
||||
|
||||
let Some(sender_id) = &message.sender_id else {
|
||||
return message;
|
||||
};
|
||||
|
||||
if let MessageContent::Text(ref mut text) = message.content {
|
||||
*text = if include_timestamp {
|
||||
let timestamp = message.timestamp.format("%Y-%m-%dT%H:%M:%SZ");
|
||||
format!("[sender={} sent_at={}] {}", sender_id, timestamp, text)
|
||||
} else {
|
||||
format!("[sender={}] {}", sender_id, text)
|
||||
};
|
||||
}
|
||||
|
||||
message
|
||||
})
|
||||
.collect();
|
||||
|
||||
Conversation { messages }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod sender_context_tests {
|
||||
use super::inject_sender_context;
|
||||
use crate::conversation::llm::{Author, Conversation, ImageDetails, Message, MessageContent};
|
||||
use crate::entity::roomconfig::TextGenerationSenderContextMode;
|
||||
use chrono::{TimeZone, Utc};
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
|
||||
use mxlink::matrix_sdk::ruma::{OwnedMxcUri, OwnedUserId};
|
||||
use mxlink::mime;
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_prefixes_text_messages() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
let user_id = OwnedUserId::try_from("@alice:example.com").unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(user_id),
|
||||
timestamp,
|
||||
content: MessageContent::Text("Hello bot".to_string()),
|
||||
}],
|
||||
};
|
||||
|
||||
let result = inject_sender_context(
|
||||
conversation,
|
||||
TextGenerationSenderContextMode::MatrixUserIdAndTimestamp,
|
||||
);
|
||||
|
||||
assert_eq!(result.messages.len(), 1);
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Text(
|
||||
"[sender=@alice:example.com sent_at=2026-03-23T14:30:00Z] Hello bot".to_string()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_can_prefix_without_timestamp() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
let user_id = OwnedUserId::try_from("@alice:example.com").unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(user_id),
|
||||
timestamp,
|
||||
content: MessageContent::Text("Hello bot".to_string()),
|
||||
}],
|
||||
};
|
||||
|
||||
let result =
|
||||
inject_sender_context(conversation, TextGenerationSenderContextMode::MatrixUserId);
|
||||
|
||||
assert_eq!(result.messages.len(), 1);
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Text("[sender=@alice:example.com] Hello bot".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_prefixes_assistant_messages() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
let user_id = OwnedUserId::try_from("@baibot:example.com").unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::Assistant,
|
||||
sender_id: Some(user_id),
|
||||
timestamp,
|
||||
content: MessageContent::Text("Hello human".to_string()),
|
||||
}],
|
||||
};
|
||||
|
||||
let result = inject_sender_context(
|
||||
conversation,
|
||||
TextGenerationSenderContextMode::MatrixUserIdAndTimestamp,
|
||||
);
|
||||
|
||||
assert_eq!(result.messages.len(), 1);
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Text(
|
||||
"[sender=@baibot:example.com sent_at=2026-03-23T14:30:00Z] Hello human".to_string()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_skips_prompt_messages() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::Prompt,
|
||||
sender_id: None,
|
||||
timestamp,
|
||||
content: MessageContent::Text("You are a bot".to_string()),
|
||||
}],
|
||||
};
|
||||
|
||||
let result =
|
||||
inject_sender_context(conversation, TextGenerationSenderContextMode::MatrixUserId);
|
||||
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Text("You are a bot".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_skips_messages_without_sender_id() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::User,
|
||||
sender_id: None,
|
||||
timestamp,
|
||||
content: MessageContent::Text("Transcribed text".to_string()),
|
||||
}],
|
||||
};
|
||||
|
||||
let result = inject_sender_context(
|
||||
conversation,
|
||||
TextGenerationSenderContextMode::MatrixUserIdAndTimestamp,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Text("Transcribed text".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_leaves_non_text_content_unchanged() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
let user_id = OwnedUserId::try_from("@alice:example.com").unwrap();
|
||||
let image_event_content = ImageMessageEventContent::plain(
|
||||
"image.png".to_string(),
|
||||
OwnedMxcUri::from("mxc://example.com/1234567890"),
|
||||
);
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(user_id),
|
||||
timestamp,
|
||||
content: MessageContent::Image(ImageDetails::new(
|
||||
image_event_content.clone(),
|
||||
mime::IMAGE_PNG,
|
||||
vec![],
|
||||
)),
|
||||
}],
|
||||
};
|
||||
|
||||
let result = inject_sender_context(
|
||||
conversation,
|
||||
TextGenerationSenderContextMode::MatrixUserIdAndTimestamp,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Image(ImageDetails::new(
|
||||
image_event_content,
|
||||
mime::IMAGE_PNG,
|
||||
vec![]
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inject_sender_context_none_leaves_text_unchanged() {
|
||||
let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap();
|
||||
let user_id = OwnedUserId::try_from("@alice:example.com").unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(user_id),
|
||||
timestamp,
|
||||
content: MessageContent::Text("Hello bot".to_string()),
|
||||
}],
|
||||
};
|
||||
|
||||
let result = inject_sender_context(conversation, TextGenerationSenderContextMode::Disabled);
|
||||
|
||||
assert_eq!(
|
||||
result.messages[0].content,
|
||||
MessageContent::Text("Hello bot".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,5 +23,6 @@ pub enum ControllerType {
|
||||
ChatCompletion(super::chat_completion::ChatCompletionControllerType),
|
||||
|
||||
ImageGeneration(String),
|
||||
ImageEdit(String),
|
||||
StickerGeneration(String),
|
||||
}
|
||||
|
||||
@@ -36,6 +36,18 @@ pub fn determine_controller(
|
||||
first_thread_message.is_mentioning_bot,
|
||||
)
|
||||
}
|
||||
MessagePayload::Image(_image_message_content) => {
|
||||
let prefix_requirement_type = message_context
|
||||
.room_config_context()
|
||||
.text_generation_prefix_requirement_type();
|
||||
|
||||
match prefix_requirement_type {
|
||||
TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore,
|
||||
TextGenerationPrefixRequirementType::No => {
|
||||
ControllerType::ChatCompletion(ChatCompletionControllerType::Image)
|
||||
}
|
||||
}
|
||||
}
|
||||
MessagePayload::Encrypted(thread_info) => {
|
||||
if thread_info.is_thread_root_only() {
|
||||
ControllerType::Error(strings::error::message_is_encrypted().to_owned())
|
||||
@@ -46,6 +58,18 @@ pub fn determine_controller(
|
||||
)
|
||||
}
|
||||
}
|
||||
MessagePayload::File(_file_message_content) => {
|
||||
let prefix_requirement_type = message_context
|
||||
.room_config_context()
|
||||
.text_generation_prefix_requirement_type();
|
||||
|
||||
match prefix_requirement_type {
|
||||
TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore,
|
||||
TextGenerationPrefixRequirementType::No => {
|
||||
ControllerType::ChatCompletion(ChatCompletionControllerType::File)
|
||||
}
|
||||
}
|
||||
}
|
||||
MessagePayload::Audio(_) => {
|
||||
ControllerType::ChatCompletion(ChatCompletionControllerType::Audio)
|
||||
}
|
||||
@@ -84,7 +108,7 @@ fn determine_text_controller(
|
||||
}
|
||||
|
||||
if let Some(prompt) = text.strip_prefix(&format!("{command_prefix} image")) {
|
||||
return ControllerType::ImageGeneration(prompt.trim().to_owned());
|
||||
return super::image::determine_controller(prompt.trim());
|
||||
}
|
||||
|
||||
if let Some(prompt) = text.strip_prefix(&format!("{command_prefix} sticker")) {
|
||||
|
||||
@@ -84,9 +84,17 @@ fn determine_text_controller() {
|
||||
expected: ControllerType::Config(controller::cfg::ConfigControllerType::Help),
|
||||
},
|
||||
TestCase {
|
||||
name: "Image generation",
|
||||
name: "Generic image command causes usage help",
|
||||
input: "!bai image Draw a cat!",
|
||||
is_mentioning_bot: false,
|
||||
room_text_generation_prefix_requirement_type:
|
||||
super::TextGenerationPrefixRequirementType::No,
|
||||
expected: ControllerType::UsageHelp,
|
||||
},
|
||||
TestCase {
|
||||
name: "Image generation",
|
||||
input: "!bai image create Draw a cat!",
|
||||
is_mentioning_bot: false,
|
||||
room_text_generation_prefix_requirement_type:
|
||||
super::TextGenerationPrefixRequirementType::No,
|
||||
expected: ControllerType::ImageGeneration("Draw a cat!".to_owned()),
|
||||
|
||||
@@ -77,6 +77,10 @@ pub async fn dispatch_controller(
|
||||
)
|
||||
.await
|
||||
}
|
||||
ControllerType::ImageEdit(prompt) => {
|
||||
super::image::edit::handle(bot, bot.matrix_link().clone(), message_context, prompt)
|
||||
.await
|
||||
}
|
||||
ControllerType::StickerGeneration(prompt) => {
|
||||
super::image::generation::handle_sticker(
|
||||
bot,
|
||||
|
||||
16
src/controller/image/determination/mod.rs
Normal file
16
src/controller/image/determination/mod.rs
Normal file
@@ -0,0 +1,16 @@
|
||||
use crate::controller::ControllerType;
|
||||
mod tests;
|
||||
|
||||
pub fn determine_controller(text: &str) -> ControllerType {
|
||||
let text = text.trim();
|
||||
|
||||
if let Some(prompt) = text.strip_prefix("create") {
|
||||
return ControllerType::ImageGeneration(prompt.trim().to_owned());
|
||||
}
|
||||
|
||||
if let Some(prompt) = text.strip_prefix("edit") {
|
||||
return ControllerType::ImageEdit(prompt.trim().to_owned());
|
||||
}
|
||||
|
||||
ControllerType::UsageHelp
|
||||
}
|
||||
38
src/controller/image/determination/tests.rs
Normal file
38
src/controller/image/determination/tests.rs
Normal file
@@ -0,0 +1,38 @@
|
||||
#[test]
|
||||
fn determine_controller() {
|
||||
struct TestCase {
|
||||
name: &'static str,
|
||||
input: &'static str,
|
||||
expected: super::ControllerType,
|
||||
}
|
||||
|
||||
let test_cases = vec![
|
||||
TestCase {
|
||||
name: "Top-level is usage help",
|
||||
input: "",
|
||||
expected: super::ControllerType::UsageHelp,
|
||||
},
|
||||
TestCase {
|
||||
name: "Top-level with some text is usage help",
|
||||
input: "Some text",
|
||||
expected: super::ControllerType::UsageHelp,
|
||||
},
|
||||
TestCase {
|
||||
name: "Image generation triggered by create prefix",
|
||||
input: "create Some prompt",
|
||||
expected: super::ControllerType::ImageGeneration("Some prompt".to_owned()),
|
||||
},
|
||||
TestCase {
|
||||
name: "Image edit triggered by edit prefix",
|
||||
input: "edit Turn this into an anime-style image",
|
||||
expected: super::ControllerType::ImageEdit(
|
||||
"Turn this into an anime-style image".to_owned(),
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
for test_case in test_cases {
|
||||
let result = super::determine_controller(test_case.input);
|
||||
assert_eq!(result, test_case.expected, "Test case: {}", test_case.name);
|
||||
}
|
||||
}
|
||||
162
src/controller/image/edit.rs
Normal file
162
src/controller/image/edit.rs
Normal file
@@ -0,0 +1,162 @@
|
||||
use mxlink::{MatrixLink, MessageResponseType};
|
||||
|
||||
use tracing::Instrument;
|
||||
|
||||
use crate::agent::AgentPurpose;
|
||||
use crate::agent::ControllerTrait;
|
||||
use crate::agent::provider::ImageEditParams;
|
||||
use crate::agent::provider::ImageSource;
|
||||
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
||||
use crate::conversation::create_llm_conversation_for_matrix_thread;
|
||||
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||
use crate::strings;
|
||||
use crate::utils::mime::get_file_extension;
|
||||
use crate::{Bot, entity::MessageContext};
|
||||
|
||||
pub async fn handle(
|
||||
bot: &Bot,
|
||||
matrix_link: MatrixLink,
|
||||
message_context: &MessageContext,
|
||||
original_prompt: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let response_type = MessageResponseType::InThread(message_context.thread_info().clone());
|
||||
|
||||
let Some(agent) = get_effective_agent_for_purpose_or_complain(
|
||||
bot,
|
||||
message_context,
|
||||
AgentPurpose::ImageGeneration,
|
||||
response_type.clone(),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if message_context.thread_info().is_thread_root_only() {
|
||||
return send_guide(bot, message_context).await;
|
||||
}
|
||||
|
||||
let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await;
|
||||
|
||||
let params = MatrixMessageProcessingParams::new(
|
||||
bot.user_id().to_owned(),
|
||||
Some(message_context.combined_admin_and_user_regexes()),
|
||||
);
|
||||
|
||||
let conversation = create_llm_conversation_for_matrix_thread(
|
||||
&matrix_link,
|
||||
message_context.room(),
|
||||
message_context.thread_info().root_event_id.clone(),
|
||||
¶ms,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let prompt = if conversation.messages.len() >= 2 {
|
||||
// Skip the first message, which contains the original prompt (which we already have)
|
||||
let other_messages = conversation.messages.iter().skip(1).cloned().collect();
|
||||
|
||||
super::prompt::build(original_prompt, other_messages)
|
||||
} else {
|
||||
original_prompt.to_owned()
|
||||
};
|
||||
|
||||
let got_go_signal = conversation.messages.iter().any(|message| {
|
||||
if let crate::conversation::llm::MessageContent::Text(text) = &message.content {
|
||||
text.to_lowercase() == "go"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
|
||||
let image_sources: Vec<ImageSource> = conversation
|
||||
.messages
|
||||
.iter()
|
||||
.filter_map(|message| {
|
||||
if let crate::conversation::llm::MessageContent::Image(image_content) = &message.content
|
||||
{
|
||||
Some(image_content.clone().into())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if !got_go_signal || image_sources.is_empty() {
|
||||
// We don't send the guide again here to avoid being annoying.
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let span = tracing::debug_span!("image_edit", agent_id = agent.identifier().as_string());
|
||||
|
||||
let result = agent
|
||||
.controller()
|
||||
.create_image_edit(&prompt, image_sources, ImageEditParams::default())
|
||||
.instrument(span)
|
||||
.await;
|
||||
|
||||
let response = match result {
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
"Error in room {} while trying to generate image edit via agent {}: {:?}",
|
||||
message_context.room_id(),
|
||||
agent.identifier(),
|
||||
err,
|
||||
);
|
||||
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::agent::error_while_serving_purpose(
|
||||
agent.identifier(),
|
||||
&AgentPurpose::ImageGeneration,
|
||||
&err,
|
||||
),
|
||||
response_type,
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let attachment_body_text = format!(
|
||||
"generated-image-edit.{}",
|
||||
get_file_extension(&response.mime_type)
|
||||
);
|
||||
|
||||
let mut event_content = matrix_link
|
||||
.media()
|
||||
.upload_and_prepare_event_content(
|
||||
message_context.room(),
|
||||
&response.mime_type,
|
||||
response.bytes,
|
||||
&attachment_body_text,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to upload and prepare event: {}", e))?;
|
||||
|
||||
matrix_link
|
||||
.messaging()
|
||||
.send_event(
|
||||
message_context.room(),
|
||||
&mut event_content,
|
||||
response_type.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_guide(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
|
||||
bot.messaging()
|
||||
.send_text_markdown_no_fail(
|
||||
message_context.room(),
|
||||
strings::image_edit::guide_how_to_proceed(),
|
||||
MessageResponseType::InThread(message_context.thread_info().clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -6,15 +6,12 @@ use crate::agent::AgentPurpose;
|
||||
use crate::agent::ControllerTrait;
|
||||
use crate::agent::provider::ImageGenerationParams;
|
||||
use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
|
||||
use crate::controller::utils::mime::get_file_extension;
|
||||
use crate::conversation::create_llm_conversation_for_matrix_thread;
|
||||
use crate::conversation::matrix::MatrixMessageProcessingParams;
|
||||
use crate::strings;
|
||||
use crate::utils::mime::get_file_extension;
|
||||
use crate::{Bot, entity::MessageContext};
|
||||
|
||||
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
|
||||
const STICKER_SIZE: &str = "256x256";
|
||||
|
||||
pub async fn handle_image(
|
||||
bot: &Bot,
|
||||
matrix_link: MatrixLink,
|
||||
@@ -43,7 +40,7 @@ pub async fn handle_image(
|
||||
);
|
||||
|
||||
let conversation = create_llm_conversation_for_matrix_thread(
|
||||
matrix_link.clone(),
|
||||
&matrix_link,
|
||||
message_context.room(),
|
||||
message_context.thread_info().root_event_id.clone(),
|
||||
¶ms,
|
||||
@@ -64,11 +61,37 @@ pub async fn handle_image(
|
||||
agent_id = agent.identifier().as_string()
|
||||
);
|
||||
|
||||
let response = agent
|
||||
let result = agent
|
||||
.controller()
|
||||
.generate_image(&prompt, ImageGenerationParams::default())
|
||||
.instrument(span)
|
||||
.await?;
|
||||
.await;
|
||||
|
||||
let response = match result {
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
"Error in room {} while trying to generate image via agent {}: {:?}",
|
||||
message_context.room_id(),
|
||||
agent.identifier(),
|
||||
err,
|
||||
);
|
||||
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::agent::error_while_serving_purpose(
|
||||
agent.identifier(),
|
||||
&AgentPurpose::ImageGeneration,
|
||||
&err,
|
||||
),
|
||||
response_type,
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let actual_prompt = response.revised_prompt.as_deref().unwrap_or(&prompt);
|
||||
|
||||
@@ -151,15 +174,41 @@ pub async fn handle_sticker(
|
||||
);
|
||||
|
||||
let params = ImageGenerationParams::default()
|
||||
.with_size_override(Some(STICKER_SIZE.to_owned()))
|
||||
.with_smallest_size_possible(true)
|
||||
.with_cheaper_model_switching_allowed(true)
|
||||
.with_cheaper_quality_switching_allowed(true);
|
||||
|
||||
let response = agent
|
||||
let result = agent
|
||||
.controller()
|
||||
.generate_image(original_prompt, params)
|
||||
.instrument(span)
|
||||
.await?;
|
||||
.await;
|
||||
|
||||
let response = match result {
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
"Error in room {} while trying to generate sticker via agent {}: {:?}",
|
||||
message_context.room_id(),
|
||||
agent.identifier(),
|
||||
err,
|
||||
);
|
||||
|
||||
bot.messaging()
|
||||
.send_error_markdown_no_fail(
|
||||
message_context.room(),
|
||||
&strings::agent::error_while_serving_purpose(
|
||||
agent.identifier(),
|
||||
&AgentPurpose::ImageGeneration,
|
||||
&err,
|
||||
),
|
||||
response_type,
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let attachment_body_text = format!(
|
||||
"generated-sticker.{}",
|
||||
|
||||
@@ -1,2 +1,6 @@
|
||||
mod determination;
|
||||
pub mod edit;
|
||||
pub mod generation;
|
||||
mod prompt;
|
||||
|
||||
pub use determination::determine_controller;
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use crate::conversation::llm::{Author, Message};
|
||||
use crate::conversation::llm::{Author, Message, MessageContent};
|
||||
|
||||
/// Builds a prompt from the original prompt and other messages in the conversation.
|
||||
///
|
||||
/// Only messages authored by the user are considered.
|
||||
///
|
||||
/// Messages that say "Again" (regardless of casing) are ignored. They are considered special messages
|
||||
/// which trigger re-generation, but do not need to be included in the prompt criteria.
|
||||
/// Messages that say "Again" or "Go" (regardless of casing) are ignored. They are considered special messages
|
||||
/// which trigger re-generation and "start" respectively, and do not need to be included in the prompt criteria.
|
||||
pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
|
||||
let mut prompt = original_prompt.to_owned();
|
||||
|
||||
@@ -14,7 +14,11 @@ pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
|
||||
.into_iter()
|
||||
.filter(|message| {
|
||||
if let Author::User = message.author {
|
||||
message.message_text.to_lowercase() != "again"
|
||||
if let MessageContent::Text(text) = &message.content {
|
||||
text.to_lowercase() != "again" && text.to_lowercase() != "go"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
@@ -24,9 +28,9 @@ pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
|
||||
if !other_messages.is_empty() {
|
||||
prompt.push_str("\nOther criteria:");
|
||||
for message in other_messages {
|
||||
prompt.push_str(
|
||||
format!("\n- {}", message.message_text.replace("\n", ". ").as_str()).as_str(),
|
||||
);
|
||||
if let MessageContent::Text(text) = &message.content {
|
||||
prompt.push_str(format!("\n- {}", text.replace("\n", ". ").as_str()).as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +40,7 @@ pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::build;
|
||||
use super::{Author, Message};
|
||||
use super::{Author, Message, MessageContent};
|
||||
|
||||
struct TestCase {
|
||||
original_prompt: &'static str,
|
||||
@@ -60,7 +64,8 @@ mod tests {
|
||||
original_prompt: "Generate a picture of a dog",
|
||||
messages: vec![Message {
|
||||
author: Author::User,
|
||||
message_text: "Must be blue".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Must be blue".to_owned()),
|
||||
timestamp,
|
||||
}],
|
||||
expected_prompt: "Generate a picture of a dog\nOther criteria:\n- Must be blue",
|
||||
@@ -71,17 +76,22 @@ mod tests {
|
||||
messages: vec![
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "Must be blue".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Must be blue".to_owned()),
|
||||
timestamp,
|
||||
},
|
||||
Message {
|
||||
author: Author::Assistant,
|
||||
message_text: "Whatever".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Whatever".to_owned()),
|
||||
timestamp,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "Must be 3-legged.\nMust be flying.".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text(
|
||||
"Must be 3-legged.\nMust be flying.".to_owned(),
|
||||
),
|
||||
timestamp,
|
||||
},
|
||||
],
|
||||
@@ -93,22 +103,26 @@ mod tests {
|
||||
messages: vec![
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "Must be blue".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Must be blue".to_owned()),
|
||||
timestamp,
|
||||
},
|
||||
Message {
|
||||
author: Author::Assistant,
|
||||
message_text: "Whatever".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Whatever".to_owned()),
|
||||
timestamp,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "Again".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Again".to_owned()),
|
||||
timestamp,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "again".to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("again".to_owned()),
|
||||
timestamp,
|
||||
},
|
||||
],
|
||||
|
||||
@@ -6,7 +6,6 @@ use crate::{
|
||||
};
|
||||
|
||||
pub mod agent;
|
||||
pub(super) mod mime;
|
||||
pub mod text_to_speech;
|
||||
|
||||
pub async fn get_text_body_or_complain<'a>(
|
||||
|
||||
@@ -3,7 +3,7 @@ use mxlink::{MatrixLink, MessageResponseType};
|
||||
|
||||
use tracing::Instrument;
|
||||
|
||||
use crate::controller::utils::mime::get_file_extension;
|
||||
use crate::utils::mime::get_file_extension;
|
||||
use crate::{
|
||||
Bot,
|
||||
agent::{AgentInstance, AgentPurpose, ControllerTrait, provider::TextToSpeechParams},
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use mxlink::matrix_sdk::ruma::OwnedUserId;
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::{
|
||||
FileMessageEventContent, ImageMessageEventContent,
|
||||
};
|
||||
use mxlink::mime::Mime;
|
||||
|
||||
use crate::agent::provider::ImageSource;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Author {
|
||||
@@ -10,10 +17,85 @@ pub enum Author {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Message {
|
||||
pub author: Author,
|
||||
pub message_text: String,
|
||||
pub sender_id: Option<OwnedUserId>,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub content: MessageContent,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImageDetails {
|
||||
pub event_content: ImageMessageEventContent,
|
||||
pub mime: Mime,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl ImageDetails {
|
||||
pub fn new(event_content: ImageMessageEventContent, mime: Mime, data: Vec<u8>) -> Self {
|
||||
Self {
|
||||
event_content,
|
||||
mime,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn filename(&self) -> String {
|
||||
self.event_content
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or(self.event_content.body.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ImageDetails> for ImageSource {
|
||||
fn from(value: ImageDetails) -> Self {
|
||||
ImageSource::new(value.filename(), value.data.clone(), value.mime.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileDetails {
|
||||
pub event_content: FileMessageEventContent,
|
||||
pub mime: Mime,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl FileDetails {
|
||||
pub fn new(event_content: FileMessageEventContent, mime: Mime, data: Vec<u8>) -> Self {
|
||||
Self {
|
||||
event_content,
|
||||
mime,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn filename(&self) -> String {
|
||||
self.event_content
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or(self.event_content.body.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Image(ImageDetails),
|
||||
File(FileDetails),
|
||||
}
|
||||
|
||||
impl PartialEq for MessageContent {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self, other) {
|
||||
(MessageContent::Text(a), MessageContent::Text(b)) => a == b,
|
||||
(MessageContent::Image(a), MessageContent::Image(b)) => {
|
||||
// We can probably do better than this by inspecting `.event_conten1t.source`, but for now this is good enough.
|
||||
a.filename() == b.filename()
|
||||
}
|
||||
(MessageContent::File(a), MessageContent::File(b)) => a.filename() == b.filename(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct Conversation {
|
||||
pub messages: Vec<Message>,
|
||||
@@ -24,31 +106,45 @@ impl Conversation {
|
||||
///
|
||||
/// Certain models (like Anthropic) cannot tolerate consecutive messages by the same author,
|
||||
/// so combining them helps avoid issues.
|
||||
///
|
||||
/// When multiple text messages by the same author are merged, the resulting message keeps a
|
||||
/// `sender_id` only if all merged messages came from the same sender. Mixed-sender merges are
|
||||
/// possible for user turns in multi-user rooms, so `sender_id` is cleared in that case to
|
||||
/// avoid incorrectly attributing the whole merged turn to the first sender.
|
||||
/// See: https://github.com/etkecc/baibot/issues/13
|
||||
pub fn combine_consecutive_messages(&self) -> Conversation {
|
||||
// We'll likely get fewer messages, but let's reserve the maximum we expect.
|
||||
let mut new_messages = Vec::with_capacity(self.messages.len());
|
||||
let mut last_seen_author: Option<Author> = None;
|
||||
let mut last_seen_text_from_author: Option<Author> = None;
|
||||
|
||||
for message in &self.messages {
|
||||
let Some(last_seen_author_clone) = last_seen_author.clone() else {
|
||||
last_seen_author = Some(message.author.clone());
|
||||
let MessageContent::Text(message_text_content) = &message.content else {
|
||||
last_seen_text_from_author = None;
|
||||
new_messages.push(message.clone());
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(last_seen_author_clone) = last_seen_text_from_author.clone() else {
|
||||
last_seen_text_from_author = Some(message.author.clone());
|
||||
new_messages.push(message.clone());
|
||||
continue;
|
||||
};
|
||||
|
||||
if message.author != last_seen_author_clone {
|
||||
last_seen_author = Some(message.author.clone());
|
||||
last_seen_text_from_author = Some(message.author.clone());
|
||||
new_messages.push(message.clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
new_messages.last_mut().unwrap().message_text.push('\n');
|
||||
new_messages
|
||||
.last_mut()
|
||||
.unwrap()
|
||||
.message_text
|
||||
.push_str(&message.message_text);
|
||||
let last_message = new_messages.last_mut().unwrap();
|
||||
if let MessageContent::Text(ref mut text) = last_message.content {
|
||||
text.push('\n');
|
||||
text.push_str(message_text_content);
|
||||
}
|
||||
|
||||
if last_message.sender_id != message.sender_id {
|
||||
last_message.sender_id = None;
|
||||
}
|
||||
}
|
||||
|
||||
Conversation {
|
||||
@@ -65,48 +161,85 @@ impl Conversation {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::{TimeZone, Utc};
|
||||
use mxlink::matrix_sdk::ruma::{OwnedMxcUri, OwnedUserId};
|
||||
use mxlink::mime;
|
||||
|
||||
#[test]
|
||||
fn combine_consecutive_messages() {
|
||||
let timestamp_1 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap();
|
||||
|
||||
let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 21, 18, 34, 15).unwrap();
|
||||
let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 21, 18, 34, 16).unwrap();
|
||||
|
||||
let timestamp_3 = Utc.with_ymd_and_hms(2024, 9, 22, 18, 34, 15).unwrap();
|
||||
let timestamp_3 = Utc.with_ymd_and_hms(2024, 9, 22, 18, 34, 17).unwrap();
|
||||
|
||||
let timestamp_4 = Utc.with_ymd_and_hms(2024, 9, 23, 18, 34, 18).unwrap();
|
||||
|
||||
let image_event_content = ImageMessageEventContent::plain(
|
||||
"image.png".to_string(),
|
||||
OwnedMxcUri::from("mxc://example.com/1234567890"),
|
||||
);
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![
|
||||
// User's turn
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "Hello".to_string(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Hello".to_string()),
|
||||
timestamp: timestamp_1,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "How are you?".to_string(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("How are you?".to_string()),
|
||||
timestamp: timestamp_2,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "I'm OK, btw.".to_string(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("I'm OK, btw.".to_string()),
|
||||
timestamp: timestamp_3,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
sender_id: None,
|
||||
content: MessageContent::Image(ImageDetails::new(
|
||||
image_event_content.clone(),
|
||||
mime::IMAGE_PNG,
|
||||
vec![],
|
||||
)),
|
||||
timestamp: timestamp_4,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Above is an image.".to_string()),
|
||||
timestamp: timestamp_4,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Would you take a look at it?".to_string()),
|
||||
timestamp: timestamp_4,
|
||||
},
|
||||
// Assistant's turn
|
||||
Message {
|
||||
author: Author::Assistant,
|
||||
message_text: "Hi there!".to_string(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("Hi there!".to_string()),
|
||||
timestamp: timestamp_2,
|
||||
},
|
||||
Message {
|
||||
author: Author::Assistant,
|
||||
message_text: "I'm doing well, thank you.".to_string(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("I'm doing well, thank you.".to_string()),
|
||||
timestamp: timestamp_3,
|
||||
},
|
||||
// User's turn
|
||||
Message {
|
||||
author: Author::User,
|
||||
message_text: "That's great!".to_string(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text("That's great!".to_string()),
|
||||
timestamp: timestamp_3,
|
||||
},
|
||||
],
|
||||
@@ -114,23 +247,79 @@ mod tests {
|
||||
|
||||
let conversation = conversation.combine_consecutive_messages();
|
||||
|
||||
assert_eq!(conversation.messages.len(), 3);
|
||||
assert_eq!(conversation.messages.len(), 5);
|
||||
|
||||
assert_eq!(conversation.messages[0].author, Author::User);
|
||||
assert_eq!(
|
||||
conversation.messages[0].message_text,
|
||||
"Hello\nHow are you?\nI'm OK, btw."
|
||||
conversation.messages[0].content,
|
||||
MessageContent::Text("Hello\nHow are you?\nI'm OK, btw.".to_string())
|
||||
);
|
||||
assert_eq!(conversation.messages[0].timestamp, timestamp_1);
|
||||
|
||||
assert_eq!(conversation.messages[1].author, Author::Assistant);
|
||||
assert_eq!(conversation.messages[1].author, Author::User);
|
||||
assert_eq!(
|
||||
conversation.messages[1].message_text,
|
||||
"Hi there!\nI'm doing well, thank you."
|
||||
conversation.messages[1].content,
|
||||
MessageContent::Image(ImageDetails::new(
|
||||
image_event_content.clone(),
|
||||
mime::IMAGE_PNG,
|
||||
vec![],
|
||||
))
|
||||
);
|
||||
assert_eq!(conversation.messages[1].timestamp, timestamp_2);
|
||||
|
||||
assert_eq!(conversation.messages[2].author, Author::User);
|
||||
assert_eq!(conversation.messages[2].message_text, "That's great!");
|
||||
assert_eq!(conversation.messages[2].timestamp, timestamp_3);
|
||||
assert_eq!(
|
||||
conversation.messages[2].content,
|
||||
MessageContent::Text("Above is an image.\nWould you take a look at it?".to_string())
|
||||
);
|
||||
assert_eq!(conversation.messages[2].timestamp, timestamp_4);
|
||||
|
||||
assert_eq!(conversation.messages[3].author, Author::Assistant);
|
||||
assert_eq!(
|
||||
conversation.messages[3].content,
|
||||
MessageContent::Text("Hi there!\nI'm doing well, thank you.".to_string())
|
||||
);
|
||||
assert_eq!(conversation.messages[3].timestamp, timestamp_2);
|
||||
|
||||
assert_eq!(conversation.messages[4].author, Author::User);
|
||||
assert_eq!(
|
||||
conversation.messages[4].content,
|
||||
MessageContent::Text("That's great!".to_string())
|
||||
);
|
||||
assert_eq!(conversation.messages[4].timestamp, timestamp_3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn combine_consecutive_messages_clears_sender_id_for_mixed_sender_turns() {
|
||||
let timestamp_1 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap();
|
||||
let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 16).unwrap();
|
||||
let sender_1 = OwnedUserId::try_from("@alice:example.com").unwrap();
|
||||
let sender_2 = OwnedUserId::try_from("@bob:example.com").unwrap();
|
||||
|
||||
let conversation = Conversation {
|
||||
messages: vec![
|
||||
Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(sender_1),
|
||||
content: MessageContent::Text("Hello".to_string()),
|
||||
timestamp: timestamp_1,
|
||||
},
|
||||
Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(sender_2),
|
||||
content: MessageContent::Text("Hi there".to_string()),
|
||||
timestamp: timestamp_2,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let conversation = conversation.combine_consecutive_messages();
|
||||
|
||||
assert_eq!(conversation.messages.len(), 1);
|
||||
assert_eq!(conversation.messages[0].sender_id, None);
|
||||
assert_eq!(
|
||||
conversation.messages[0].content,
|
||||
MessageContent::Text("Hello\nHi there".to_string())
|
||||
);
|
||||
assert_eq!(conversation.messages[0].timestamp, timestamp_1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,8 +12,7 @@ fn test_messages_by_the_bot_are_identified_correctly() {
|
||||
|
||||
let matrix_message = super::super::matrix::MatrixMessage {
|
||||
sender_id: bot_user_id.to_owned(),
|
||||
message_type: super::super::matrix::MatrixMessageType::Text,
|
||||
message_text: "Hello!".to_owned(),
|
||||
content: super::super::matrix::MatrixMessageContent::Text("Hello!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
@@ -21,7 +20,11 @@ fn test_messages_by_the_bot_are_identified_correctly() {
|
||||
let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap();
|
||||
|
||||
assert_eq!(llm_message.author, Author::Assistant);
|
||||
assert_eq!(llm_message.message_text, "Hello!");
|
||||
assert_eq!(llm_message.sender_id, Some(bot_user_id.clone()));
|
||||
assert_eq!(
|
||||
llm_message.content,
|
||||
MessageContent::Text("Hello!".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -37,8 +40,7 @@ fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_con
|
||||
|
||||
let matrix_message = super::super::matrix::MatrixMessage {
|
||||
sender_id: bot_user_id.to_owned(),
|
||||
message_type: super::super::matrix::MatrixMessageType::Notice,
|
||||
message_text,
|
||||
content: super::super::matrix::MatrixMessageContent::Notice(message_text),
|
||||
mentioned_users: vec![],
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
@@ -46,7 +48,11 @@ fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_con
|
||||
let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap();
|
||||
|
||||
assert_eq!(llm_message.author, Author::User);
|
||||
assert_eq!(llm_message.message_text, source_message_text);
|
||||
assert_eq!(llm_message.sender_id, None);
|
||||
assert_eq!(
|
||||
llm_message.content,
|
||||
MessageContent::Text(source_message_text.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -61,8 +67,7 @@ fn test_notice_error_messages_by_bot_are_ignored() {
|
||||
|
||||
let matrix_message = super::super::matrix::MatrixMessage {
|
||||
sender_id: bot_user_id.to_owned(),
|
||||
message_type: super::super::matrix::MatrixMessageType::Notice,
|
||||
message_text,
|
||||
content: super::super::matrix::MatrixMessageContent::Notice(message_text),
|
||||
mentioned_users: vec![],
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
@@ -72,6 +77,30 @@ fn test_notice_error_messages_by_bot_are_ignored() {
|
||||
assert!(llm_message.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_user_messages_preserve_sender_id() {
|
||||
let bot_user_id =
|
||||
OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID");
|
||||
|
||||
let user_id = OwnedUserId::try_from("@alice:example.com").expect("Failed to parse user ID");
|
||||
|
||||
let matrix_message = super::super::matrix::MatrixMessage {
|
||||
sender_id: user_id.clone(),
|
||||
content: super::super::matrix::MatrixMessageContent::Text("Hello!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap();
|
||||
|
||||
assert_eq!(llm_message.author, Author::User);
|
||||
assert_eq!(llm_message.sender_id, Some(user_id));
|
||||
assert_eq!(
|
||||
llm_message.content,
|
||||
MessageContent::Text("Hello!".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_other_notice_messages_by_the_bot_are_ignored() {
|
||||
// Also see `test_notice_error_messages_by_bot_are_ignored()`.
|
||||
@@ -86,8 +115,7 @@ fn test_other_notice_messages_by_the_bot_are_ignored() {
|
||||
|
||||
let matrix_message = super::super::matrix::MatrixMessage {
|
||||
sender_id: bot_user_id.to_owned(),
|
||||
message_type: super::super::matrix::MatrixMessageType::Notice,
|
||||
message_text: message_text.to_owned(),
|
||||
content: super::super::matrix::MatrixMessageContent::Notice(message_text.to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
@@ -2,7 +2,7 @@ use tiktoken_rs::CoreBPE;
|
||||
use tiktoken_rs::get_bpe_from_tokenizer;
|
||||
use tiktoken_rs::tokenizer;
|
||||
|
||||
use super::{Author, Message};
|
||||
use super::{Author, Message, MessageContent};
|
||||
|
||||
fn get_bpe_for_model(model: &str) -> CoreBPE {
|
||||
let tokenizer = tokenizer::get_tokenizer(model)
|
||||
@@ -71,7 +71,11 @@ fn calculate_token_size_for_message(bpe: &CoreBPE, model: &str, message: &Messag
|
||||
Author::Prompt => bpe.encode_with_special_tokens("system").len() as i32,
|
||||
};
|
||||
|
||||
let text_length = bpe.encode_with_special_tokens(&message.message_text).len() as i32;
|
||||
let text_length = match &message.content {
|
||||
MessageContent::Text(text) => bpe.encode_with_special_tokens(text).len() as i32,
|
||||
MessageContent::Image(..) => 0,
|
||||
MessageContent::File(..) => 0,
|
||||
};
|
||||
|
||||
(text_length + role_length + tokens_per_message + tokens_per_name) as u32
|
||||
}
|
||||
@@ -85,7 +89,8 @@ pub mod test {
|
||||
|
||||
let message = super::Message {
|
||||
author: super::Author::User,
|
||||
message_text: "Hello there!".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("Hello there!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
@@ -104,7 +109,8 @@ pub mod test {
|
||||
|
||||
let prompt = super::Message {
|
||||
author: super::Author::Prompt,
|
||||
message_text: "You are a bot!".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("You are a bot!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let prompt_length = 10;
|
||||
@@ -118,7 +124,8 @@ pub mod test {
|
||||
|
||||
let first = super::Message {
|
||||
author: super::Author::User,
|
||||
message_text: "Hello there!".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("Hello there!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let first_length = 8;
|
||||
@@ -132,7 +139,8 @@ pub mod test {
|
||||
|
||||
let second = super::Message {
|
||||
author: super::Author::Assistant,
|
||||
message_text: "Hello!".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("Hello!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let second_length = 7;
|
||||
@@ -146,8 +154,10 @@ pub mod test {
|
||||
|
||||
let third = super::Message {
|
||||
author: super::Author::User,
|
||||
message_text: "This is the 3rd message in this conversation. It shall be preserved."
|
||||
.to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text(
|
||||
"This is the 3rd message in this conversation. It shall be preserved.".to_owned(),
|
||||
),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let third_length = 21;
|
||||
@@ -161,7 +171,10 @@ pub mod test {
|
||||
|
||||
let forth = super::Message {
|
||||
author: super::Author::Assistant,
|
||||
message_text: "This is yet another message that shall be preserved.".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text(
|
||||
"This is yet another message that shall be preserved.".to_owned(),
|
||||
),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let forth_length = 15;
|
||||
@@ -186,13 +199,13 @@ pub mod test {
|
||||
assert_eq!(2, new_conversation_messages.len());
|
||||
|
||||
assert_eq!(
|
||||
new_conversation_messages.first().unwrap().message_text,
|
||||
third.message_text
|
||||
new_conversation_messages.first().unwrap().content,
|
||||
third.content
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
new_conversation_messages.last().unwrap().message_text,
|
||||
forth.message_text
|
||||
new_conversation_messages.last().unwrap().content,
|
||||
forth.content
|
||||
);
|
||||
}
|
||||
|
||||
@@ -206,7 +219,8 @@ pub mod test {
|
||||
|
||||
let prompt = super::Message {
|
||||
author: super::Author::User,
|
||||
message_text: "あなたはボットです。".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("あなたはボットです。".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let prompt_length = 14;
|
||||
@@ -220,7 +234,8 @@ pub mod test {
|
||||
|
||||
let first = super::Message {
|
||||
author: super::Author::User,
|
||||
message_text: "こんにちは!".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("こんにちは!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let first_length = 7;
|
||||
@@ -234,7 +249,8 @@ pub mod test {
|
||||
|
||||
let second = super::Message {
|
||||
author: super::Author::Assistant,
|
||||
message_text: "こんにちは。今日は元気ですか。".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text("こんにちは。今日は元気ですか。".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let second_length = 15;
|
||||
@@ -248,7 +264,10 @@ pub mod test {
|
||||
|
||||
let third = super::Message {
|
||||
author: super::Author::User,
|
||||
message_text: "これは第3のメッセージなので、保存されます。".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text(
|
||||
"これは第3のメッセージなので、保存されます。".to_string(),
|
||||
),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let third_length = 22;
|
||||
@@ -262,7 +281,10 @@ pub mod test {
|
||||
|
||||
let forth = super::Message {
|
||||
author: super::Author::Assistant,
|
||||
message_text: "これはもう一つの保存されますメッセージです。".to_owned(),
|
||||
sender_id: None,
|
||||
content: super::MessageContent::Text(
|
||||
"これはもう一つの保存されますメッセージです。".to_string(),
|
||||
),
|
||||
timestamp: chrono::Utc::now(),
|
||||
};
|
||||
let forth_length = 21;
|
||||
@@ -287,13 +309,13 @@ pub mod test {
|
||||
assert_eq!(2, new_conversation_messages.len());
|
||||
|
||||
assert_eq!(
|
||||
new_conversation_messages.first().unwrap().message_text,
|
||||
third.message_text
|
||||
new_conversation_messages.first().unwrap().content,
|
||||
third.content
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
new_conversation_messages.last().unwrap().message_text,
|
||||
forth.message_text
|
||||
new_conversation_messages.last().unwrap().content,
|
||||
forth.content
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use matrix_sdk::ruma::OwnedUserId;
|
||||
use mxlink::matrix_sdk::ruma::OwnedUserId;
|
||||
|
||||
use super::{Author, Message};
|
||||
use crate::conversation::matrix::{MatrixMessage, MatrixMessageType};
|
||||
use super::entity::{Author, FileDetails, ImageDetails, Message, MessageContent};
|
||||
use crate::conversation::matrix::{MatrixMessage, MatrixMessageContent};
|
||||
use crate::utils::text_to_speech as text_to_speech_utils;
|
||||
|
||||
pub fn convert_matrix_message_to_llm_message(
|
||||
@@ -16,23 +16,47 @@ pub fn convert_matrix_message_to_llm_message(
|
||||
}
|
||||
|
||||
fn convert_bot_message(matrix_message: &MatrixMessage) -> Option<Message> {
|
||||
match matrix_message.message_type {
|
||||
MatrixMessageType::Text => {
|
||||
convert_bot_text_message(&matrix_message.message_text, &matrix_message.timestamp)
|
||||
}
|
||||
MatrixMessageType::Notice => {
|
||||
convert_bot_notice_message(&matrix_message.message_text, &matrix_message.timestamp)
|
||||
match &matrix_message.content {
|
||||
MatrixMessageContent::Text(text) => convert_bot_text_message(
|
||||
text,
|
||||
&matrix_message.timestamp,
|
||||
matrix_message.sender_id.clone(),
|
||||
),
|
||||
MatrixMessageContent::Notice(text) => {
|
||||
convert_bot_notice_message(text, &matrix_message.timestamp)
|
||||
}
|
||||
MatrixMessageContent::Image(image_content, mime_type, media_bytes) => Some(Message {
|
||||
author: Author::Assistant,
|
||||
sender_id: Some(matrix_message.sender_id.clone()),
|
||||
content: MessageContent::Image(ImageDetails::new(
|
||||
image_content.clone(),
|
||||
mime_type.clone(),
|
||||
media_bytes.clone(),
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message {
|
||||
author: Author::Assistant,
|
||||
sender_id: Some(matrix_message.sender_id.clone()),
|
||||
content: MessageContent::File(FileDetails::new(
|
||||
file_content.clone(),
|
||||
mime_type.clone(),
|
||||
media_bytes.clone(),
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_bot_text_message(
|
||||
text: &str,
|
||||
timestamp: &chrono::DateTime<chrono::Utc>,
|
||||
sender_id: OwnedUserId,
|
||||
) -> Option<Message> {
|
||||
Some(Message {
|
||||
author: Author::Assistant,
|
||||
message_text: text.to_owned(),
|
||||
sender_id: Some(sender_id),
|
||||
content: MessageContent::Text(text.to_owned()),
|
||||
timestamp: timestamp.to_owned(),
|
||||
})
|
||||
}
|
||||
@@ -50,9 +74,11 @@ fn convert_bot_notice_message(
|
||||
|
||||
if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) {
|
||||
// This is a transcription message. We remove the prefix and consider it as a message sent by the user.
|
||||
// sender_id is None because the original speaker is unknown.
|
||||
return Some(Message {
|
||||
author: Author::User,
|
||||
message_text: text.to_owned(),
|
||||
sender_id: None,
|
||||
content: MessageContent::Text(text.to_owned()),
|
||||
timestamp: timestamp.to_owned(),
|
||||
});
|
||||
}
|
||||
@@ -61,9 +87,38 @@ fn convert_bot_notice_message(
|
||||
}
|
||||
|
||||
fn convert_user_message(matrix_message: &MatrixMessage) -> Option<Message> {
|
||||
Some(Message {
|
||||
author: Author::User,
|
||||
message_text: matrix_message.message_text.clone(),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
})
|
||||
match &matrix_message.content {
|
||||
MatrixMessageContent::Text(text) => Some(Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(matrix_message.sender_id.clone()),
|
||||
content: MessageContent::Text(text.clone()),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
MatrixMessageContent::Notice(text) => Some(Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(matrix_message.sender_id.clone()),
|
||||
content: MessageContent::Text(text.clone()),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
MatrixMessageContent::Image(image_content, mime_type, media_bytes) => Some(Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(matrix_message.sender_id.clone()),
|
||||
content: MessageContent::Image(ImageDetails::new(
|
||||
image_content.clone(),
|
||||
mime_type.clone(),
|
||||
media_bytes.clone(),
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message {
|
||||
author: Author::User,
|
||||
sender_id: Some(matrix_message.sender_id.clone()),
|
||||
content: MessageContent::File(FileDetails::new(
|
||||
file_content.clone(),
|
||||
mime_type.clone(),
|
||||
media_bytes.clone(),
|
||||
)),
|
||||
timestamp: matrix_message.timestamp.to_owned(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,20 +2,25 @@ use chrono::{DateTime, Utc};
|
||||
use regex::Regex;
|
||||
|
||||
use mxlink::matrix_sdk::ruma::OwnedUserId;
|
||||
use mxlink::matrix_sdk::ruma::events::room::message::{
|
||||
FileMessageEventContent, ImageMessageEventContent,
|
||||
};
|
||||
use mxlink::mime::Mime;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MatrixMessage {
|
||||
pub sender_id: OwnedUserId,
|
||||
pub message_type: MatrixMessageType,
|
||||
pub message_text: String,
|
||||
pub content: MatrixMessageContent,
|
||||
pub mentioned_users: Vec<OwnedUserId>,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum MatrixMessageType {
|
||||
Text,
|
||||
Notice,
|
||||
pub enum MatrixMessageContent {
|
||||
Text(String),
|
||||
Notice(String),
|
||||
Image(ImageMessageEventContent, Mime, Vec<u8>),
|
||||
File(FileMessageEventContent, Mime, Vec<u8>),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -6,6 +6,6 @@ mod utils;
|
||||
pub(crate) use room_display_name_fetcher::RoomDisplayNameFetcher;
|
||||
pub(crate) use room_event_fetcher::RoomEventFetcher;
|
||||
|
||||
pub(crate) use entity::{MatrixMessage, MatrixMessageProcessingParams, MatrixMessageType};
|
||||
pub(crate) use entity::{MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams};
|
||||
|
||||
pub(crate) use utils::*;
|
||||
|
||||
@@ -18,9 +18,11 @@ use mxlink::matrix_sdk::{
|
||||
},
|
||||
};
|
||||
use mxlink::{MatrixLink, ThreadGetMessagesParams, ThreadInfo};
|
||||
use tracing::Instrument;
|
||||
|
||||
use super::{MatrixMessage, MatrixMessageProcessingParams, MatrixMessageType, RoomEventFetcher};
|
||||
use super::{MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams, RoomEventFetcher};
|
||||
use crate::entity::{InteractionContext, InteractionTrigger, MessagePayload};
|
||||
use crate::utils::mime::get_mime_type_from_file_name;
|
||||
|
||||
struct DetailedMessagePayload {
|
||||
is_mentioning_bot: bool,
|
||||
@@ -28,7 +30,7 @@ struct DetailedMessagePayload {
|
||||
}
|
||||
|
||||
pub async fn get_matrix_messages_in_thread(
|
||||
matrix_link: MatrixLink,
|
||||
matrix_link: &MatrixLink,
|
||||
room: &Room,
|
||||
thread_id: OwnedEventId,
|
||||
) -> Result<Vec<MatrixMessage>, mxlink::matrix_sdk::Error> {
|
||||
@@ -40,18 +42,20 @@ pub async fn get_matrix_messages_in_thread(
|
||||
let mut messages: Vec<MatrixMessage> = Vec::new();
|
||||
|
||||
for matrix_native_message in messages_native {
|
||||
let Some(message) = convert_matrix_native_event_to_matrix_message(&matrix_native_message)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let message_result =
|
||||
convert_matrix_native_event_to_matrix_message(matrix_link, &matrix_native_message)
|
||||
.await?;
|
||||
|
||||
messages.push(message);
|
||||
if let Some(message) = message_result {
|
||||
messages.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(messages)
|
||||
}
|
||||
|
||||
pub async fn get_matrix_messages_in_reply_chain(
|
||||
matrix_link: &MatrixLink,
|
||||
event_fetcher: &Arc<RoomEventFetcher>,
|
||||
room: &Room,
|
||||
event_id: OwnedEventId,
|
||||
@@ -62,12 +66,13 @@ pub async fn get_matrix_messages_in_reply_chain(
|
||||
let mut messages: Vec<MatrixMessage> = Vec::new();
|
||||
|
||||
for matrix_native_message in messages_native {
|
||||
let Some(message) = convert_matrix_native_event_to_matrix_message(&matrix_native_message)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let message_result =
|
||||
convert_matrix_native_event_to_matrix_message(matrix_link, &matrix_native_message)
|
||||
.await?;
|
||||
|
||||
messages.push(message);
|
||||
if let Some(message) = message_result {
|
||||
messages.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(messages)
|
||||
@@ -149,8 +154,11 @@ pub async fn process_matrix_messages(
|
||||
|
||||
let mut message = message.clone();
|
||||
|
||||
if i == 0 && !params.first_message_prefixes_to_strip.is_empty() {
|
||||
let mut message_text = message.message_text.clone();
|
||||
if i == 0
|
||||
&& !params.first_message_prefixes_to_strip.is_empty()
|
||||
&& let MatrixMessageContent::Text(message_text) = &message.content
|
||||
{
|
||||
let mut message_text = message_text.clone();
|
||||
|
||||
for prefix in ¶ms.first_message_prefixes_to_strip {
|
||||
if let Some(message_text_stripped) = message_text.strip_prefix(prefix) {
|
||||
@@ -158,14 +166,15 @@ pub async fn process_matrix_messages(
|
||||
}
|
||||
}
|
||||
|
||||
message.message_text = message_text.trim().to_owned();
|
||||
message.content = MatrixMessageContent::Text(message_text.trim().to_owned());
|
||||
}
|
||||
|
||||
// We only strip `bot_user_prefixes_to_strip`-defined prefixes from messages that mention the bot user.
|
||||
if !params.bot_user_prefixes_to_strip.is_empty()
|
||||
&& message.mentioned_users.contains(¶ms.bot_user_id)
|
||||
&& let MatrixMessageContent::Text(message_text) = &message.content
|
||||
{
|
||||
let mut message_text = message.message_text.clone();
|
||||
let mut message_text = message_text.clone();
|
||||
|
||||
for prefix in ¶ms.bot_user_prefixes_to_strip {
|
||||
if let Some(message_text_stripped) = message_text.strip_prefix(prefix) {
|
||||
@@ -173,7 +182,7 @@ pub async fn process_matrix_messages(
|
||||
}
|
||||
}
|
||||
|
||||
message.message_text = message_text.trim().to_owned();
|
||||
message.content = MatrixMessageContent::Text(message_text.trim().to_owned());
|
||||
}
|
||||
|
||||
messages_filtered.push(message);
|
||||
@@ -207,23 +216,26 @@ fn is_message_from_allowed_sender(
|
||||
false
|
||||
}
|
||||
|
||||
pub fn convert_matrix_native_event_to_matrix_message(
|
||||
pub async fn convert_matrix_native_event_to_matrix_message(
|
||||
matrix_link: &MatrixLink,
|
||||
matrix_native_event: &AnySyncMessageLikeEvent,
|
||||
) -> Option<MatrixMessage> {
|
||||
) -> Result<Option<MatrixMessage>, mxlink::matrix_sdk::Error> {
|
||||
let Some(content) = matrix_native_event.original_content() else {
|
||||
// Redacted message
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let AnyMessageLikeEventContent::RoomMessage(room_message) = content else {
|
||||
// Some state event, etc.
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let (text, is_notice) = match &room_message.msgtype {
|
||||
MessageType::Text(text_content) => (text_content.body.clone(), false),
|
||||
MessageType::Notice(notice_content) => (notice_content.body.clone(), true),
|
||||
_ => return None,
|
||||
MessageType::Image(image_content) => (image_content.body.clone(), false),
|
||||
MessageType::File(file_content) => (file_content.body.clone(), false),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
let is_reply = matches!(room_message.relates_to, Some(Relation::Reply { .. }));
|
||||
@@ -248,17 +260,109 @@ pub fn convert_matrix_native_event_to_matrix_message(
|
||||
.map(|m| m.user_ids.iter().map(|u| u.to_owned()).collect())
|
||||
.unwrap_or(vec![]);
|
||||
|
||||
Some(MatrixMessage {
|
||||
if let MessageType::Image(image_content) = &room_message.msgtype {
|
||||
let media_request = mxlink::matrix_sdk::media::MediaRequestParameters {
|
||||
source: image_content.source.to_owned(),
|
||||
format: mxlink::matrix_sdk::media::MediaFormat::File,
|
||||
};
|
||||
|
||||
let file_name = image_content
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or(image_content.body.clone());
|
||||
|
||||
let mime_type = get_mime_type_from_file_name(&file_name);
|
||||
|
||||
tracing::debug!("Determined mime type {} for file {}", mime_type, file_name);
|
||||
|
||||
let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type);
|
||||
|
||||
let media_bytes = matrix_link
|
||||
.client()
|
||||
.media()
|
||||
.get_media_content(&media_request, true)
|
||||
.instrument(span)
|
||||
.await?;
|
||||
|
||||
return Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
content: MatrixMessageContent::Image(image_content.clone(), mime_type, media_bytes),
|
||||
mentioned_users,
|
||||
timestamp,
|
||||
}));
|
||||
}
|
||||
|
||||
if let MessageType::File(file_content) = &room_message.msgtype {
|
||||
let media_request = mxlink::matrix_sdk::media::MediaRequestParameters {
|
||||
source: file_content.source.to_owned(),
|
||||
format: mxlink::matrix_sdk::media::MediaFormat::File,
|
||||
};
|
||||
|
||||
let file_name = file_content
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or(file_content.body.clone());
|
||||
|
||||
let mime_type = file_content
|
||||
.info
|
||||
.as_ref()
|
||||
.and_then(|info| info.mimetype.clone())
|
||||
.and_then(|mimetype| mimetype.parse::<mxlink::mime::Mime>().ok())
|
||||
.unwrap_or_else(|| get_mime_type_from_file_name(&file_name));
|
||||
|
||||
tracing::debug!("Determined mime type {} for file {}", mime_type, file_name);
|
||||
|
||||
if mime_type == mxlink::mime::APPLICATION_OCTET_STREAM {
|
||||
tracing::debug!(
|
||||
"Skipping file {} with unsupported MIME type {}. It will be represented as a text message.",
|
||||
file_name,
|
||||
mime_type,
|
||||
);
|
||||
|
||||
return Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
content: MatrixMessageContent::Text(format!(
|
||||
"[A file ({}) was attached but skipped because its content type ({}) is not supported. Let the user know.]",
|
||||
file_name, mime_type,
|
||||
)),
|
||||
mentioned_users,
|
||||
timestamp,
|
||||
}));
|
||||
}
|
||||
|
||||
let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type);
|
||||
|
||||
let media_bytes = matrix_link
|
||||
.client()
|
||||
.media()
|
||||
.get_media_content(&media_request, true)
|
||||
.instrument(span)
|
||||
.await?;
|
||||
|
||||
tracing::debug!(
|
||||
"Downloaded {} bytes for file {}",
|
||||
media_bytes.len(),
|
||||
file_name
|
||||
);
|
||||
|
||||
return Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
content: MatrixMessageContent::File(file_content.clone(), mime_type, media_bytes),
|
||||
mentioned_users,
|
||||
timestamp,
|
||||
}));
|
||||
}
|
||||
|
||||
Ok(Some(MatrixMessage {
|
||||
sender_id: matrix_native_event.sender().to_owned(),
|
||||
message_type: if is_notice {
|
||||
MatrixMessageType::Notice
|
||||
content: if is_notice {
|
||||
MatrixMessageContent::Notice(text)
|
||||
} else {
|
||||
MatrixMessageType::Text
|
||||
MatrixMessageContent::Text(text)
|
||||
},
|
||||
message_text: text,
|
||||
mentioned_users,
|
||||
timestamp,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/// Determines the interaction context for an incoming (new) room event.
|
||||
|
||||
@@ -3,7 +3,7 @@ use chrono::{TimeZone, Utc};
|
||||
use mxlink::matrix_sdk::ruma::OwnedUserId;
|
||||
|
||||
use crate::conversation::matrix::{
|
||||
MatrixMessage, MatrixMessageProcessingParams, MatrixMessageType,
|
||||
MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -17,24 +17,21 @@ fn is_message_from_allowed_sender() {
|
||||
|
||||
let bot_message = MatrixMessage {
|
||||
sender_id: bot_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "Hello!".to_owned(),
|
||||
content: MatrixMessageContent::Text("Hello!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let allowed_user_message = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "Hello!".to_owned(),
|
||||
content: MatrixMessageContent::Text("Hello!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let unallowed_user_message = MatrixMessage {
|
||||
sender_id: unallowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "Hello!".to_owned(),
|
||||
content: MatrixMessageContent::Text("Hello!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
@@ -88,48 +85,42 @@ async fn process_matrix_messages() {
|
||||
|
||||
let allowed_user_message = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "Hello from the user!".to_owned(),
|
||||
content: MatrixMessageContent::Text("Hello from the user!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let allowed_user_message_with_prefix = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "!bai Hello from the user!".to_owned(),
|
||||
content: MatrixMessageContent::Text("!bai Hello from the user!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let allowed_user_message_with_prefix_no_space = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "!baiHello from the user!".to_owned(),
|
||||
content: MatrixMessageContent::Text("!baiHello from the user!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let allowed_user_message_with_prefix_full_width_space = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "!bai Hello from the user!".to_owned(),
|
||||
content: MatrixMessageContent::Text("!bai Hello from the user!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let bot_message = MatrixMessage {
|
||||
sender_id: bot_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "Hello from the bot!".to_owned(),
|
||||
content: MatrixMessageContent::Text("Hello from the bot!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let allowed_user_message_with_bot_mention = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "@baibot: Hello from the user!".to_owned(),
|
||||
content: MatrixMessageContent::Text("@baibot: Hello from the user!".to_owned()),
|
||||
mentioned_users: vec![bot_user_id.to_owned()],
|
||||
timestamp,
|
||||
};
|
||||
@@ -137,16 +128,14 @@ async fn process_matrix_messages() {
|
||||
// The message text is the same as above - it mentions the bot, but the actually-mentioned user is another user.
|
||||
let allowed_user_message_with_another_user_mention = MatrixMessage {
|
||||
sender_id: allowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: allowed_user_message_with_bot_mention.message_text.clone(),
|
||||
content: allowed_user_message_with_bot_mention.content.clone(),
|
||||
mentioned_users: vec![allowed_user_id.to_owned()],
|
||||
timestamp,
|
||||
};
|
||||
|
||||
let unallowed_user_message = MatrixMessage {
|
||||
sender_id: unallowed_user_id.to_owned(),
|
||||
message_type: MatrixMessageType::Text,
|
||||
message_text: "Hello from an unallowed user!".to_owned(),
|
||||
content: MatrixMessageContent::Text("Hello from an unallowed user!".to_owned()),
|
||||
mentioned_users: vec![],
|
||||
timestamp,
|
||||
};
|
||||
@@ -285,7 +274,10 @@ async fn process_matrix_messages() {
|
||||
|
||||
let processed_message_texts = processed_messages
|
||||
.iter()
|
||||
.map(|message| message.message_text.clone())
|
||||
.map(|message| match &message.content {
|
||||
MatrixMessageContent::Text(text) => text.clone(),
|
||||
_ => "".to_owned(),
|
||||
})
|
||||
.collect::<Vec<String>>();
|
||||
|
||||
assert_eq!(
|
||||
|
||||
@@ -12,7 +12,7 @@ use super::matrix::{
|
||||
};
|
||||
|
||||
pub async fn create_llm_conversation_for_matrix_thread(
|
||||
matrix_link: MatrixLink,
|
||||
matrix_link: &MatrixLink,
|
||||
room: &mxlink::matrix_sdk::Room,
|
||||
thread_id: OwnedEventId,
|
||||
params: &MatrixMessageProcessingParams,
|
||||
@@ -27,12 +27,14 @@ pub async fn create_llm_conversation_for_matrix_thread(
|
||||
}
|
||||
|
||||
pub async fn create_llm_conversation_for_matrix_reply_chain(
|
||||
matrix_link: &MatrixLink,
|
||||
event_fetcher: &Arc<RoomEventFetcher>,
|
||||
room: &mxlink::matrix_sdk::Room,
|
||||
event_id: OwnedEventId,
|
||||
params: &MatrixMessageProcessingParams,
|
||||
) -> Result<Conversation, mxlink::matrix_sdk::Error> {
|
||||
let messages = get_matrix_messages_in_reply_chain(event_fetcher, room, event_id).await?;
|
||||
let messages =
|
||||
get_matrix_messages_in_reply_chain(matrix_link, event_fetcher, room, event_id).await?;
|
||||
|
||||
let llm_messages = filter_messages_and_convert_to_llm_messages(messages, params).await;
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user