unfunny/baibot-withmcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update tiktoken-rs to 0.11, adding support for newer GPT models

Browse Source 
Supersedes https://github.com/etkecc/baibot/pull/116

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Slavi Pantaleev
2026-04-11 10:52:22 +03:00
parent 1563e83672
commit a865a26093
3 changed files with 33 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Cargo.lock generated
View File

@@ -339,18 +339,18 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bit-set"
version = "0.5.3"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
version = "0.6.3"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bitflags"
@@ -1048,7 +1048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -1137,9 +1137,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fancy-regex"
version = "0.13.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
dependencies = [
"bit-set",
"regex-automata",
@@ -1738,7 +1738,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2 0.6.3",
"tokio",
"tower-service",
"tracing",
@@ -2631,7 +2631,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -3008,7 +3008,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.2",
"rustls 0.23.37",
"socket2 0.5.10",
"socket2 0.6.3",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -3045,7 +3045,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.10",
"socket2 0.6.3",
"tracing",
"windows-sys 0.60.2",
]
@@ -3595,7 +3595,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4001,7 +4001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -4142,7 +4142,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4207,9 +4207,9 @@ dependencies = [
[[package]]
name = "tiktoken-rs"
version = "0.9.1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d"
checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"
dependencies = [
"anyhow",
"base64 0.22.1",

2
Cargo.toml
View File

@@ -33,7 +33,7 @@ serde = { version = "1.0.*", features = ["derive"], default-features = false }
serde_json = "1.0.*"
serde_yaml_ng = "0.10.*"
tempfile = "3.27.*"
tiktoken-rs = { version = "0.9.*", default-features = false }
tiktoken-rs = { version = "0.11.*", default-features = false }
tokio = { version = "1.51.*", features = ["rt", "rt-multi-thread", "macros"] }
tracing = "0.1.*"
tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }

32
src/conversation/llm/tokenization.rs
View File

@@ -1,15 +1,15 @@
use tiktoken_rs::CoreBPE;
use tiktoken_rs::get_bpe_from_tokenizer;
use tiktoken_rs::bpe_for_tokenizer;
use tiktoken_rs::tokenizer;
use super::{Author, Message, MessageContent};
fn get_bpe_for_model(model: &str) -> CoreBPE {
fn get_bpe_for_model(model: &str) -> &'static CoreBPE {
let tokenizer = tokenizer::get_tokenizer(model)
.or_else(|| tokenizer::get_tokenizer("gpt-4"))
.unwrap();
get_bpe_from_tokenizer(tokenizer).unwrap()
bpe_for_tokenizer(tokenizer).unwrap()
}
pub fn shorten_messages_list_to_context_size(
@@ -26,7 +26,7 @@ pub fn shorten_messages_list_to_context_size(
// We want to retain the prompt in all cases, so we always count it first.
// We also always reserve enough tokens for the maximum response we expect.
let mut current_context_length: u32 = if let Some(prompt_message) = prompt_message {
calculate_token_size_for_message(&bpe, model, prompt_message)
calculate_token_size_for_message(bpe, model, prompt_message)
+ max_response_tokens.unwrap_or(0)
} else {
0
@@ -37,7 +37,7 @@ pub fn shorten_messages_list_to_context_size(
let mut messages_to_keep: Vec<Message> = Vec::new();
for message in messages {
let tokens_for_message = calculate_token_size_for_message(&bpe, model, &message);
let tokens_for_message = calculate_token_size_for_message(bpe, model, &message);
if current_context_length + tokens_for_message > max_context_tokens {
break;
@@ -94,7 +94,7 @@ pub mod test {
timestamp: chrono::Utc::now(),
};
let tokens = super::calculate_token_size_for_message(&bpe, model, &message);
let tokens = super::calculate_token_size_for_message(bpe, model, &message);
assert_eq!(8, tokens);
}
@@ -117,7 +117,7 @@ pub mod test {
assert_eq!(
prompt_length,
super::calculate_token_size_for_message(&bpe, model, &prompt)
super::calculate_token_size_for_message(bpe, model, &prompt)
);
let mut conversation_messages = Vec::new();
@@ -132,7 +132,7 @@ pub mod test {
assert_eq!(
first_length,
super::calculate_token_size_for_message(&bpe, model, &first)
super::calculate_token_size_for_message(bpe, model, &first)
);
conversation_messages.push(first);
@@ -147,7 +147,7 @@ pub mod test {
assert_eq!(
second_length,
super::calculate_token_size_for_message(&bpe, model, &second)
super::calculate_token_size_for_message(bpe, model, &second)
);
conversation_messages.push(second);
@@ -164,7 +164,7 @@ pub mod test {
assert_eq!(
third_length,
super::calculate_token_size_for_message(&bpe, model, &third)
super::calculate_token_size_for_message(bpe, model, &third)
);
conversation_messages.push(third.clone());
@@ -181,7 +181,7 @@ pub mod test {
assert_eq!(
forth_length,
super::calculate_token_size_for_message(&bpe, model, &forth)
super::calculate_token_size_for_message(bpe, model, &forth)
);
conversation_messages.push(forth.clone());
@@ -227,7 +227,7 @@ pub mod test {
assert_eq!(
prompt_length,
super::calculate_token_size_for_message(&bpe, model, &prompt)
super::calculate_token_size_for_message(bpe, model, &prompt)
);
let mut conversation_messages = Vec::new();
@@ -242,7 +242,7 @@ pub mod test {
assert_eq!(
first_length,
super::calculate_token_size_for_message(&bpe, model, &first)
super::calculate_token_size_for_message(bpe, model, &first)
);
conversation_messages.push(first);
@@ -257,7 +257,7 @@ pub mod test {
assert_eq!(
second_length,
super::calculate_token_size_for_message(&bpe, model, &second)
super::calculate_token_size_for_message(bpe, model, &second)
);
conversation_messages.push(second);
@@ -274,7 +274,7 @@ pub mod test {
assert_eq!(
third_length,
super::calculate_token_size_for_message(&bpe, model, &third)
super::calculate_token_size_for_message(bpe, model, &third)
);
conversation_messages.push(third.clone());
@@ -291,7 +291,7 @@ pub mod test {
assert_eq!(
forth_length,
super::calculate_token_size_for_message(&bpe, model, &forth)
super::calculate_token_size_for_message(bpe, model, &forth)
);
conversation_messages.push(forth.clone());