diff --git a/Cargo.lock b/Cargo.lock index b728960..abce240 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,18 +339,18 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bit-set" -version = "0.5.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.6.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -1048,7 +1048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1137,9 +1137,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fancy-regex" -version = "0.13.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" dependencies = [ "bit-set", "regex-automata", @@ -1738,7 +1738,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2631,7 +2631,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3008,7 +3008,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls 0.23.37", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -3045,7 +3045,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -3595,7 +3595,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4001,7 +4001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4142,7 +4142,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4207,9 +4207,9 @@ dependencies = [ [[package]] name = "tiktoken-rs" -version = "0.9.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d" +checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index a51bd31..df64014 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ serde = { version = "1.0.*", features = ["derive"], default-features = false } serde_json = "1.0.*" serde_yaml_ng = "0.10.*" tempfile = "3.27.*" -tiktoken-rs = { version = "0.9.*", default-features = false } +tiktoken-rs = { version = "0.11.*", default-features = false } tokio = { version = "1.51.*", features = ["rt", "rt-multi-thread", "macros"] } tracing = "0.1.*" tracing-subscriber = { version = "0.3.*", features = ["env-filter"] } diff --git a/src/conversation/llm/tokenization.rs b/src/conversation/llm/tokenization.rs index 6f64b32..9616258 100644 --- a/src/conversation/llm/tokenization.rs +++ b/src/conversation/llm/tokenization.rs @@ -1,15 +1,15 @@ use tiktoken_rs::CoreBPE; -use tiktoken_rs::get_bpe_from_tokenizer; +use tiktoken_rs::bpe_for_tokenizer; use tiktoken_rs::tokenizer; use super::{Author, Message, MessageContent}; -fn get_bpe_for_model(model: &str) -> CoreBPE { +fn get_bpe_for_model(model: &str) -> &'static CoreBPE { let tokenizer = tokenizer::get_tokenizer(model) .or_else(|| tokenizer::get_tokenizer("gpt-4")) .unwrap(); - get_bpe_from_tokenizer(tokenizer).unwrap() + bpe_for_tokenizer(tokenizer).unwrap() } pub fn shorten_messages_list_to_context_size( @@ -26,7 +26,7 @@ pub fn shorten_messages_list_to_context_size( // We want to retain the prompt in all cases, so we always count it first. // We also always reserve enough tokens for the maximum response we expect. let mut current_context_length: u32 = if let Some(prompt_message) = prompt_message { - calculate_token_size_for_message(&bpe, model, prompt_message) + calculate_token_size_for_message(bpe, model, prompt_message) + max_response_tokens.unwrap_or(0) } else { 0 @@ -37,7 +37,7 @@ pub fn shorten_messages_list_to_context_size( let mut messages_to_keep: Vec = Vec::new(); for message in messages { - let tokens_for_message = calculate_token_size_for_message(&bpe, model, &message); + let tokens_for_message = calculate_token_size_for_message(bpe, model, &message); if current_context_length + tokens_for_message > max_context_tokens { break; @@ -94,7 +94,7 @@ pub mod test { timestamp: chrono::Utc::now(), }; - let tokens = super::calculate_token_size_for_message(&bpe, model, &message); + let tokens = super::calculate_token_size_for_message(bpe, model, &message); assert_eq!(8, tokens); } @@ -117,7 +117,7 @@ pub mod test { assert_eq!( prompt_length, - super::calculate_token_size_for_message(&bpe, model, &prompt) + super::calculate_token_size_for_message(bpe, model, &prompt) ); let mut conversation_messages = Vec::new(); @@ -132,7 +132,7 @@ pub mod test { assert_eq!( first_length, - super::calculate_token_size_for_message(&bpe, model, &first) + super::calculate_token_size_for_message(bpe, model, &first) ); conversation_messages.push(first); @@ -147,7 +147,7 @@ pub mod test { assert_eq!( second_length, - super::calculate_token_size_for_message(&bpe, model, &second) + super::calculate_token_size_for_message(bpe, model, &second) ); conversation_messages.push(second); @@ -164,7 +164,7 @@ pub mod test { assert_eq!( third_length, - super::calculate_token_size_for_message(&bpe, model, &third) + super::calculate_token_size_for_message(bpe, model, &third) ); conversation_messages.push(third.clone()); @@ -181,7 +181,7 @@ pub mod test { assert_eq!( forth_length, - super::calculate_token_size_for_message(&bpe, model, &forth) + super::calculate_token_size_for_message(bpe, model, &forth) ); conversation_messages.push(forth.clone()); @@ -227,7 +227,7 @@ pub mod test { assert_eq!( prompt_length, - super::calculate_token_size_for_message(&bpe, model, &prompt) + super::calculate_token_size_for_message(bpe, model, &prompt) ); let mut conversation_messages = Vec::new(); @@ -242,7 +242,7 @@ pub mod test { assert_eq!( first_length, - super::calculate_token_size_for_message(&bpe, model, &first) + super::calculate_token_size_for_message(bpe, model, &first) ); conversation_messages.push(first); @@ -257,7 +257,7 @@ pub mod test { assert_eq!( second_length, - super::calculate_token_size_for_message(&bpe, model, &second) + super::calculate_token_size_for_message(bpe, model, &second) ); conversation_messages.push(second); @@ -274,7 +274,7 @@ pub mod test { assert_eq!( third_length, - super::calculate_token_size_for_message(&bpe, model, &third) + super::calculate_token_size_for_message(bpe, model, &third) ); conversation_messages.push(third.clone()); @@ -291,7 +291,7 @@ pub mod test { assert_eq!( forth_length, - super::calculate_token_size_for_message(&bpe, model, &forth) + super::calculate_token_size_for_message(bpe, model, &forth) ); conversation_messages.push(forth.clone());