llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use CharTrie and fix multi-byte char issue

Browse Source
This commit is contained in:
Geens
2025-04-13 16:05:27 +02:00
parent f4de9e7571
commit be48130a8f
6 changed files with 136 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
lib/xml_schema_validator/src/char_trie.rs
View File

@@ -2,6 +2,7 @@ use crate::*;
use pyo3::prelude::*;
// Node in our character trie
#[derive(Default)]
struct CharTrieNode {
// Store character and node pairs, keep sorted for binary search
children: Vec<(char, CharTrieNode)>,
@@ -10,20 +11,13 @@ struct CharTrieNode {
}
impl CharTrieNode {
fn new() -> Self {
Self {
children: Vec::new(),
token_id: None,
}
}
// Add a child for character c, maintaining sorted order
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
Ok(index) => &mut self.children[index].1,
Err(index) => {
// Insert at the right position to maintain sort order
self.children.insert(index, (c, CharTrieNode::new()));
self.children.insert(index, (c, CharTrieNode::default()));
&mut self.children[index].1
}
}
@@ -34,7 +28,38 @@ impl CharTrieNode {
self.token_id = Some(token_id);
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
fn find_invalid_tokens(
&self,
py: Python<'_>,
xml_validator: &XmlSchemaValidator,
) -> Vec<Py<PyAny>> {
let res = self.children.iter().map(|(c, n)|{
let mut node_validator = xml_validator.clone();
if node_validator.append_char(*c).is_ok() {
// If the node is valid, recursively check its children
n.find_invalid_tokens(py, &node_validator)
} else {
// If the node is invalid, add it and its children to the list of invalid tokens
n.all_tokens(py)
}
})
.flatten()
.collect();
res
}
fn all_tokens(&self, py: Python<'_>) -> Vec<Py<PyAny>> {
let mut tokens = Vec::new();
if let Some(token_id) = &self.token_id {
tokens.push(token_id.clone_ref(py));
}
for (_, child) in &self.children {
tokens.extend(child.all_tokens(py));
}
tokens
}
fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
children: self
.children
@@ -51,17 +76,12 @@ impl CharTrieNode {
}
// The main trie structure
#[derive(Default)]
pub struct CharTrie {
root: CharTrieNode,
}
impl CharTrie {
pub fn new() -> Self {
Self {
root: CharTrieNode::new(),
}
}
// Insert a token's character sequence into the trie
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
let mut current = &mut self.root;
@@ -80,32 +100,8 @@ impl CharTrie {
&self,
py: Python<'_>,
xml_validator: &XmlSchemaValidator,
token_map: &Vec<(Py<PyAny>, String)>,
) -> Vec<Py<PyAny>> {
let mut invalid_tokens = Vec::new();
for (token_id, token_text) in token_map {
// Check if this token would make the XML invalid
if !self.is_valid_continuation(xml_validator, token_text) {
invalid_tokens.push(token_id.clone_ref(py));
}
}
invalid_tokens
}
// Check if a token would be a valid continuation
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
// First, quick check with just the first character
if let Some(first_char) = token_text.chars().next() {
if !xml_validator.can_continue_with(first_char) {
return false;
}
}
// If first character is valid, try the full token
let mut validator_clone = xml_validator.clone();
validator_clone.append(token_text).is_ok()
self.root.find_invalid_tokens(py, xml_validator)
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
@@ -113,4 +109,4 @@ impl CharTrie {
root: self.root.clone_ref(py),
}
}
}
}

51
lib/xml_schema_validator/src/py_xml_logits_processor_core.rs
View File

@@ -6,7 +6,6 @@ use pyo3::types::PyDict;
#[pyclass]
pub struct XmlLogitsProcessorCore {
xml_schema_validator: XmlSchemaValidator,
tokens: Vec<(PyObject, String)>,
trie: char_trie::CharTrie,
}
@@ -22,13 +21,12 @@ impl XmlLogitsProcessorCore {
.iter()
.map(|(id, value)| (id.unbind(), value.unbind().to_string()))
.collect();
let mut trie = char_trie::CharTrie::new();
let mut trie = char_trie::CharTrie::default();
for (token_id, token_text) in tokens.iter() {
trie.insert(token_text, token_id.clone_ref(py));
}
Ok(Self {
xml_schema_validator,
tokens,
trie,
})
}
@@ -43,7 +41,7 @@ impl XmlLogitsProcessorCore {
fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult<Vec<PyObject>> {
Ok(self
.trie
.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
.find_invalid_tokens(py, &self.xml_schema_validator))
}
/// Check if the validator has reached the end of the XML
@@ -55,11 +53,6 @@ impl XmlLogitsProcessorCore {
fn copy(&self, py: Python<'_>) -> PyResult<Self> {
Ok(Self {
xml_schema_validator: self.xml_schema_validator.clone(),
tokens: self
.tokens
.iter()
.map(|(a, b)| (a.clone_ref(py), b.clone()))
.collect(),
trie: self.trie.clone_ref(py),
})
}
@@ -116,6 +109,29 @@ mod tests {
XmlLogitsProcessorCore::new(py, tokens.into(), schema_text)
}
#[test]
fn test_delete_quote() {
Python::with_gil(|py| {
py.run(&std::ffi::CString::new(r#"
from transformers import AutoTokenizer
from xml_schema_validator import XmlLogitsProcessor
import os
import torch
model_path = os.environ["SIA_LOCAL_MODEL"]
api_token = os.environ["SIA_HF_API_KEY"]
xml_schema_actions = open("example_schema.xsd").read()
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
input_ids = torch.tensor([tokenizer.encode("")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
input_ids = torch.tensor([tokenizer.encode("<delete id=\"")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
"#).unwrap(), None, None).unwrap();
})
}
#[test]
fn test_append_valid_xml() {
Python::with_gil(|py| {
@@ -182,12 +198,6 @@ mod tests {
"Should accept complete document"
);
// DEBUG: Print the token state to see why EOF detection fails
println!(
"Current tokens: {:?}",
processor.xml_schema_validator.current_tokens
);
assert!(
processor.eof().unwrap(),
"Should be at EOF after complete document"
@@ -276,9 +286,6 @@ mod tests {
let processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
let names = processor.get_element_names().unwrap();
// Print out all elements from schema for debugging
println!("Elements in schema: {:?}", names);
// Check if reasoning exists
assert!(
names.contains(&"reasoning".to_string()),
@@ -322,7 +329,6 @@ mod tests {
// Test basic operations to verify core functionality works
let element_names = processor.get_element_names().unwrap();
println!("Element names in test schema: {:?}", element_names);
assert!(
element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'"
@@ -353,16 +359,11 @@ mod tests {
// Now test with actual schema from disk if available
if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") {
println!("\nTesting with actual schema from disk:");
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let element_names = processor.get_element_names().unwrap();
println!("Element names in actual schema: {:?}", element_names);
if element_names.contains(&"reasoning".to_string()) {
println!("'reasoning' element found in actual schema");
// Test appending "<reasoning>" to a fresh processor
let mut processor = processor.copy(py).unwrap();
let result = processor.append("<reasoning>");
@@ -378,8 +379,6 @@ mod tests {
.append(&text);
println!("Appending '{}' to fresh processor: {}", text, result);
}
} else {
println!("Warning: 'reasoning' element NOT found in actual schema!");
}
}
});

10
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -55,10 +55,10 @@ impl AttributeValue {
}
fn value_closed(&self) -> bool {
if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") {
let trimmed = self.buffer.trim();
let trimmed = &trimmed[..trimmed.len() - 2];
if Self::in_escape_sequence(trimmed) {
let mut buffer = self.buffer.trim().to_owned();
if buffer.len() > 2 && buffer.ends_with("\"") {
buffer.pop();
if Self::in_escape_sequence(&buffer) {
false
} else {
true
@@ -125,6 +125,8 @@ mod tests {
fn test_unfinished_attribute_value() {
let values = [
"<delete id=\"12",
"<delete id=\"…",
"<delete id=\" ",
"<single timeout=\"1.",
"<single limit=\"50",
];