llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed attribute parsing

Browse Source
This commit is contained in:
Geens
2025-04-12 18:02:55 +02:00
parent bd3adf78e6
commit 9564879edc
23 changed files with 2905 additions and 2786 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
lib/xml_schema_validator/example_schema.xsd Normal file
View File

@@ -0,0 +1,118 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Always answer with a single element.
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
<!--
Delete command removes an entry from the context by its ID.
Use it to remove unnecessary items and stop background processes.
When you delete something, it is gone.
Make sure all important info is stored in files.
Example:
<delete id="1234567890"/>
-->
<xs:element name="delete">
<xs:complexType>
<xs:attribute name="id" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<!--
Stop command terminates the agent gracefully.
For the main SIA instance this will trigger an update and restart.
For sub-instances this is the correct way to stop after all tasks are complete.
Example:
<stop id="1234567890"/>
-->
<xs:element name="stop">
<xs:complexType/>
</xs:element>
<!--
Single script that runs once and completes.
Output is stored in context until explicitly deleted.
Used for one-time operations like file manipulation.
Single scripts are limited to 1024 characters and 1 second timeout by default.
These limits can be changed with attributes.
Example:
<single>
ls /
</single>
-->
<xs:element name="single">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
<xs:attribute name="timeout" type="xs:float" use="optional"/>
<xs:attribute name="limit" type="xs:integer" use="optional"/>
</xs:complexType>
</xs:element>
<!--
Repeat script runs each time the context is generated.
After a command is issued, all repeat scripts in context are run again.
Useful for monitoring changing files or viewing results immediately after changing a file.
Repeat scripts should execute quickly to avoid blocking the agent.
Repeat scripts are limited to 1024 characters and 1 second timeout by default.
These limits can be changed with attributes.
Example:
<repeat>
ls /
</repeat>
-->
<xs:element name="repeat">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
<xs:attribute name="timeout" type="xs:float" use="optional"/>
<xs:attribute name="limit" type="xs:integer" use="optional"/>
</xs:complexType>
</xs:element>
<!--
As an agent it is important to reason about your actions and their results.
In a reasoning action you can write freeform text.
This is also stored in context until deleted.
Example:
<reasoning>
I should explore the file system for interesting files.
</reasoning>
-->
<xs:element name="reasoning">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
</xs:complexType>
</xs:element>
<!--
Read all available text on stdin and store it in context.
Do this only if the context indicates there is data in the stdin buffer.
Example:
<read_stdin/>
-->
<xs:element name="read_stdin">
<xs:complexType/>
</xs:element>
<!--
Write to stdout.
This is your main way of contacting the user.
Make sure you have properly reasoned about what to say and if it is necessary before issuing a write_stdout command.
Example:
<write_stdout>
Hello world!
</write_stdout>
-->
<xs:element name="write_stdout">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:schema>

2
lib/xml_schema_validator/python/xml_schema_validator/__init__.py
View File

@@ -12,7 +12,7 @@ class XmlLogitsProcessor(LogitsProcessor):
by setting their logits to negative infinity.
"""
def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core=None):
def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core: XmlLogitsProcessorCore=None):
"""
Initialize the processor with a schema and tokenizer.

19
lib/xml_schema_validator/src/char_trie.rs
View File

@@ -36,8 +36,16 @@ impl CharTrieNode {
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
children: self
.children
.iter()
.map(|(ch, node)| (*ch, node.clone_ref(py)))
.collect(),
token_id: self
.token_id
.iter()
.map(|token_id| token_id.clone_ref(py))
.nth(0),
}
}
}
@@ -68,7 +76,12 @@ impl CharTrie {
}
// Find all tokens that could be invalid given the current XML state
pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
pub fn find_invalid_tokens(
&self,
py: Python<'_>,
xml_validator: &XmlSchemaValidator,
token_map: &Vec<(Py<PyAny>, String)>,
) -> Vec<Py<PyAny>> {
let mut invalid_tokens = Vec::new();
for (token_id, token_text) in token_map {

224
lib/xml_schema_validator/src/lib.rs
View File

@@ -11,8 +11,8 @@ mod python;
mod schema;
mod token;
use std::sync::Arc;
use error::Error;
use std::sync::Arc;
use token::*;
/// An XML validator that checks XML fragments against an XSD schema
@@ -33,10 +33,14 @@ impl XmlSchemaValidator {
let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?;
let schema_arc = Arc::new(schema);
let current_tokens = schema_arc.xs_element.iter().map(|element| {
let current_tokens = schema_arc
.xs_element
.iter()
.map(|element| {
let element_arc = Arc::new(element.clone());
Token::StartOfFile(StartOfFile::new(element_arc))
}).collect();
})
.collect();
Ok(Self {
current_tokens,
@@ -64,7 +68,10 @@ impl XmlSchemaValidator {
new_tokens.append(&mut token.append(c));
}
if new_tokens.is_empty() {
return Err(Error::InvalidXml(format!("No valid continuations for character: '{}'", c)));
return Err(Error::InvalidXml(format!(
"No valid continuations for character: '{}'",
c
)));
}
self.current_tokens = new_tokens;
Ok(())
@@ -79,7 +86,9 @@ impl XmlSchemaValidator {
/// Get a list of all element names in the schema
pub fn get_element_names(&self) -> Vec<String> {
self.schema.xs_element.iter()
self.schema
.xs_element
.iter()
.map(|element| element.name.clone())
.collect()
}
@@ -89,19 +98,22 @@ impl XmlSchemaValidator {
mod tests {
use super::*;
pub fn example_validator() -> XmlSchemaValidator {
let schema_text = std::fs::read_to_string("example_schema.xsd").unwrap();
XmlSchemaValidator::new(&schema_text).unwrap()
}
#[test]
fn test_element_open() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let mut validator = example_validator();
let input = "<reasoning>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
}
#[test]
fn test_element_open_whitespace() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let mut validator = example_validator();
let input = "<reasoning >";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
}
@@ -126,7 +138,9 @@ mod tests {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<reasoning>test</wrong>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).expect_err("Should fail with mismatched closing tag");
validator
.append(input)
.expect_err("Should fail with mismatched closing tag");
}
#[test]
@@ -145,12 +159,15 @@ mod tests {
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// First, validate up to the opening tag
assert!(validator.append("<delete>").is_err(), "Should fail with missing required attribute");
assert!(
validator.append("<delete>").is_err(),
"Should fail with missing required attribute"
);
// Create a new validator and try closing tag approach
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Try to validate an incomplete delete tag that's missing the required id attribute
let input = "<delete></delete>";
let input = "<delete/>";
let result = validator.append(input);
assert!(result.is_err(), "Should fail without required id attribute");
@@ -158,8 +175,10 @@ mod tests {
if let Err(err) = result {
println!("Error message: {}", err);
let error_str = err.to_string().to_lowercase();
assert!(error_str.contains("required") || error_str.contains("no valid continuations"),
"Error should mention missing required attribute or invalid continuation");
assert!(
error_str.contains("required") || error_str.contains("no valid continuations"),
"Error should mention missing required attribute or invalid continuation"
);
}
}
@@ -167,11 +186,26 @@ mod tests {
fn test_invalid_nesting() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.clone().append("<reasoning><reasoning>").expect_err("Should fail because of nested tag");
validator.clone().append("<reasoning> <reasoning>").expect_err("Should fail because of nested tag, even with whitespace");
validator.clone().append("<reasoning>foo<reasoning>").expect_err("Should fail because of nested tag, even with text");
validator.clone().append("<reasoning> foo <reasoning>").expect_err("Should fail because of nested tag, even with text and whitespace");
validator.clone().append("<reasoning> foo <reasoning/>").expect_err("Should fail because of nested tag, even with self closing tag");
validator
.clone()
.append("<reasoning><reasoning>")
.expect_err("Should fail because of nested tag");
validator
.clone()
.append("<reasoning> <reasoning>")
.expect_err("Should fail because of nested tag, even with whitespace");
validator
.clone()
.append("<reasoning>foo<reasoning>")
.expect_err("Should fail because of nested tag, even with text");
validator
.clone()
.append("<reasoning> foo <reasoning>")
.expect_err("Should fail because of nested tag, even with text and whitespace");
validator
.clone()
.append("<reasoning> foo <reasoning/>")
.expect_err("Should fail because of nested tag, even with self closing tag");
}
#[test]
@@ -204,7 +238,8 @@ mod tests {
#[test]
fn test_reasoning_with_content() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<reasoning>I should explore the file system for interesting files.</reasoning>";
let input =
"<reasoning>I should explore the file system for interesting files.</reasoning>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
@@ -223,7 +258,11 @@ mod tests {
fn test_create_validator() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text);
assert!(validator.is_ok(), "Failed to create validator: {:?}", validator.err());
assert!(
validator.is_ok(),
"Failed to create validator: {:?}",
validator.err()
);
}
#[test]
@@ -231,8 +270,11 @@ mod tests {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
let names = validator.get_element_names();
assert!(names.contains(&"reasoning".to_string()),
"Element names should contain 'reasoning', got: {:?}", names);
assert!(
names.contains(&"reasoning".to_string()),
"Element names should contain 'reasoning', got: {:?}",
names
);
}
// Test basic XML fragment validation - debugging the "<reasoning>" failure
@@ -255,7 +297,11 @@ mod tests {
// Try the whole string at once
let mut v4 = validator.clone();
let result = v4.append("<reasoning>");
assert!(result.is_ok(), "Failed to append '<reasoning>': {:?}", result.err());
assert!(
result.is_ok(),
"Failed to append '<reasoning>': {:?}",
result.err()
);
}
// Test validation of multiple fragments
@@ -265,10 +311,19 @@ mod tests {
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Test sequential appends
assert!(validator.append("<").is_ok(), "Failed to append '<'");
assert!(validator.append("reasoning").is_ok(), "Failed to append 'reasoning'");
assert!(
validator.append("reasoning").is_ok(),
"Failed to append 'reasoning'"
);
assert!(validator.append(">").is_ok(), "Failed to append '>'");
assert!(validator.append("test content").is_ok(), "Failed to append content");
assert!(validator.append("</reasoning>").is_ok(), "Failed to append closing tag");
assert!(
validator.append("test content").is_ok(),
"Failed to append content"
);
assert!(
validator.append("</reasoning>").is_ok(),
"Failed to append closing tag"
);
// Should be at EOF now
assert!(validator.eof(), "Should be at EOF after complete XML");
}
@@ -279,15 +334,30 @@ mod tests {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
// Test at start
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
assert!(validator.can_continue_with('<'), "Should accept '<' at start");
assert!(!validator.can_continue_with('a'), "Should reject 'a' at start");
assert!(
validator.can_continue_with('<'),
"Should accept '<' at start"
);
assert!(
!validator.can_continue_with('a'),
"Should reject 'a' at start"
);
// Test after opening tag
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append("<reasoning>").unwrap();
// Any character should be valid inside a mixed content element
assert!(validator.can_continue_with('a'), "Should accept 'a' inside element");
assert!(validator.can_continue_with('1'), "Should accept '1' inside element");
assert!(validator.can_continue_with('<'), "Should accept '<' inside element");
assert!(
validator.can_continue_with('a'),
"Should accept 'a' inside element"
);
assert!(
validator.can_continue_with('1'),
"Should accept '1' inside element"
);
assert!(
validator.can_continue_with('<'),
"Should accept '<' inside element"
);
}
}
@@ -307,7 +377,8 @@ mod diagnostic_tests {
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:schema>"#.to_string()
</xs:schema>"#
.to_string()
}
// Test every character in an opening tag one by one
@@ -361,16 +432,23 @@ mod diagnostic_tests {
});
// Test the transition from ElementOpenName to ElementOpenEnd
let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string()).unwrap();
let element_name =
token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string());
let tokens = element_name.append('>');
assert!(!tokens.is_empty(), "Should produce valid tokens after '>'");
println!("Token after appending '>' to ElementOpenName: {:?}", tokens[0]);
println!(
"Token after appending '>' to ElementOpenName: {:?}",
tokens[0]
);
// Check what type the token is
match &tokens[0] {
Token::ElementOpenEnd(element_open) => {
println!("Successfully transitioned to ElementOpen: {:?}", element_open);
},
println!(
"Successfully transitioned to ElementOpen: {:?}",
element_open
);
}
other => {
panic!("Expected ElementOpen, got {:?}", other);
}
@@ -392,7 +470,8 @@ mod diagnostic_tests {
});
// Test transition from ElementName to ElementSelfClose
let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string()).unwrap();
let element_name =
token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string());
let tokens = element_name.append('/');
assert!(!tokens.is_empty(), "Should handle '/' after element name");
@@ -407,7 +486,7 @@ mod diagnostic_tests {
Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"),
other => panic!("Expected EndOfFile, got {:?}", other),
}
},
}
other => panic!("Expected ElementSelfClose, got {:?}", other),
}
}
@@ -427,8 +506,14 @@ mod diagnostic_tests {
println!("State after '<reasoning ': {:?}", validator.current_tokens);
// Test completion with >
assert!(validator.can_continue_with('>'), "Should accept '>' after whitespace");
assert!(validator.append_char('>').is_ok(), "Failed to append '>' after whitespace");
assert!(
validator.can_continue_with('>'),
"Should accept '>' after whitespace"
);
assert!(
validator.append_char('>').is_ok(),
"Failed to append '>' after whitespace"
);
}
// Test attribute handling
@@ -452,22 +537,49 @@ mod diagnostic_tests {
// Test attribute name
for c in "id".chars() {
assert!(validator.can_continue_with(c), "Should accept '{}' in attribute name", c);
assert!(validator.append_char(c).is_ok(), "Failed to append '{}' in attribute name", c);
assert!(
validator.can_continue_with(c),
"Should accept '{}' in attribute name",
c
);
assert!(
validator.append_char(c).is_ok(),
"Failed to append '{}' in attribute name",
c
);
}
// Test attribute equals and quotes
for c in "=\"123\"".chars() {
println!("Testing attribute character: '{}'", c);
assert!(validator.can_continue_with(c), "Should accept '{}' in attribute value", c);
assert!(validator.append_char(c).is_ok(), "Failed to append '{}' in attribute value", c);
assert!(
validator.can_continue_with(c),
"Should accept '{}' in attribute value",
c
);
assert!(
validator.append_char(c).is_ok(),
"Failed to append '{}' in attribute value",
c
);
}
// Test closing the self-closing tag
println!("Current state before closing: {:?}", validator.current_tokens);
println!(
"Current state before closing: {:?}",
validator.current_tokens
);
for c in "/>".chars() {
assert!(validator.can_continue_with(c), "Should accept '{}' to close tag", c);
assert!(validator.append_char(c).is_ok(), "Failed to append '{}' to close tag", c);
assert!(
validator.can_continue_with(c),
"Should accept '{}' to close tag",
c
);
assert!(
validator.append_char(c).is_ok(),
"Failed to append '{}' to close tag",
c
);
}
// Check if we've reached EOF
@@ -477,7 +589,8 @@ mod diagnostic_tests {
// Test for the specific error patterns seen in the test output
#[test]
fn test_specific_error_cases() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap_or_else(|_| get_simple_schema());
let schema_text = std::fs::read_to_string("../../action_schema.xsd")
.unwrap_or_else(|_| get_simple_schema());
// Test case 1: Simple opening tag (noticed in `test_element_open`)
let input1 = "<reasoning>";
@@ -590,8 +703,12 @@ mod diagnostic_tests {
for c in open_tag.chars() {
progress.push(c);
let result = element_validator.can_continue_with(c);
println!(" Can continue with '{}' after '{}': {}",
c, &progress[..progress.len()-1], result);
println!(
" Can continue with '{}' after '{}': {}",
c,
&progress[..progress.len() - 1],
result
);
if !result {
// This is helpful to diagnose where the process fails
@@ -600,7 +717,10 @@ mod diagnostic_tests {
}
if let Err(e) = element_validator.append_char(c) {
println!(" ❌ Failed to append '{}' for element '{}': {:?}", c, name, e);
println!(
" ❌ Failed to append '{}' for element '{}': {:?}",
c, name, e
);
break;
}
}

134
lib/xml_schema_validator/src/py_xml_logits_processor_core.rs
View File

@@ -1,6 +1,6 @@
use crate::*;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use crate::*;
/// A minimal LogitsProcessor that enforces valid XML according to a schema.
#[pyclass]
@@ -18,7 +18,10 @@ impl XmlLogitsProcessorCore {
Ok(xml_schema_validator) => xml_schema_validator,
Err(e) => return Err(pyo3::exceptions::PyValueError::new_err(e.to_string())),
};
let tokens: Vec<(Py<PyAny>, String)> = tokens.iter().map(|(id, value)| (id.unbind(), value.unbind().to_string())).collect();
let tokens: Vec<(Py<PyAny>, String)> = tokens
.iter()
.map(|(id, value)| (id.unbind(), value.unbind().to_string()))
.collect();
let mut trie = char_trie::CharTrie::new();
for (token_id, token_text) in tokens.iter() {
trie.insert(token_text, token_id.clone_ref(py));
@@ -38,7 +41,9 @@ impl XmlLogitsProcessorCore {
/// Get a list of tokens that would lead to invalid XML
fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult<Vec<PyObject>> {
Ok(self.trie.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
Ok(self
.trie
.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
}
/// Check if the validator has reached the end of the XML
@@ -50,7 +55,11 @@ impl XmlLogitsProcessorCore {
fn copy(&self, py: Python<'_>) -> PyResult<Self> {
Ok(Self {
xml_schema_validator: self.xml_schema_validator.clone(),
tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect(),
tokens: self
.tokens
.iter()
.map(|(a, b)| (a.clone_ref(py), b.clone()))
.collect(),
trie: self.trie.clone_ref(py),
})
}
@@ -79,8 +88,8 @@ pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()>
#[cfg(test)]
mod tests {
use super::*;
use pyo3::Python;
use pyo3::types::PyDict;
use pyo3::Python;
fn create_test_processor<'py>(py: Python<'py>) -> PyResult<XmlLogitsProcessorCore> {
let tokens = PyDict::new(py);
@@ -116,8 +125,10 @@ mod tests {
assert!(processor.append("<reasoning>"), "Should accept opening tag");
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>test</reasoning>"),
"Should accept complete element with content");
assert!(
processor.append("<reasoning>test</reasoning>"),
"Should accept complete element with content"
);
// Test with real schema from disk if available
if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
@@ -126,9 +137,12 @@ mod tests {
tokens.set_item(1, "reasoning").unwrap();
tokens.set_item(2, ">").unwrap();
let mut processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
assert!(processor.append("<reasoning>"),
"Should accept <reasoning> with actual schema");
let mut processor =
XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
assert!(
processor.append("<reasoning>"),
"Should accept <reasoning> with actual schema"
);
}
});
}
@@ -139,11 +153,16 @@ mod tests {
let mut processor = create_test_processor(py).unwrap();
// Test invalid XML fragments
assert!(!processor.append("<invalid>"), "Should reject invalid element");
assert!(
!processor.append("<invalid>"),
"Should reject invalid element"
);
let mut processor = create_test_processor(py).unwrap();
assert!(!processor.append("<reasoning></invalid>"),
"Should reject mismatched tags");
assert!(
!processor.append("<reasoning></invalid>"),
"Should reject mismatched tags"
);
});
}
@@ -158,18 +177,29 @@ mod tests {
// After complete document, should be at EOF
let mut processor = create_test_processor(py).unwrap();
// Use a properly formed XML document that matches the schema
assert!(processor.append("<reasoning>test</reasoning>"),
"Should accept complete document");
assert!(
processor.append("<reasoning>test</reasoning>"),
"Should accept complete document"
);
// DEBUG: Print the token state to see why EOF detection fails
println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens);
println!(
"Current tokens: {:?}",
processor.xml_schema_validator.current_tokens
);
assert!(processor.eof().unwrap(), "Should be at EOF after complete document");
assert!(
processor.eof().unwrap(),
"Should be at EOF after complete document"
);
// After opening tag, should not be at EOF
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>"), "Should accept opening tag");
assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag");
assert!(
!processor.eof().unwrap(),
"Should not be at EOF after opening tag"
);
});
}
@@ -179,18 +209,28 @@ mod tests {
let processor = create_test_processor(py).unwrap();
// At start, only '<' is valid
assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start");
assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start");
assert!(
processor.can_continue_with("<").unwrap(),
"Should accept '<' at start"
);
assert!(
!processor.can_continue_with("a").unwrap(),
"Should reject 'a' at start"
);
// Test after opening tag
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>"), "Should accept opening tag");
// In mixed content, any character should be valid
assert!(processor.can_continue_with("a").unwrap(),
"Should accept 'a' in mixed content");
assert!(processor.can_continue_with("<").unwrap(),
"Should accept '<' in mixed content");
assert!(
processor.can_continue_with("a").unwrap(),
"Should accept 'a' in mixed content"
);
assert!(
processor.can_continue_with("<").unwrap(),
"Should accept '<' in mixed content"
);
});
}
@@ -203,8 +243,11 @@ mod tests {
let copy = processor.copy(py).unwrap();
// Original and copy should behave the same
assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(),
"Copy should have same EOF state");
assert_eq!(
processor.eof().unwrap(),
copy.eof().unwrap(),
"Copy should have same EOF state"
);
// Modify copy, original should be unchanged
let mut copy = processor.copy(py).unwrap();
@@ -220,8 +263,10 @@ mod tests {
let processor = create_test_processor(py).unwrap();
let names = processor.get_element_names().unwrap();
assert!(names.contains(&"reasoning".to_string()),
"Should contain 'reasoning' element");
assert!(
names.contains(&"reasoning".to_string()),
"Should contain 'reasoning' element"
);
assert_eq!(names.len(), 1, "Should have exactly one element");
let tokens = PyDict::new(py);
@@ -235,8 +280,10 @@ mod tests {
println!("Elements in schema: {:?}", names);
// Check if reasoning exists
assert!(names.contains(&"reasoning".to_string()),
"Real schema should contain 'reasoning'");
assert!(
names.contains(&"reasoning".to_string()),
"Real schema should contain 'reasoning'"
);
});
}
@@ -270,13 +317,16 @@ mod tests {
tokens.set_item(10, ">").unwrap();
// Create processor with this schema and tokens
let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
// Test basic operations to verify core functionality works
let element_names = processor.get_element_names().unwrap();
println!("Element names in test schema: {:?}", element_names);
assert!(element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'");
assert!(
element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'"
);
// Now try to append each character of "<reasoning>" one by one
let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>'];
@@ -287,11 +337,16 @@ mod tests {
full_text.push(c);
let fragment = c.to_string();
let result = processor.can_continue_with(&fragment).unwrap();
println!("Can continue with '{}' after '{}': {}",
c, &full_text[..full_text.len()-1], result);
println!(
"Can continue with '{}' after '{}': {}",
c,
&full_text[..full_text.len() - 1],
result
);
// Create a new processor to test appending the fragment built so far
let mut fresh_processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let mut fresh_processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let append_result = fresh_processor.append(&full_text);
println!("Appending full text '{}': {}", full_text, append_result);
}
@@ -300,7 +355,8 @@ mod tests {
if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") {
println!("\nTesting with actual schema from disk:");
let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let element_names = processor.get_element_names().unwrap();
println!("Element names in actual schema: {:?}", element_names);
@@ -316,7 +372,9 @@ mod tests {
let mut text = String::new();
for c in "<reasoning>".chars() {
text.push(c);
let result = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap()
let result =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema)
.unwrap()
.append(&text);
println!("Appending '{}' to fresh processor: {}", text, result);
}

2
lib/xml_schema_validator/src/py_xml_schema_validator.rs
View File

@@ -1,5 +1,5 @@
use pyo3::prelude::*;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
/// Python wrapper for XmlValidator
#[pyclass]

61
lib/xml_schema_validator/src/schema.rs
View File

@@ -1,6 +1,14 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsSchema {
#[serde(rename = "@xmlns:xs")]
pub xmlns_xs: String,
@@ -12,7 +20,15 @@ pub struct XsSchema {
pub xs_element: Vec<XsElement>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsElement {
#[serde(rename = "@name")]
pub name: String,
@@ -22,7 +38,15 @@ pub struct XsElement {
pub xs_complex_type: XsComplexType,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsComplexType {
#[serde(rename = "@mixed")]
pub mixed: Option<String>,
@@ -34,7 +58,15 @@ pub struct XsComplexType {
pub xs_sequence: Option<XsSequence>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsAttribute {
#[serde(rename = "@name")]
pub name: String,
@@ -44,7 +76,15 @@ pub struct XsAttribute {
pub xs_attribute_use: String,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsSequence {
#[serde(rename = "$text")]
pub text: Option<String>,
@@ -52,7 +92,15 @@ pub struct XsSequence {
pub xs_any: XsAny,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsAny {
#[serde(rename = "@minOccurs")]
pub min_occurs: String,
@@ -61,4 +109,3 @@ pub struct XsAny {
#[serde(rename = "@processContents")]
pub process_contents: String,
}

102
lib/xml_schema_validator/src/token/attribute_equals.rs Normal file
View File

@@ -0,0 +1,102 @@
use crate::*;
use std::sync::Arc;
/// Represents the equals sign after an XML attribute name
#[derive(Clone, Debug)]
pub struct AttributeEquals {
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
}
impl AttributeEquals {
pub fn new(
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
) -> Self {
Self {
element,
used_attributes,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '"' {
// Opening quote starts the attribute value
return vec![Token::AttributeValue(AttributeValue::new(
Arc::clone(&self.element),
self.used_attributes,
))];
} else if c.is_whitespace() {
// Allow whitespace between equals sign and opening quote
return vec![Token::AttributeEquals(self)];
} else {
// Any other character is invalid
return vec![];
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attribute_equals_quote() {
let values = [
"<delete id=\"",
"<single timeout=\"",
"<single limit=\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_attribute_equals_with_whitespace() {
let values = [
"<delete id= \"",
"<single timeout = \"",
"<single limit \t = \n \"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_after_equals() {
let values = [
"<delete id=a", // Only quote should be accepted after equals
"<single timeout=<", // Invalid character after equals
"<single timeout=1", // Unquoted numbers not supported yet
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

257
lib/xml_schema_validator/src/token/attribute_name.rs
View File

@@ -1,196 +1,165 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents an attribute name of an XML element
/// Represents an XML attribute name that is in the process of being parsed
#[derive(Clone, Debug)]
pub struct AttributeName {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// Buffer containing the attribute name characters processed so far
buffer: String,
/// Track which attributes have already been set for this element
attributes_set: Vec<String>,
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
}
impl AttributeName {
/// Create a new AttributeName with an empty buffer
pub fn new(element: Arc<schema::XsElement>) -> Self {
pub fn new(first_char: char, element: Arc<schema::XsElement>) -> Self {
Self {
buffer: first_char.to_string(),
element,
buffer: String::new(),
attributes_set: Vec::new(),
used_attributes: Vec::new(),
}
}
/// Create a new AttributeName with the given buffer
pub fn with_buffer(element: Arc<schema::XsElement>, buffer: String, attributes_set: Vec<String>) -> Self {
pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
Self {
buffer: first_char.to_string(),
element,
buffer,
attributes_set,
used_attributes,
}
}
/// Create a new AttributeName with attributes already set
pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element,
buffer: String::new(),
attributes_set,
}
pub fn append(mut self, c: char) -> Vec<Token> {
if c == '=' {
// Check if we've matched a valid attribute name
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if let Some(attribute) = attributes
.iter()
.find(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.contains(&attribute) {
return vec![];
}
// Add this attribute to the used set
self.used_attributes.push(attribute.clone());
return vec![Token::AttributeEquals(AttributeEquals::new(
Arc::clone(&self.element),
self.used_attributes,
))];
}
}
return vec![];
} else if c.is_whitespace() {
// Check if we've matched a valid attribute name and allow whitespace after it
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.any(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
return vec![];
}
/// Append a character to the attribute name and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
if c.is_whitespace() && self.buffer.is_empty() {
return vec![Token::AttributeName(self)];
} else if c == '=' {
// Found the equals sign, transition to attribute value
// We need to verify that this attribute name is valid for this element
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.name == self.buffer {
// Valid attribute, transition to attribute value
return vec![Token::AttributeValue(AttributeValue::new(
Arc::clone(&self.element),
self.buffer,
self.attributes_set,
))];
}
}
}
// Invalid attribute name
vec![]
return vec![];
} else {
// Continue building the attribute name
let new_buffer = self.buffer + &c.to_string();
// Add character to buffer and check if we're still building a valid attribute name
let new_buffer = self.buffer.clone() + &c.to_string();
// Check if this could be a valid attribute for this element
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.name.starts_with(&new_buffer) {
// This could be a valid attribute, continue parsing
return vec![Token::AttributeName(AttributeName::with_buffer(
Arc::clone(&self.element),
new_buffer,
self.attributes_set.clone(),
))];
}
// Check if any attribute name starts with our buffer
if attributes
.iter()
.any(|attribute| attribute.name.starts_with(&new_buffer))
{
return vec![Token::AttributeName(AttributeName {
buffer: new_buffer,
element: Arc::clone(&self.element),
used_attributes: self.used_attributes,
})];
}
}
// No matching attribute found
vec![]
// Character doesn't match what we expect for this attribute name
return vec![];
}
}
}
#[cfg(test)]
mod attribute_name_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_attribute_name_append() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Build attribute name character by character
let mut current_token = token;
for c in "id".chars() {
let next_tokens = current_token.append(c);
assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c);
match &next_tokens[0] {
Token::AttributeName(next) => current_token = next.clone(),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
// Test equals sign
let next_tokens = current_token.append('=');
assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name");
match &next_tokens[0] {
Token::AttributeValue(_) => (),
_ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
fn test_valid_attribute_name() {
let values = [
"<delete id=",
"<single timeout=",
"<single limit=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_attribute_name_whitespace() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Test with whitespace
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name");
match &next_tokens[0] {
Token::AttributeName(_) => (),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
fn test_attribute_name_with_whitespace() {
let values = [
"<delete id =",
"<single timeout =",
"<single limit\t=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_attribute_name() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Test with invalid attribute name
let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute
assert!(next_tokens.is_empty(), "Should reject invalid attribute name");
let values = [
"<delete invalid=",
"<single wrongattr=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
#[test]
fn test_attribute_tracking() {
let element = create_element_with_attributes();
fn test_duplicate_attribute() {
let mut validator = crate::tests::example_validator();
// First attribute is valid
validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
// Create AttributeName with pre-existing attributes
let attributes_set = vec!["other".to_string()];
let token = AttributeName::with_attributes(Arc::clone(&element), attributes_set.clone());
// Build attribute name and transition to AttributeValue
let mut current_token = token;
for c in "id".chars() {
let next_tokens = current_token.append(c);
match &next_tokens[0] {
Token::AttributeName(next) => current_token = next.clone(),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
// Test equals sign
let next_tokens = current_token.append('=');
match &next_tokens[0] {
Token::AttributeValue(value) => {
// Verify that pre-existing attributes are preserved
assert!(value.attributes_set.contains(&"other".to_string()),
"Pre-existing attribute should be preserved");
},
_ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
}
// Try to add the same attribute again
let result = validator.append("id=");
assert!(result.is_err(), "Duplicate attribute should be rejected");
}
}

334
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -1,4 +1,3 @@
use std::sync::Arc;
use crate::*;
/// Represents an attribute value of an XML element
@@ -6,248 +5,187 @@ use crate::*;
pub struct AttributeValue {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// The name of the attribute being processed
attribute_name: String,
/// Buffer containing the attribute value characters processed so far
buffer: String,
/// Whether we've encountered the opening quote
in_quotes: bool,
/// Whether we've seen the closing quote
closed: bool,
/// Track which attributes have already been set for this element
pub attributes_set: Vec<String>,
pub attributes_set: Vec<schema::XsAttribute>,
}
impl AttributeValue {
/// Create a new AttributeValue with a list of attributes already set
pub fn new(element: Arc<schema::XsElement>, attribute_name: String, attributes_set: Vec<String>) -> Self {
pub fn new(
element: Arc<schema::XsElement>,
attributes_set: Vec<schema::XsAttribute>,
) -> Self {
Self {
element,
attribute_name,
buffer: String::new(),
in_quotes: false,
closed: false,
buffer: "\"".to_string(),
attributes_set,
}
}
/// Append a character to the attribute value and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
if self.closed {
// We've already closed the attribute value with a quote
// Add this attribute to the list of attributes set
let mut new_attributes = self.attributes_set.clone();
if !new_attributes.contains(&self.attribute_name) {
new_attributes.push(self.attribute_name.clone());
pub fn append(mut self, c: char) -> Vec<Token> {
// Handle escaped characters
if Self::in_escape_sequence(&self.buffer) {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
} else if self.value_closed() {
return self.next_tokens(c);
} else if '"' == c {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
} else {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
}
}
/// Checks if an XML character entity reference is in progress and correctly terminated
/// Returns true if the sequence is complete (ends with a semicolon)
pub fn in_escape_sequence(buffer: &str) -> bool {
if let Some(amp_pos) = buffer.rfind('&') {
if buffer[amp_pos..].find(';').is_some() {
return false;
} else {
return true;
}
}
false
}
fn value_closed(&self) -> bool {
if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") {
let trimmed = self.buffer.trim();
let trimmed = &trimmed[..trimmed.len() - 2];
if Self::in_escape_sequence(trimmed) {
false
} else {
true
}
} else {
false
}
}
fn next_tokens(mut self, c: char) -> Vec<Token> {
let all_required_attributes_set: bool = self
.element
.as_ref()
.xs_complex_type
.xs_attribute
.as_deref()
.unwrap()
.iter()
.all(|attr| {
if attr.xs_attribute_use == "required" {
self.attributes_set.contains(attr)
} else {
true
}
});
let all_attributes_set: bool = self
.element
.as_ref()
.xs_complex_type
.xs_attribute
.as_deref()
.unwrap()
.iter()
.all(|attr| self.attributes_set.contains(attr));
if c.is_whitespace() {
// Check if there are more attributes to parse
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// If there are other required attributes that haven't been set yet
for attr in attributes {
if attr.name != self.attribute_name && attr.xs_attribute_use == "required" && !new_attributes.contains(&attr.name) {
return vec![Token::AttributeName(AttributeName::with_attributes(
Arc::clone(&self.element),
new_attributes,
))];
}
}
}
// No more required attributes, whitespace after attribute value, continue to next state
return vec![
Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
];
} else if c == '>' {
// End of opening tag
return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
} else if c == '/' {
// Beginning of self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
self.buffer.push(c);
vec![Token::AttributeValue(self)]
} else {
// Unexpected character after attribute value
return vec![];
let mut tokens = vec![];
if all_required_attributes_set {
if '/' == c {
tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
} else if '>' == c {
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
}
}
if !self.in_quotes {
if c.is_whitespace() {
// Skip whitespace before the quotes
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: false,
closed: false,
attributes_set: self.attributes_set,
})];
} else if c == '"' || c == '\'' {
// Start of quotes
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: true,
closed: false,
attributes_set: self.attributes_set,
})];
} else {
// Unexpected character before quotes
return vec![];
}
} else {
// We're inside quotes
if c == '"' || c == '\'' {
// End of quotes, validate the attribute value
// In a real implementation, we'd check if the value is valid for this attribute type
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: true,
closed: true,
attributes_set: self.attributes_set,
})];
} else {
// Continue building the attribute value
let new_buffer = self.buffer + &c.to_string();
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: new_buffer,
in_quotes: true,
closed: false,
attributes_set: self.attributes_set,
})];
if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
tokens.push(Token::AttributeName(AttributeName::new_with_used(
c,
self.element,
self.attributes_set,
)));
}
tokens
}
}
}
#[cfg(test)]
mod attribute_value_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_attribute_value_quotes() {
let element = create_element_with_attributes();
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
fn test_unfinished_attribute_value() {
let values = [
"<delete id=\"12",
"<single timeout=\"1.",
"<single limit=\"50",
];
// Test with opening quote
let next_tokens = token.append('"');
assert!(!next_tokens.is_empty(), "Should accept opening quote");
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
let token = next_tokens[0].clone();
match token.clone() {
Token::AttributeValue(value) => {
assert!(value.in_quotes, "Should mark as in quotes");
assert!(!value.closed, "Should not be closed yet");
},
_ => panic!("Expected AttributeValue, got {:?}", token),
}
// Test with content inside quotes
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept content inside quotes");
let token = next_tokens[0].clone();
// Test with closing quote
let next_tokens = token.append('"');
assert!(!next_tokens.is_empty(), "Should accept closing quote");
let token = next_tokens[0].clone();
match token {
Token::AttributeValue(value) => {
assert!(value.closed, "Should be closed");
},
_ => panic!("Expected AttributeValue, got {:?}", token),
// Should still be in AttributeValue state
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::AttributeValue(_))),
"{value}"
);
}
}
#[test]
fn test_after_attribute_value_closed() {
let element = create_element_with_attributes();
let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
token.in_quotes = true;
token.closed = true;
fn test_type_validation() {
// Test integer validation
let mut validator = crate::tests::example_validator();
validator.append("<single limit=\"123\"").expect("Valid integer");
// Test with space after closed attribute value
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after closed attribute");
// Test float validation
validator = crate::tests::example_validator();
validator.append("<single timeout=\"1.5\"").expect("Valid float");
// Test with > after closed attribute value
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
let mut token = token;
token.in_quotes = true;
token.closed = true;
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
match &next_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
// Test string validation
validator = crate::tests::example_validator();
validator.append("<delete id=\"abc123\"").expect("Valid string");
}
// Test with / after closed attribute value (for self-closing)
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
let mut token = token;
token.in_quotes = true;
token.closed = true;
#[test]
fn test_multiple_attributes() {
let values = [
"<single limit=\"100\" timeout=\"1.5\"",
"<single timeout=\"0.5\" limit=\"200\"",
];
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
}
}
#[test]
fn test_attribute_tracking() {
let element = create_element_with_attributes();
let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
token.in_quotes = true;
token.closed = true;
fn test_attribute_value_escape_chars() {
// Test handling of escaped characters in attribute values
let values = [
"<delete id=\"escaped&quot;quote\"",
"<delete id=\"escaped&amp;amp\"",
];
// After attribute is closed, it should be tracked
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
match &next_tokens[0] {
Token::ElementOpenEnd(element_open) => {
// Verify that the id attribute is tracked
assert!(element_open.attributes_set.contains(&"id".to_string()),
"Attribute 'id' should be tracked");
},
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
}
}
}

40
lib/xml_schema_validator/src/token/element_close_name.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents a closing element tag that is in the process of being parsed
/// after the '</' and now parsing the element name
@@ -14,10 +14,7 @@ pub struct ElementCloseName {
impl ElementCloseName {
/// Create a new instance with the given buffer
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self {
element,
buffer,
}
Self { element, buffer }
}
pub fn append(self, c: char) -> Vec<Token> {
@@ -40,10 +37,10 @@ impl ElementCloseName {
}
#[cfg(test)]
mod element_close_name_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
@@ -73,12 +70,15 @@ mod element_close_name_tests {
// Test with next character in name
let next_tokens = token.append('e');
assert!(!next_tokens.is_empty(), "Should accept next character in name");
assert!(
!next_tokens.is_empty(),
"Should accept next character in name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
},
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
@@ -90,19 +90,28 @@ mod element_close_name_tests {
// Test with last character of name
let next_tokens = token.append('g');
assert!(!next_tokens.is_empty(), "Should accept last character of name");
assert!(
!next_tokens.is_empty(),
"Should accept last character of name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'");
},
assert_eq!(
name.buffer, "reasoning",
"Buffer should contain 'reasoning'"
);
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
// Test with > after complete name
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after complete name");
assert!(
!next_tokens.is_empty(),
"Should accept '>' after complete name"
);
match &next_tokens[0] {
Token::EndOfFile(_) => (),
@@ -117,7 +126,10 @@ mod element_close_name_tests {
// Test with whitespace after complete name
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name");
assert!(
!next_tokens.is_empty(),
"Should accept whitespace after complete name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),

13
lib/xml_schema_validator/src/token/element_close_start.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents the start of an XML element closing tag (the '</' sequence)
#[derive(Clone, Debug)]
@@ -35,10 +35,10 @@ impl ElementCloseStart {
}
#[cfg(test)]
mod element_close_start_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
@@ -73,7 +73,7 @@ mod element_close_start_tests {
match &next_tokens[0] {
Token::ElementCloseStart(start) => {
assert!(start.has_slash, "Should mark as having slash");
},
}
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
@@ -86,7 +86,10 @@ mod element_close_start_tests {
// Test with first character of element name
let next_tokens = token.append('r');
assert!(!next_tokens.is_empty(), "Should accept first letter of element name");
assert!(
!next_tokens.is_empty(),
"Should accept first letter of element name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),

121
lib/xml_schema_validator/src/token/element_open_end.rs
View File

@@ -1,68 +1,34 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents a fully opened XML element, containing the schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenEnd {
element: Arc<schema::XsElement>,
// Track which attributes have been set for this element
pub attributes_set: Vec<String>,
}
impl ElementOpenEnd {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
attributes_set: Vec::new(),
}
}
pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element,
attributes_set,
}
}
pub fn append(self, c: char) -> Vec<Token> {
// Check if all required attributes are present before proceeding
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
// Missing a required attribute, this is an error
return vec![];
}
}
}
// Check if the element is a mixed content element that allows text
let allows_text = self.element.xs_complex_type.mixed
if '<' == c {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Check if this element allows mixed content
let is_mixed = self.element.xs_complex_type.mixed
.as_ref()
.map(|mixed| mixed == "true")
.map(|val| val == "true")
.unwrap_or(false);
if allows_text {
if c.is_whitespace() {
// Handle whitespace in mixed content
vec![
Token::TextContent(TextContent::new(Arc::clone(&self.element))),
Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))
]
} else if c == '<' {
// Start of a closing tag or nested element
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
if is_mixed || c.is_whitespace() {
// Allow text content for mixed elements, or whitespace for any element
vec![Token::TextContent(TextContent::new(self.element))]
} else {
// Text content
vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))]
}
} else {
// Element doesn't allow text content, only expect closing tag
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else if c.is_whitespace() {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Unexpected content in non-mixed element
// Reject text content for non-mixed elements
vec![]
}
}
@@ -70,10 +36,10 @@ impl ElementOpenEnd {
}
#[cfg(test)]
mod element_open_end_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
@@ -117,26 +83,6 @@ mod element_open_end_tests {
Arc::new(element)
}
fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_append_text_in_mixed_content() {
let element = create_test_element();
@@ -144,7 +90,10 @@ mod element_open_end_tests {
// Test appending text in mixed content element
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should allow text content in mixed element");
assert!(
!next_tokens.is_empty(),
"Should allow text content in mixed element"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
@@ -174,7 +123,10 @@ mod element_open_end_tests {
// Test whitespace in mixed content
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content");
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in mixed content"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
@@ -189,32 +141,23 @@ mod element_open_end_tests {
// Test appending text in non-mixed content (should reject)
let next_tokens = token.clone().append('a');
assert!(next_tokens.is_empty(), "Should reject text in non-mixed element");
assert!(
next_tokens.is_empty(),
"Should reject text in non-mixed element"
);
// Test whitespace (should accept leading to closing tag)
let next_tokens = token.clone().append(' ');
assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed");
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in non-mixed"
);
// Test closing tag start
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed");
}
#[test]
fn test_required_attributes() {
let element = create_element_with_required_attributes();
// No attributes set yet
let token = ElementOpenEnd::new(Arc::clone(&element));
let next_tokens = token.append('<');
assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
// With required attribute set
let token = ElementOpenEnd::with_attributes(
Arc::clone(&element),
vec!["id".to_string()]
assert!(
!next_tokens.is_empty(),
"Should allow closing tag start in non-mixed"
);
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag with required attributes set");
}
}

340
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
@@ -9,72 +9,88 @@ pub struct ElementOpenName {
}
impl ElementOpenName {
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Result<Self> {
if element.name.starts_with(buffer.as_str()) {
Ok(Self {
buffer,
element,
})
} else {
Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name)))
}
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self { buffer, element }
}
pub fn append(self, c: char) -> Vec<Token> {
if c.is_whitespace() {
pub fn append(mut self, c: char) -> Vec<Token> {
if c == '>' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, handle whitespace after name
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if !attributes.is_empty() {
// Transition to attribute name parsing
return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))];
}
}
// No attributes
return vec![
Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new()),
Token::ElementOpenName(self),
];
// Check if they are required
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
{
return vec![];
} else {
// Whitespace in the middle of an element name is invalid
return vec![];
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
&self.element,
)))];
}
} else if c == '>' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, end of opening tag
// Check if this element has required attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" {
// Found a required attribute but no attributes are set
// Return an empty vector to indicate error
return vec![];
}
}
}
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
} else {
// '>' before full element name is invalid
return vec![];
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
&self.element,
)))];
}
} else if c == '/' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
} else {
// '/' before full element name is invalid
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if they are required
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
{
return vec![];
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else if c.is_whitespace() {
// Check if we've matched the full element name
if self.buffer.contains(&self.element.name) {
if let Some(last) = self.buffer.chars().last() {
if last.is_whitespace() {
return vec![Token::ElementOpenName(self)];
}
}
self.buffer.push(c);
return vec![Token::ElementOpenName(self)];
} else {
return vec![];
}
} else if !self.buffer.is_empty() && self.buffer.chars().last().unwrap().is_whitespace() {
// Check for start of attribute name
if self.buffer.contains(&self.element.name) {
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.filter(|attribute| attribute.name.starts_with(c))
.next()
.is_some()
{
return vec![Token::AttributeName(AttributeName::new(
c,
self.element.clone(),
))];
}
}
}
return vec![];
} else {
// Add character to buffer and check if we're still building a valid element name
let new_buffer = self.buffer.clone() + &c.to_string();
@@ -94,179 +110,77 @@ impl ElementOpenName {
}
#[cfg(test)]
mod element_open_name_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_append_character_by_character() {
let element = create_test_element();
// Test each character in "reasoning"
let mut buffer = String::new();
for c in "reasoning".chars() {
buffer.push(c);
let result = ElementOpenName::new(Arc::clone(&element), buffer.clone());
assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer);
}
}
#[test]
fn test_append_full_name() {
let element = create_test_element();
// Test with the full element name
let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string());
assert!(result.is_ok(), "Failed to create ElementOpenName with full element name");
let element_name = result.unwrap();
// Test transition after full name with '>'
let next_tokens = element_name.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
match &next_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_space_after_name() {
let element = create_test_element();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
// Test transition after full name with space
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after full element name");
assert!(next_tokens.len() == 4);
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
fn test_attribute_name() {
let values = [
"<delete i", // id
"<single t", // timeout
"<single l", // limit
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_valid_self_closing() {
let values = ["<stop/", "<stop /"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some());
}
#[test]
fn test_append_with_attributes() {
let element = create_element_with_attributes();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
// Test transition after full name with space (expecting attribute name)
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes");
// The space should lead to a whitespace token, which should then lead to attribute name
match &next_tokens[0] {
Token::AttributeName(_) => (),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
.is_some(),
"{value}"
);
}
}
#[test]
fn test_append_slash_after_name() {
let element = create_test_element();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
// Test transition after full name with /
let next_tokens = element_name.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/' after full element name");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
fn test_valid_closing() {
let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_continuation() {
let element = create_test_element();
// Test with a partial element name and an invalid continuation character
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character in element name");
// Test whitespace in middle of name - need to create a new instance since append consumes self
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append(' ');
assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name");
// Test > in middle of name - need to create a new instance since append consumes self
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append('>');
assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name");
}
#[test]
fn test_required_attributes_validation() {
let element = create_element_with_attributes();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
// Try to close the tag with > directly, skipping the required attribute
let next_tokens = element_name.append('>');
assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
fn test_invalid_closing() {
let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

53
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
@@ -9,17 +9,56 @@ pub struct ElementOpenStart {
impl ElementOpenStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
if let Ok(element_name) = element_name {
vec![Token::ElementOpenName(element_name)]
if self.element.name.starts_with(c) {
vec![Token::ElementOpenName(ElementOpenName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_accept_valid_element_names() {
let values = [
"<d", // delete
"<s", // stop / single
"<r", // repeat / reasoning / read_stdin
"<w", // write_stdout
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_not_accept_other() {
let values = ["< ", "<a", "</"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

142
lib/xml_schema_validator/src/token/element_self_close.rs
View File

@@ -1,54 +1,18 @@
use std::sync::Arc;
use crate::*;
/// Represents a self-closing XML element
#[derive(Clone, Debug)]
pub struct ElementSelfClose {
/// Optional reference to the element schema for attribute validation
element: Option<Arc<schema::XsElement>>,
/// Track which attributes have been set
attributes_set: Vec<String>,
}
pub struct ElementSelfClose {}
impl ElementSelfClose {
/// Create a new self-closing element
pub fn new() -> Self {
Self {
element: None,
attributes_set: Vec::new(),
}
Self {}
}
/// Create a new self-closing element with element reference and attribute tracking
pub fn new_with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element: Some(element),
attributes_set,
}
}
/// Append a character to the self-closing tag and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
// Before we close, verify that all required attributes are present
if let Some(element) = &self.element {
if c == '>' {
if let Some(attributes) = &element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
// Missing required attribute, this is an error
return vec![];
}
}
}
return vec![Token::EndOfFile(EndOfFile::new())];
}
} else if c == '>' {
// No element reference, can't validate attributes, just close
return vec![Token::EndOfFile(EndOfFile::new())];
}
if c.is_whitespace() {
vec![Token::ElementSelfClose(self)]
} else {
vec![]
}
@@ -56,89 +20,35 @@ impl ElementSelfClose {
}
#[cfg(test)]
mod element_self_close_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_self_close_with_greater_than() {
let token = ElementSelfClose::new();
// Test closing with >
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag");
assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_self_close_with_whitespace() {
let token = ElementSelfClose::new();
// Test with whitespace
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_self_close_with_invalid_char() {
let token = ElementSelfClose::new();
// Test with invalid character
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag");
}
#[test]
fn test_required_attributes_validation() {
let element = create_element_with_required_attributes();
// Without required attributes
let token = ElementSelfClose::new_with_attributes(Arc::clone(&element), vec![]);
let next_tokens = token.append('>');
assert!(next_tokens.is_empty(), "Should reject closing without required attributes");
// With required attributes
let token = ElementSelfClose::new_with_attributes(
Arc::clone(&element),
vec!["id".to_string()]
fn test_valid() {
let values = ["<stop/>"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::EndOfFile(_)))
.next()
.is_some(),
"{value}"
);
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept closing with required attributes");
}
}
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
#[test]
fn test_invalid() {
let values = ["<stop/?", "<stop/a", "<stop/ "];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

5
lib/xml_schema_validator/src/token/end_of_file.rs
View File

@@ -2,8 +2,7 @@ use crate::*;
/// Represents the final state at the end of an XML file
#[derive(Clone, Debug)]
pub struct EndOfFile {
}
pub struct EndOfFile {}
impl EndOfFile {
pub fn new() -> Self {
@@ -16,7 +15,7 @@ impl EndOfFile {
}
#[cfg(test)]
mod end_of_file_tests {
mod tests {
use super::*;
#[test]

73
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -1,46 +1,50 @@
mod attribute_equals;
mod attribute_name;
mod attribute_value;
mod element_close_name;
mod element_close_start;
mod element_open_name;
mod element_open_end;
mod element_open_name;
mod element_open_start;
mod element_self_close;
mod end_of_file;
mod start_of_file;
mod text_content;
mod attribute_name;
mod attribute_value;
mod element_self_close;
pub use attribute_equals::AttributeEquals;
pub use attribute_name::AttributeName;
pub use attribute_value::AttributeValue;
pub use element_close_name::ElementCloseName;
pub use element_close_start::ElementCloseStart;
pub use element_open_name::ElementOpenName;
pub use element_open_end::ElementOpenEnd;
pub use element_open_name::ElementOpenName;
pub use element_open_start::ElementOpenStart;
pub use element_self_close::ElementSelfClose;
pub use end_of_file::EndOfFile;
pub use start_of_file::StartOfFile;
pub use text_content::TextContent;
pub use attribute_name::AttributeName;
pub use attribute_value::AttributeValue;
pub use element_self_close::ElementSelfClose;
/// Represents the different states of XML token parsing
#[derive(Clone, Debug)]
pub enum Token {
AttributeEquals(AttributeEquals),
AttributeName(AttributeName),
AttributeValue(AttributeValue),
ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart),
ElementOpenName(ElementOpenName),
ElementOpenEnd(ElementOpenEnd),
ElementOpenName(ElementOpenName),
ElementOpenStart(ElementOpenStart),
ElementSelfClose(ElementSelfClose),
EndOfFile(EndOfFile),
StartOfFile(StartOfFile),
TextContent(TextContent),
AttributeName(AttributeName),
AttributeValue(AttributeValue),
ElementSelfClose(ElementSelfClose),
}
impl Token {
pub fn append(self, c: char) -> Vec<Token> {
match self {
Token::AttributeEquals(attribute_equals) => attribute_equals.append(c),
Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementOpenName(element_name) => element_name.append(c),
@@ -66,10 +70,10 @@ impl Token {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
pub fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
@@ -90,26 +94,6 @@ mod tests {
Arc::new(element)
}
#[test]
fn test_start_of_file_transition() {
let element = create_test_element();
let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
// Should only accept '<'
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
// Should reject other characters
let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should reject 'a'");
}
#[test]
fn test_element_start_transition() {
let element = create_test_element();
@@ -133,7 +117,8 @@ mod tests {
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
let token =
Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
// Continue building name
let next_tokens = token.append('e');
@@ -146,7 +131,9 @@ mod tests {
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&element,
)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
@@ -155,8 +142,11 @@ mod tests {
}
current_tokens = new_tokens;
assert!(!current_tokens.is_empty(),
"Should have valid transition after appending '{}'", c);
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Check final state
@@ -264,8 +254,11 @@ mod tests {
}
current_tokens = new_tokens;
assert!(!current_tokens.is_empty(),
"Should have valid transition after appending '{}'", c);
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Final > to close - clone the token to avoid moving out of the vector

67
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
#[derive(Clone, Debug)]
@@ -14,7 +14,9 @@ impl StartOfFile {
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&self.element,
)))]
} else {
vec![]
}
@@ -22,54 +24,29 @@ impl StartOfFile {
}
#[cfg(test)]
mod start_of_file_tests {
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
#[test]
fn test_accept_open_start_token() {
let mut validator = crate::tests::example_validator();
validator.append("<").unwrap();
assert!(!validator.current_tokens.is_empty());
assert!(validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenStart(_)))
.next()
.is_some());
}
#[test]
fn test_start_of_file_less_than() {
let element = create_test_element();
let token = StartOfFile::new(Arc::clone(&element));
// Test with <
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
fn test_not_accept_other() {
let values = ["a", " ", "\t", ">", "/"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
#[test]
fn test_start_of_file_invalid() {
let element = create_test_element();
let token = StartOfFile::new(Arc::clone(&element));
// Test with invalid character
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}

16
lib/xml_schema_validator/src/token/text_content.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::*;
use std::sync::Arc;
/// Represents the text content within an XML element
#[derive(Clone, Debug)]
@@ -9,18 +9,18 @@ pub struct TextContent {
impl TextContent {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
vec![Token::TextContent(TextContent::new(
vec![Token::ElementCloseStart(ElementCloseStart::new(
Arc::clone(&self.element),
))]
} else {
vec![Token::TextContent(TextContent::new(Arc::clone(
&self.element,
)))]
}
}
}
@@ -28,8 +28,8 @@ impl TextContent {
#[cfg(test)]
mod text_content_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {

62
lib/xml_schema_validator/tests/test_logits_processor.py
View File

@@ -1,13 +1,12 @@
from transformers import AutoTokenizer
from xml_schema_validator import XmlLogitsProcessor
import os
import time
import torch
model_path = "../../model"
api_token = os.environ["SIA_HF_API_KEY"]
xml_schema_actions = open("../../action_schema.xsd").read()
xml_schema_actions = open("example_schema.xsd").read()
xml_schema_only_root_node = """<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
<xs:element name="root">
@@ -60,59 +59,26 @@ def test_token_masking():
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
# Create dummy input_ids and scores
# Process empty input to set prompt length
input_ids = torch.tensor([tokenizer.encode("")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
# Process valid continuation
input_ids = torch.tensor([tokenizer.encode("<reasoning>")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
# Process the scores
processed_scores = processor(input_ids, scores)
# Verify that valid continuations still have positive scores
assert processed_scores[0, tokenizer.encode(" ", add_special_tokens=False)[0]] > -float('inf')
space_token = tokenizer.encode(" ", add_special_tokens=False)[0]
assert processed_scores[0, space_token] > -float('inf')
# Verify that invalid continuations are masked
assert processed_scores[0, tokenizer.encode("<invalid>", add_special_tokens=False)[0]] == -float('inf')
def test_performance():
"""Test performance with larger XML documents"""
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
# Generate a larger XML document
large_xml = "<reasoning>" + "Test content. " * 100 + "</reasoning>"
# Measure time to process
start_time = time.time()
processor.core.append(large_xml)
processing_time = time.time() - start_time
print(f"Processing time for large XML: {processing_time:.4f} seconds")
# You might want to assert that processing time is below a threshold
def test_subword_tokens():
"""Test handling of subword tokens that might split XML tags"""
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
schema_text = open("../../action_schema.xsd").read()
processor = XmlLogitsProcessor(tokenizer, schema_text)
# Find some XML tag that gets split into multiple tokens by your tokenizer
tag = "<reasoning>"
tokens = tokenizer.encode(tag, add_special_tokens=False)
if len(tokens) > 1:
print(f"Tag '{tag}' is split into {len(tokens)} tokens")
# Test that the processor can handle these split tokens
import torch
input_ids = torch.tensor([[tokens[0]]])
scores = torch.ones((1, len(tokenizer)))
input_ids = torch.tensor([tokenizer.encode("</invalid>")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
# The next token in the sequence should have a high score
assert processed_scores[0, tokens[1]] > -float('inf')
assert processed_scores[0, tokenizer.eos_token_id] == 1
assert processed_scores[0, space_token] == -float('inf')
if __name__ == "__main__":
# Run individual tests for debugging
@@ -120,5 +86,3 @@ if __name__ == "__main__":
test_xml_schema_parsing()
test_basic_xml_validation()
test_token_masking()
test_performance()
test_subword_tokens()